@malloy-publisher/server 0.0.199 → 0.0.200

package/src/heap_check.ts

@@ -0,0 +1,144 @@
+import { metrics } from "@opentelemetry/api";
+import * as v8 from "v8";
+
+import { logger } from "./logger";
+
+/**
+ * Subset of the winston logger surface this module needs. Defined
+ * structurally so tests can pass a plain object stub and so we
+ * don't take a direct dependency on the concrete winston type.
+ */
+interface HeapCheckLogger {
+   warn: (...args: unknown[]) => unknown;
+   info: (...args: unknown[]) => unknown;
+}
+
+/**
+ * Observable gauges for heap configuration so dashboards can render
+ * "configured heap ceiling" alongside the RSS / back-pressure
+ * timeseries from `PackageMemoryGovernor`. The values are observed
+ * on every scrape rather than cached — `v8.getHeapStatistics()` is
+ * cheap (a single VM call) and serving the live value avoids stale
+ * reads.
+ *
+ * Lazy init for the same reason as `query_timeout.ts`: instruments
+ * captured before `setGlobalMeterProvider` are bound to NoOp. We
+ * call `installHeapGauges` from `checkHeapConfiguration` so it
+ * runs once at startup, after the OTel SDK is up.
+ *
+ * Mirrors the governor's `publisher_*` unlabeled style.
+ */
+let heapGaugesInstalled = false;
+function installHeapGauges(): void {
+   if (heapGaugesInstalled) return;
+   heapGaugesInstalled = true;
+   const meter = metrics.getMeter("publisher");
+   meter
+      .createObservableGauge("publisher_heap_size_limit_bytes", {
+         description:
+            "V8 heap_size_limit (--max-old-space-size). Compare with PUBLISHER_MAX_MEMORY_BYTES.",
+         unit: "By",
+      })
+      .addCallback((observation) => {
+         observation.observe(v8.getHeapStatistics().heap_size_limit);
+      });
+   meter
+      .createObservableGauge("publisher_heap_used_bytes", {
+         description:
+            "Current V8 used_heap_size in bytes. Watch this alongside publisher_process_rss_bytes; the two diverge under native-allocator pressure (DuckDB, etc.).",
+         unit: "By",
+      })
+      .addCallback((observation) => {
+         observation.observe(v8.getHeapStatistics().used_heap_size);
+      });
+}
+
+/**
+ * Visible for tests; production code never calls this. Resets the
+ * lazy guard so a re-installation captures into a fresh
+ * MeterProvider.
+ */
+export function resetHeapTelemetryForTesting(): void {
+   heapGaugesInstalled = false;
+}
+
+/**
+ * Minimum V8 heap ceiling (`--max-old-space-size`) the publisher
+ * expects in production. Below this the row/byte caps from earlier
+ * steps still apply, but a single buffered model query at the
+ * default 50 MB byte cap plus the surrounding `Result` allocation
+ * can plausibly chew through the remaining headroom and OOM the
+ * process before back-pressure trips.
+ *
+ * 2 GiB is the smallest value at which the defaults are comfortably
+ * survivable. Operators running explicitly tuned-down pods (e.g. a
+ * lightweight smoke-test deploy) can ignore the warning.
+ */
+const MIN_RECOMMENDED_HEAP_BYTES = 2 * 1024 * 1024 * 1024;
+
+/**
+ * Probe `v8.getHeapStatistics().heap_size_limit` at startup and
+ * emit a single structured warning when the process is configured
+ * with a heap ceiling below {@link MIN_RECOMMENDED_HEAP_BYTES}.
+ *
+ * Why warn (not exit):
+ * - Smaller pods are legitimate (CI, local dev, smoke tests).
+ *   Hard-failing them would be hostile to those workflows.
+ * - The earlier steps already bound memory growth per request; this
+ *   is a "you probably want to know" signal, not a safety
+ *   interlock.
+ * - A warning at boot is grep-able in pod logs / dashboards and
+ *   surfaces faster than waiting for the first OOMKill.
+ *
+ * Returns the observed heap limit so the caller (server.ts startup)
+ * can also surface it in startup metrics if desired.
+ */
+/**
+ * Test seam: parameters allow injecting a fake heap-stats getter
+ * and logger so the bun/sinon "ES modules cannot be stubbed"
+ * restriction doesn't force a module-level mock. Production calls
+ * (server.ts startup) use the defaults; tests pass in stubs.
+ */
+export interface CheckHeapOptions {
+   getHeapStatistics?: () => Pick<v8.HeapInfo, "heap_size_limit">;
+   log?: HeapCheckLogger;
+}
+
+export function checkHeapConfiguration(options: CheckHeapOptions = {}): {
+   heapSizeLimitBytes: number;
+   warned: boolean;
+} {
+   // Install heap-related observable gauges on the same startup
+   // tick. Idempotent — re-calling in tests / warmup paths is
+   // safe and re-uses the same instruments.
+   installHeapGauges();
+   const getStats = options.getHeapStatistics ?? v8.getHeapStatistics;
+   const log = options.log ?? logger;
+   const stats = getStats();
+   const heapSizeLimitBytes = stats.heap_size_limit;
+   const limitMiB = Math.round(heapSizeLimitBytes / (1024 * 1024));
+   const recommendedMiB = Math.round(
+      MIN_RECOMMENDED_HEAP_BYTES / (1024 * 1024),
+   );
+
+   if (heapSizeLimitBytes < MIN_RECOMMENDED_HEAP_BYTES) {
+      log.warn(
+         `V8 heap_size_limit is ${limitMiB} MiB, below the recommended ${recommendedMiB} MiB. ` +
+            `Pass --max-old-space-size=${recommendedMiB} (or higher) on the node process to keep ` +
+            `the row/byte caps (PUBLISHER_MAX_QUERY_ROWS / PUBLISHER_MAX_RESPONSE_BYTES) within ` +
+            `safe margin. With a smaller heap, a single large query can OOM the pod before the ` +
+            `memory governor's back-pressure has a chance to trip.`,
+         {
+            heapSizeLimitBytes,
+            recommendedHeapSizeBytes: MIN_RECOMMENDED_HEAP_BYTES,
+         },
+      );
+      return { heapSizeLimitBytes, warned: true };
+   }
+
+   log.info(
+      `V8 heap_size_limit is ${limitMiB} MiB (>= recommended ${recommendedMiB} MiB).`,
+      { heapSizeLimitBytes },
+   );
+   return { heapSizeLimitBytes, warned: false };
+}

package/src/mcp/handler_utils.ts

@@ -7,6 +7,7 @@ import {
    ModelNotFoundError,
    ModelCompilationError,
    EnvironmentNotFoundError,
+   ServiceUnavailableError,
 } from "../errors";
 import {
    getNotFoundError,
@@ -132,6 +133,9 @@ export async function getModelForQuery(
          environmentName,
          false,
       );
+      // Shed load before any disk / DB work; mirrors the HTTP query
+      // controllers so MCP traffic obeys the same back-pressure rules.
+      environment.assertCanAdmitQuery();
       const pkg = await environment.getPackage(packageName, false);
       const model = pkg.getModel(modelPath);
       if (!model || model.getModelType() === "notebook") {
@@ -163,6 +167,16 @@ export async function getModelForQuery(
             `${environmentName}/${packageName}/${modelPath}`,
             error,
          );
+      } else if (error instanceof ServiceUnavailableError) {
+         // Back-pressure: don't dress this up as a 404/500. Surface the
+         // server's own message so the MCP caller knows to retry.
+         errorDetails = {
+            message: error.message,
+            suggestions: [
+               "Retry after the publisher's memory usage drops below the configured low-water mark.",
+               "If this happens repeatedly, raise PUBLISHER_MAX_MEMORY_BYTES or scale up the pod.",
+            ],
+         } satisfies ErrorDetails;
       } else {
          // Unexpected error during setup
          errorDetails = getInternalError("executeQuery (Setup)", error);

package/src/mcp/tools/execute_query_tool.ts

@@ -2,7 +2,13 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { ErrorCode, McpError } from "@modelcontextprotocol/sdk/types.js";
 import { z } from "zod";
 import type { GivenValue } from "@malloydata/malloy";
+import { getQueryTimeoutMs } from "../../config";
 import { logger } from "../../logger";
+import {
+   tryAcquireQuerySlot,
+   type QuerySlotHandle,
+} from "../../query_concurrency";
+import { runWithQueryTimeout } from "../../query_timeout";
 import { EnvironmentStore } from "../../service/environment_store";
 import { getMalloyErrorDetails, type ErrorDetails } from "../error_messages";
 import { buildMalloyUri, getModelForQuery } from "../handler_utils";
@@ -128,16 +134,30 @@ export function registerExecuteQueryTool(
          logger.info(
             `[MCP Tool executeQuery] Model found. Proceeding to execute query.`,
          );
+         // Per-pod concurrency slot. MCP shares the same slot pool
+         // as the HTTP query routes so a hot agent loop can't
+         // bypass PUBLISHER_MAX_CONCURRENT_QUERIES. `mcp:executeQuery`
+         // is a fixed label so the dashboard can separate MCP load
+         // from HTTP route load. Acquisition can throw
+         // ServiceUnavailableError; the existing catch below surfaces
+         // it as the standard MCP error-content payload.
+         let querySlot: QuerySlotHandle | null = null;
          try {
+            querySlot = tryAcquireQuerySlot("mcp:executeQuery");
             // If ad-hoc query is provided, use it directly in the 3rd arg
             if (query) {
-               const { result } = await model.getQueryResults(
-                  undefined,
-                  undefined,
-                  query,
-                  filterParams,
-                  undefined,
-                  givens as Record<string, GivenValue> | undefined,
+               const { result } = await runWithQueryTimeout(
+                  (abortSignal) =>
+                     model.getQueryResults(
+                        undefined,
+                        undefined,
+                        query,
+                        filterParams,
+                        undefined,
+                        givens as Record<string, GivenValue> | undefined,
+                        abortSignal,
+                     ),
+                  getQueryTimeoutMs(),
                );
                const { validateRenderTags } = await import(
                   "@malloydata/render-validator"
@@ -179,13 +199,18 @@ export function registerExecuteQueryTool(
 
                return { isError: false, content };
             } else if (queryName) {
-               const { result } = await model.getQueryResults(
-                  sourceName,
-                  queryName,
-                  undefined,
-                  filterParams,
-                  undefined,
-                  givens as Record<string, GivenValue> | undefined,
+               const { result } = await runWithQueryTimeout(
+                  (abortSignal) =>
+                     model.getQueryResults(
+                        sourceName,
+                        queryName,
+                        undefined,
+                        filterParams,
+                        undefined,
+                        givens as Record<string, GivenValue> | undefined,
+                        abortSignal,
+                     ),
+                  getQueryTimeoutMs(),
                );
                const { validateRenderTags } = await import(
                   "@malloydata/render-validator"
@@ -271,6 +296,11 @@ export function registerExecuteQueryTool(
                   },
                ],
             };
+         } finally {
+            // Release on every exit path — success, error, or
+            // unreachable code-path throw. `release()` is idempotent
+            // so a double-fault during cleanup can't double-decrement.
+            querySlot?.release();
          }
       },
    );

package/src/oom_guards.integration.spec.ts

@@ -0,0 +1,261 @@
+/**
+ * End-to-end integration test for the OOM-mitigation guardrails
+ * (Steps 1-6). Mounts a real Express app with the same middleware
+ * + controller wiring `server.ts` uses, then exercises each layer
+ * of the defense via HTTP requests.
+ *
+ * What's covered, in firing order:
+ *   1. Admission gate — memory governor flags back-pressure →
+ *      `ConnectionController` short-circuits with 503 BEFORE the
+ *      connector is touched.
+ *   2. Concurrency middleware — when the pod is at
+ *      `PUBLISHER_MAX_CONCURRENT_QUERIES`, new requests get 503
+ *      BEFORE the controller is invoked.
+ *   3. Timeout — when the underlying driver takes longer than
+ *      `PUBLISHER_QUERY_TIMEOUT_MS`, the request gets 504 and the
+ *      abort signal fires.
+ *   4. Row cap — when the driver returns more rows than
+ *      `PUBLISHER_MAX_QUERY_ROWS`, the request gets 413.
+ *
+ * Mocked: only the Malloy `Connection.runSQL`. Everything else
+ * (admission gate, concurrency middleware, timeout helper, cap
+ * detection, error-to-HTTP mapping) is real production code.
+ */
+
+import type { Connection, RunSQLOptions } from "@malloydata/malloy";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import express from "express";
+import sinon from "sinon";
+import request from "supertest";
+
+import { ConnectionController } from "./controller/connection.controller";
+import { internalErrorToHttpError, ServiceUnavailableError } from "./errors";
+import {
+   queryConcurrency,
+   resetActiveQueryCountForTesting,
+   resetQueryConcurrencyTelemetryForTesting,
+} from "./query_concurrency";
+import type { EnvironmentStore } from "./service/environment_store";
+
+function buildApp(
+   runSQL: sinon.SinonStub,
+   opts: { assertCanAdmitQuery?: sinon.SinonStub } = {},
+): {
+   app: express.Express;
+   assertCanAdmitQuery: sinon.SinonStub;
+   runSQL: sinon.SinonStub;
+} {
+   const fakeConnection = { runSQL } as unknown as Connection;
+   const assertCanAdmitQuery =
+      opts.assertCanAdmitQuery ?? sinon.stub().returns(undefined);
+   const fakeEnv = { assertCanAdmitQuery };
+   const fakeStore = {
+      getEnvironment: sinon.stub().resolves(fakeEnv),
+   } as unknown as EnvironmentStore;
+   const controller = new ConnectionController(fakeStore);
+   // Bypass the connector lookup; the test only cares about the
+   // guardrail chain on top of a stub.
+   sinon
+      .stub(
+         controller as unknown as {
+            getMalloyConnection: (...args: unknown[]) => Promise<Connection>;
+         },
+         "getMalloyConnection",
+      )
+      .resolves(fakeConnection);
+
+   const app = express();
+   app.use(express.json());
+
+   // Same shape as `server.ts` registers for `/sqlQuery` — middleware
+   // first, then a thin route handler that delegates to the controller.
+   app.post(
+      "/api/v0/environments/:env/connections/:conn/sqlQuery",
+      queryConcurrency(),
+      async (req, res) => {
+         try {
+            const result = await controller.getConnectionQueryData(
+               req.params.env,
+               req.params.conn,
+               req.body.sqlStatement as string,
+               (req.body.options as string | undefined) ?? "",
+            );
+            res.status(200).json(result);
+         } catch (error) {
+            const { json, status } = internalErrorToHttpError(error as Error);
+            res.status(status).json(json);
+         }
+      },
+   );
+
+   // Mirror the global error handler from `server.ts` — required
+   // for the concurrency middleware's `next(err)` path to map
+   // `ServiceUnavailableError` to a 503 instead of Express's
+   // default 500. Without this the integration test would be
+   // exercising a different error path than production.
+   app.use(
+      (
+         err: Error,
+         _req: express.Request,
+         res: express.Response,
+         _next: express.NextFunction,
+      ): void => {
+         const { json, status } = internalErrorToHttpError(err);
+         res.status(status).json(json);
+      },
+   );
+
+   return { app, assertCanAdmitQuery, runSQL };
+}
+
+describe("OOM guardrails: end-to-end chain", () => {
+   const savedEnv = {
+      max: process.env.PUBLISHER_MAX_QUERY_ROWS,
+      timeout: process.env.PUBLISHER_QUERY_TIMEOUT_MS,
+      concurrency: process.env.PUBLISHER_MAX_CONCURRENT_QUERIES,
+   };
+
+   beforeEach(() => {
+      resetActiveQueryCountForTesting();
+      resetQueryConcurrencyTelemetryForTesting();
+   });
+
+   afterEach(() => {
+      sinon.restore();
+      resetActiveQueryCountForTesting();
+      resetQueryConcurrencyTelemetryForTesting();
+      // Restore env so later tests don't see this suite's bleed-through.
+      const restore = (key: keyof typeof savedEnv, envVar: string): void => {
+         if (savedEnv[key] === undefined) delete process.env[envVar];
+         else process.env[envVar] = savedEnv[key]!;
+      };
+      restore("max", "PUBLISHER_MAX_QUERY_ROWS");
+      restore("timeout", "PUBLISHER_QUERY_TIMEOUT_MS");
+      restore("concurrency", "PUBLISHER_MAX_CONCURRENT_QUERIES");
+   });
+
+   it("admission gate fires FIRST: 503 before the connector is touched", async () => {
+      const runSQL = sinon.stub().resolves({ rows: [], totalRows: 0 });
+      const assertCanAdmitQuery = sinon
+         .stub()
+         .throws(
+            new ServiceUnavailableError(
+               "Publisher is under memory pressure and cannot accept new queries.",
+            ),
+         );
+      const { app } = buildApp(runSQL, { assertCanAdmitQuery });
+
+      const res = await request(app)
+         .post("/api/v0/environments/e/connections/c/sqlQuery")
+         .send({ sqlStatement: "SELECT 1" });
+
+      expect(res.status).toBe(503);
+      // Critical load-shedding invariant: connector must NOT have
+      // been called when the gate rejects.
+      expect(runSQL.called).toBe(false);
+   });
+
+   it("concurrency cap fires SECOND: 503 before the controller is touched, once the slot pool is full", async () => {
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "1";
+      // First request: runSQL takes ~200 ms so the slot stays
+      // held while we fire the second. Using a real timer rather
+      // than an unresolved deferred keeps the test deterministic
+      // even when supertest spawns one ephemeral http server per
+      // `request(app)` call — both sockets will eventually close
+      // on their own regardless of orchestration order.
+      const runSQL = sinon
+         .stub()
+         .callsFake(
+            () =>
+               new Promise((resolve) =>
+                  setTimeout(() => resolve({ rows: [], totalRows: 0 }), 200),
+               ),
+         );
+      const { app } = buildApp(runSQL);
+
+      // Drive both requests in parallel — `Promise.all` keeps the
+      // event loop turning on the first request's pending timer
+      // while the second runs through the middleware.
+      const [first, second] = await Promise.all([
+         request(app)
+            .post("/api/v0/environments/e/connections/c/sqlQuery")
+            .send({ sqlStatement: "SELECT 1" }),
+         (async () => {
+            // Yield a few ticks so the first request actually
+            // enters the middleware and holds the slot before
+            // the second one is dispatched. 20 ms is far longer
+            // than any scheduler hop but well below the 200 ms
+            // runSQL timer, leaving the slot definitely held.
+            await new Promise((r) => setTimeout(r, 20));
+            return request(app)
+               .post("/api/v0/environments/e/connections/c/sqlQuery")
+               .send({ sqlStatement: "SELECT 2" });
+         })(),
+      ]);
+
+      expect(first.status).toBe(200);
+      expect(second.status).toBe(503);
+      // Connector should still have been called only once — the
+      // middleware rejected the second request before it reached
+      // the controller.
+      expect(runSQL.callCount).toBe(1);
+   });
+
+   it("timeout fires THIRD: 504 when the driver outlasts PUBLISHER_QUERY_TIMEOUT_MS, and the abort signal is delivered to the driver", async () => {
+      process.env.PUBLISHER_QUERY_TIMEOUT_MS = "20";
+      let observedSignal: AbortSignal | undefined;
+      const runSQL = sinon.stub().callsFake(
+         (_sql: string, opts: RunSQLOptions) =>
+            new Promise((_resolve, reject) => {
+               observedSignal = opts.abortSignal;
+               opts.abortSignal?.addEventListener("abort", () =>
+                  // The driver MUST reject when its abort signal
+                  // fires; otherwise the timeout helper has nothing
+                  // to convert to 504 (the success-after-timeout
+                  // race is explicitly allowed).
+                  reject(new Error("driver: aborted")),
+               );
+            }),
+      );
+      const { app } = buildApp(runSQL);
+
+      const res = await request(app)
+         .post("/api/v0/environments/e/connections/c/sqlQuery")
+         .send({ sqlStatement: "SELECT pg_sleep(1)" });
+
+      expect(res.status).toBe(504);
+      expect(observedSignal).toBeDefined();
+      expect(observedSignal?.aborted).toBe(true);
+   });
+
+   it("row cap fires LAST: 413 when the driver overshoots PUBLISHER_MAX_QUERY_ROWS", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "2";
+      // Driver returns 3 rows even though we asked for cap+1 = 3
+      // — the sentinel pattern catches this and 413s.
+      const rows = [{ a: 1 }, { a: 2 }, { a: 3 }];
+      const runSQL = sinon.stub().resolves({ rows, totalRows: rows.length });
+      const { app } = buildApp(runSQL);
+
+      const res = await request(app)
+         .post("/api/v0/environments/e/connections/c/sqlQuery")
+         .send({ sqlStatement: "SELECT * FROM big_table" });
+
+      expect(res.status).toBe(413);
+      expect(JSON.stringify(res.body)).toContain("more than 2 rows");
+   });
+
+   it("happy path: all gates pass-through, 200 with the result body", async () => {
+      const rows = [{ a: 1 }];
+      const runSQL = sinon.stub().resolves({ rows, totalRows: rows.length });
+      const { app } = buildApp(runSQL);
+
+      const res = await request(app)
+         .post("/api/v0/environments/e/connections/c/sqlQuery")
+         .send({ sqlStatement: "SELECT 1" });
+
+      expect(res.status).toBe(200);
+      expect(res.body).toEqual({
+         data: JSON.stringify({ rows, totalRows: rows.length }),
+      });
+   });
+});

package/src/path_safety.ts

@@ -44,7 +44,9 @@ const MAX_ENVIRONMENT_PATH_LEN = 4096;
  * production package name we've seen fits within it, and tightening
  * here costs nothing.
  */
-export function assertSafePackageName(packageName: unknown): void {
+export function assertSafePackageName(
+   packageName: unknown,
+): asserts packageName is string {
    if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
       throw new BadRequestError(
          `Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`,
@@ -58,7 +60,9 @@ export function assertSafePackageName(packageName: unknown): void {
  * live in subdirectories like `models/foo.malloy`); backslashes,
  * absolute paths, NUL bytes, and `..` / `.` segments are not.
  */
-export function assertSafeRelativeModelPath(modelPath: unknown): void {
+export function assertSafeRelativeModelPath(
+   modelPath: unknown,
+): asserts modelPath is string {
    if (
       typeof modelPath !== "string" ||
       modelPath.length === 0 ||
@@ -90,7 +94,9 @@ export function assertSafeRelativeModelPath(modelPath: unknown): void {
  * sanitizer-barrier pattern CodeQL's `js/path-injection` query
  * recognises.
  */
-export function assertSafeEnvironmentPath(environmentPath: unknown): void {
+export function assertSafeEnvironmentPath(
+   environmentPath: unknown,
+): asserts environmentPath is string {
    if (typeof environmentPath !== "string") {
       throw new BadRequestError(`Invalid environment path: must be a string`);
    }

package/src/query_cap_metrics.spec.ts

@@ -0,0 +1,89 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+
+import {
+   recordQueryCapExceeded,
+   resetQueryCapTelemetryForTesting,
+} from "./query_cap_metrics";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "./test_helpers/metrics_harness";
+
+describe("query_cap_metrics", () => {
+   let harness: MetricsHarness;
+
+   beforeEach(async () => {
+      harness = await startMetricsHarness();
+      // Drop cached instruments so they re-init against the new
+      // provider; otherwise this test's writes go to a counter
+      // bound to the previous provider's reader.
+      resetQueryCapTelemetryForTesting();
+   });
+
+   afterEach(async () => {
+      delete process.env.PUBLISHER_MAX_QUERY_ROWS;
+      delete process.env.PUBLISHER_MAX_RESPONSE_BYTES;
+      resetQueryCapTelemetryForTesting();
+      await harness.shutdown();
+   });
+
+   it("publisher_query_cap_exceeded_total ticks per call, labeled by cap_type and source", async () => {
+      recordQueryCapExceeded("rows", "connection_sql");
+      recordQueryCapExceeded("rows", "connection_sql");
+      recordQueryCapExceeded("bytes", "model_query");
+      recordQueryCapExceeded("rows", "notebook_cell");
+
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "rows",
+            source: "connection_sql",
+         }),
+      ).toBe(2);
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "bytes",
+            source: "model_query",
+         }),
+      ).toBe(1);
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "rows",
+            source: "notebook_cell",
+         }),
+      ).toBe(1);
+   });
+
+   it("publisher_max_query_rows gauge reports the live env-var value", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "12345";
+      // Prime telemetry — the gauges install on the first
+      // counter-emitting call (`recordQueryCapExceeded`); in
+      // production that's the first 413, in tests we trigger it
+      // explicitly so the gauge is observable without a 413.
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(
+         12345,
+      );
+   });
+
+   it("publisher_max_response_bytes gauge reports the live env-var value", async () => {
+      process.env.PUBLISHER_MAX_RESPONSE_BYTES = "9876543";
+      recordQueryCapExceeded("bytes", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_response_bytes")).toBe(
+         9876543,
+      );
+   });
+
+   it("publisher_max_query_rows gauge reports 0 when the cap is opted out", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "0";
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(0);
+   });
+
+   it("publisher_max_query_rows gauge reports -1 on misconfig so dashboards reveal the bad value", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "not-a-number";
+      // Misconfig must not crash the scrape; -1 is the agreed
+      // signal mirroring `publisher_query_timeout_ms`.
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(-1);
+   });
+});