@malloy-publisher/server 0.0.235 → 0.0.236

package/src/service/model_limits.spec.ts

@@ -1,209 +0,0 @@
-import { describe, expect, it } from "bun:test";
-
-import { PayloadTooLargeError } from "../errors";
-import {
-   assertWithinModelResponseLimits,
-   queryRowLimitSource,
-   resolveModelQueryRowLimit,
-} from "./model_limits";
-
-describe("resolveModelQueryRowLimit", () => {
-   it("uses the user's LIMIT when set and below the maxRows ceiling", () => {
-      expect(
-         resolveModelQueryRowLimit(500, {
-            defaultLimit: 1000,
-            maxRows: 100_000,
-         }),
-      ).toBe(500);
-   });
-
-   it("falls back to defaultLimit when the user's LIMIT is undefined", () => {
-      expect(
-         resolveModelQueryRowLimit(undefined, {
-            defaultLimit: 1000,
-            maxRows: 100_000,
-         }),
-      ).toBe(1000);
-   });
-
-   it("falls back to defaultLimit when the user's LIMIT is 0 (Malloy returns 0 for 'no limit')", () => {
-      // Malloy's PreparedResult returns 0 from `resultExplore.limit` when the
-      // query has no LIMIT clause; treat that as "no user limit".
-      expect(
-         resolveModelQueryRowLimit(0, {
-            defaultLimit: 1000,
-            maxRows: 100_000,
-         }),
-      ).toBe(1000);
-   });
-
-   it("clamps a too-high user LIMIT to the maxRows + 1 sentinel", () => {
-      expect(
-         resolveModelQueryRowLimit(1_000_000, {
-            defaultLimit: 1000,
-            maxRows: 100_000,
-         }),
-      ).toBe(100_001);
-   });
-
-   it("clamps a too-high defaultLimit to the maxRows + 1 sentinel", () => {
-      // Operator misconfigured DEFAULT > MAX; the hard cap still wins.
-      expect(
-         resolveModelQueryRowLimit(undefined, {
-            defaultLimit: 1_000_000,
-            maxRows: 50,
-         }),
-      ).toBe(51);
-   });
-
-   it("returns the requested limit unchanged when maxRows is 0 (cap disabled)", () => {
-      expect(
-         resolveModelQueryRowLimit(1_000_000, {
-            defaultLimit: 1000,
-            maxRows: 0,
-         }),
-      ).toBe(1_000_000);
-   });
-
-   it("returns the default unchanged when maxRows is 0 and no user limit", () => {
-      expect(
-         resolveModelQueryRowLimit(undefined, {
-            defaultLimit: 1000,
-            maxRows: 0,
-         }),
-      ).toBe(1000);
-   });
-
-   it("rejects negative user limits by treating them as 'no limit'", () => {
-      // Defensive — shouldn't happen in practice, but a -1 from a malformed
-      // PreparedResult shouldn't propagate as a negative rowLimit to the driver.
-      expect(
-         resolveModelQueryRowLimit(-1, {
-            defaultLimit: 1000,
-            maxRows: 100_000,
-         }),
-      ).toBe(1000);
-   });
-});
-
-describe("assertWithinModelResponseLimits", () => {
-   it("does not throw when both counts are below their caps", () => {
-      expect(() =>
-         assertWithinModelResponseLimits(
-            500,
-            1_000,
-            { maxRows: 1000, maxBytes: 10_000 },
-            "model_query",
-         ),
-      ).not.toThrow();
-   });
-
-   it("does not throw when row count equals the cap exactly (sentinel hasn't fired)", () => {
-      expect(() =>
-         assertWithinModelResponseLimits(
-            1000,
-            1_000,
-            { maxRows: 1000, maxBytes: 10_000 },
-            "model_query",
-         ),
-      ).not.toThrow();
-   });
-
-   it("throws PayloadTooLargeError with the row-cap message on row overflow", () => {
-      expect(() =>
-         assertWithinModelResponseLimits(
-            1001,
-            1_000,
-            { maxRows: 1000, maxBytes: 10_000 },
-            "model_query",
-         ),
-      ).toThrow(PayloadTooLargeError);
-      expect(() =>
-         assertWithinModelResponseLimits(
-            1001,
-            1_000,
-            { maxRows: 1000, maxBytes: 10_000 },
-            "model_query",
-         ),
-      ).toThrow("more than 1000 rows");
-   });
-
-   it("throws PayloadTooLargeError with the byte-cap message on byte overflow", () => {
-      expect(() =>
-         assertWithinModelResponseLimits(
-            10,
-            50_000,
-            { maxRows: 1000, maxBytes: 10_000 },
-            "model_query",
-         ),
-      ).toThrow(PayloadTooLargeError);
-      expect(() =>
-         assertWithinModelResponseLimits(
-            10,
-            50_000,
-            { maxRows: 1000, maxBytes: 10_000 },
-            "model_query",
-         ),
-      ).toThrow("exceeded 10000 bytes");
-   });
-
-   it("prefers the row-cap message when both caps would have fired (row check runs first)", () => {
-      expect(() =>
-         assertWithinModelResponseLimits(
-            2000,
-            50_000,
-            { maxRows: 1000, maxBytes: 10_000 },
-            "model_query",
-         ),
-      ).toThrow("more than 1000 rows");
-   });
-
-   it("disables row cap when maxRows is 0", () => {
-      expect(() =>
-         assertWithinModelResponseLimits(
-            1_000_000,
-            1_000,
-            { maxRows: 0, maxBytes: 10_000 },
-            "model_query",
-         ),
-      ).not.toThrow();
-   });
-
-   it("disables byte cap when maxBytes is 0", () => {
-      expect(() =>
-         assertWithinModelResponseLimits(
-            10,
-            1_000_000_000,
-            { maxRows: 1000, maxBytes: 0 },
-            "model_query",
-         ),
-      ).not.toThrow();
-   });
-});
-
-describe("queryRowLimitSource", () => {
-   /**
-    * Must mirror resolveModelQueryRowLimit's own `requested` condition: if the
-    * two ever disagree, the reported source describes a different limit than
-    * the one actually pushed into the SQL.
-    */
-   it("reports the query when it carried its own positive limit", () => {
-      expect(queryRowLimitSource(10)).toBe("query");
-      expect(queryRowLimitSource(1)).toBe("query");
-   });
-
-   it("reports the server default when the query carried none", () => {
-      expect(queryRowLimitSource(undefined)).toBe("server_default");
-      expect(queryRowLimitSource(0)).toBe("server_default");
-      expect(queryRowLimitSource(-1)).toBe("server_default");
-   });
-
-   it("agrees with which limit resolveModelQueryRowLimit actually used", () => {
-      const config = { defaultLimit: 1000, maxRows: 0 };
-      for (const userLimit of [undefined, 0, -1, 1, 10, 5000]) {
-         const used = resolveModelQueryRowLimit(userLimit, config);
-         const cameFromQuery = queryRowLimitSource(userLimit) === "query";
-         expect(cameFromQuery).toBe(used !== config.defaultLimit);
-      }
-   });
-});

package/src/service/model_limits.ts

@@ -1,131 +0,0 @@
-/**
- * Memory guards for the Malloy model-query path (the `runnable.run`
- * flow used by `getQueryResults` and notebook cell execution).
- *
- * Two layered defenses:
- *
- *   1. {@link resolveModelQueryRowLimit} — compute the effective
- *      `rowLimit` to push down to `runnable.run`. The user's Malloy
- *      `LIMIT` clause wins when present; otherwise the operator-
- *      tunable default ({@link getDefaultQueryRowLimit}) fills in.
- *      Either way the result is clamped to `maxRows + 1` so the
- *      database itself stops producing rows when a user-supplied
- *      `LIMIT 1_000_000` would otherwise blow up the process.
- *
- *   2. {@link assertWithinModelResponseLimits} — post-run overflow
- *      detection. If the connector returned `maxRows + 1` rows
- *      (the sentinel) or the JSON-serialized response exceeds the
- *      byte cap, throw `PayloadTooLargeError` so the caller sees a
- *      clean HTTP 413.
- *
- * Caveat on the byte cap: this path runs `runnable.run` (buffered),
- * not `runStream`, so by the time we measure bytes the result has
- * already been materialized in memory. The byte cap here is loud-
- * failure detection — it surfaces oversize responses with a 413
- * instead of letting the client receive a half-transmitted payload
- * — not OOM prevention. True prevention requires streaming +
- * `Result` reconstruction from `DataRecord`s, which is out of scope
- * for this step (the model-query streaming path entangles with
- * Malloy's `Result` schema metadata in non-trivial ways).
- *
- * Both helpers are pure so they can be unit-tested without spinning
- * up a model runtime; the caller injects the env-derived limits.
- */
-
-import { PayloadTooLargeError } from "../errors";
-import {
-   recordQueryCapExceeded,
-   type QueryCapSource,
-} from "../query_cap_metrics";
-
-export interface ResolveRowLimitConfig {
-   /**
-    * Result of {@link getDefaultQueryRowLimit}. Applied when the
-    * user's Malloy query doesn't carry a `LIMIT` clause.
-    */
-   defaultLimit: number;
-   /**
-    * Result of {@link getMaxQueryRows}. The effective row limit is
-    * clamped to `maxRows + 1` so a sentinel-count overflow check can
-    * distinguish "ran right up to the cap" from "would have
-    * overflowed". A value of `0` disables the cap.
-    */
-   maxRows: number;
-}
-
-/**
- * Compute the `rowLimit` to pass to `runnable.run`. The +1 sentinel
- * mirrors the Step 1 / Step 2 patterns on the connection-query path
- * so behavior is uniform across all query surfaces.
- */
-export function resolveModelQueryRowLimit(
-   userLimit: number | undefined,
-   { defaultLimit, maxRows }: ResolveRowLimitConfig,
-): number {
-   const requested = userLimit && userLimit > 0 ? userLimit : defaultLimit;
-   if (maxRows <= 0) return requested;
-   return Math.min(requested, maxRows + 1);
-}
-
-/** Whether the cap came from the query itself or from the server default. */
-export type QueryRowLimitSource = "query" | "server_default";
-
-/**
- * Which of the two the cap in {@link resolveModelQueryRowLimit} came from.
- *
- * The distinction is the whole difference between "the database cut this off
- * and you were not told" and "you asked for exactly this many". A `limit:` or
- * `top:` the author wrote is deliberate and complete for what it asked; only the
- * silently-applied default means rows were probably left behind.
- *
- * The condition deliberately mirrors `requested` above and must stay in step
- * with it, so a change to which limit wins cannot leave the reported source
- * describing the other one.
- */
-export function queryRowLimitSource(
-   userLimit: number | undefined,
-): QueryRowLimitSource {
-   return userLimit && userLimit > 0 ? "query" : "server_default";
-}
-
-export interface ModelResponseLimitsConfig {
-   /** Result of {@link getMaxQueryRows}. `0` disables the row cap. */
-   maxRows: number;
-   /** Result of {@link getMaxResponseBytes}. `0` disables the byte cap. */
-   maxBytes: number;
-}
-
-/**
- * Throw {@link PayloadTooLargeError} (HTTP 413) when a model-query
- * response exceeds either configured cap. `rowCount` should be the
- * raw row count Malloy actually fetched (typically
- * `result._queryResult.data.rawData.length`); `serializedBytes`
- * should be the byte length of the JSON-stringified response that
- * would otherwise be returned to the client.
- *
- * Row check uses the `> maxRows` sentinel (not `>= maxRows`), since
- * {@link resolveModelQueryRowLimit} asked the connector for
- * `maxRows + 1` and we want to fail only when that sentinel fires.
- */
-export function assertWithinModelResponseLimits(
-   rowCount: number,
-   serializedBytes: number,
-   { maxRows, maxBytes }: ModelResponseLimitsConfig,
-   source: QueryCapSource,
-): void {
-   if (maxRows > 0 && rowCount > maxRows) {
-      // Tick the counter *before* throwing so it reflects the
-      // event even if a downstream `catch` swallows the error
-      // (notebook handlers and MCP tools both do this in places).
-      recordQueryCapExceeded("rows", source);
-      throw new PayloadTooLargeError(
-         `Query returned more than ${maxRows} rows. Refine the query (add a LIMIT or more selective WHERE) or raise PUBLISHER_MAX_QUERY_ROWS.`,
-      );
-   }
-   if (maxBytes > 0 && serializedBytes > maxBytes) {
-      recordQueryCapExceeded("bytes", source);
-      throw new PayloadTooLargeError(
-         `Query response exceeded ${maxBytes} bytes (was ${serializedBytes}). Project fewer columns, add a LIMIT, or raise PUBLISHER_MAX_RESPONSE_BYTES.`,
-      );
-   }
-}

package/src/service/model_storage_serve.spec.ts

@@ -1,193 +0,0 @@
-// End-to-end routing contract for the `storage=` virtual-source serve path,
-// exercised through the real Model.getQueryResults against a real in-memory
-// DuckDB. The ORIGINAL source returns 0; the serve binding points at a table
-// returning 60 — so the value observed proves whether the query was routed
-// through the serve-shape transform (60) or served live from the original
-// model (0). This pins the mode gate, the binding gate, and the safe fallback.
-import { DuckDBConnection } from "@malloydata/db-duckdb";
-import {
-   FixedConnectionMap,
-   InMemoryURLReader,
-   modelDefToModelInfo,
-   Runtime,
-} from "@malloydata/malloy";
-import { afterEach, describe, expect, it } from "bun:test";
-import { Model } from "./model";
-import type { ServeBinding } from "./materialization_serve_transform";
-
-const ROOT = "file:///storage-serve-e2e/";
-const QUERY = "run: X -> { aggregate: t is total.sum() }";
-const BINDING: ServeBinding = {
-   sourceName: "X",
-   connectionName: "duckdb",
-   virtualHandle: "h",
-   tablePath: "mz_real",
-   schema: [{ name: "total", type: "BIGINT" }],
-};
-
-/**
- * A Model whose original source `X` yields total=0, with a real DuckDB
- * connection carrying `mz_real` (total=60) that the serve binding rebinds to.
- */
-async function buildModel(opts?: {
-   originalText?: string;
-   storageSQL?: string;
-}): Promise<Model> {
-   const duckdb = new DuckDBConnection("duckdb", ":memory:");
-   // DuckDB shares a process-global :memory: db by connection name, so a prior
-   // test's table can linger — replace it.
-   await duckdb.runSQL(
-      opts?.storageSQL ??
-         "CREATE OR REPLACE TABLE mz_real AS SELECT 60 AS total",
-   );
-   const connMap = new Map<string, DuckDBConnection>([["duckdb", duckdb]]);
-   const originalText =
-      opts?.originalText ?? `source: X is duckdb.sql("SELECT 0 AS total")`;
-   const urlReader = new InMemoryURLReader(
-      new Map([[`${ROOT}m.malloy`, originalText]]),
-   );
-   const runtime = new Runtime({
-      urlReader,
-      connections: new FixedConnectionMap(connMap, "duckdb"),
-   });
-   const mm = runtime.loadModel(new URL(`${ROOT}m.malloy`), {
-      importBaseURL: new URL(ROOT),
-   });
-   const compiled = await mm.getModel();
-   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-   const modelDef = (compiled as any)._modelDef;
-   const modelInfo = modelDefToModelInfo(modelDef);
-   const model = new Model(
-      "pkg",
-      "m.malloy",
-      {},
-      "model",
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      mm as any,
-      modelDef,
-      undefined,
-      undefined,
-      undefined,
-      undefined,
-      undefined,
-      undefined,
-      undefined,
-      modelInfo,
-   );
-   model.setServeMalloyConfig(connMap);
-   return model;
-}
-
-async function runTotal(model: Model): Promise<number> {
-   const res = await model.getQueryResults(
-      undefined,
-      undefined,
-      QUERY,
-      {},
-      true,
-   );
-   // compactResult is the row value array: [{ t: <sum> }].
-   const rows = res.compactResult as unknown as { t: number }[];
-   return Number(rows[0].t);
-}
-
-describe("storage= serve routing (end-to-end)", () => {
-   afterEach(() => {
-      delete process.env.PERSIST_STORAGE_MODE;
-   });
-
-   it("mode=on + a binding routes the query to the materialized table", async () => {
-      process.env.PERSIST_STORAGE_MODE = "on";
-      const model = await buildModel();
-      model.setServeBindings([BINDING]);
-      expect(await runTotal(model)).toBe(60);
-   });
-
-   it("mode=off serves live even with a binding present (kill switch)", async () => {
-      process.env.PERSIST_STORAGE_MODE = "off";
-      const model = await buildModel();
-      model.setServeBindings([BINDING]);
-      expect(await runTotal(model)).toBe(0);
-   });
-
-   it("mode=write-only serves live (build-only rung)", async () => {
-      process.env.PERSIST_STORAGE_MODE = "write-only";
-      const model = await buildModel();
-      model.setServeBindings([BINDING]);
-      expect(await runTotal(model)).toBe(0);
-   });
-
-   it("mode=on with no bindings serves live", async () => {
-      process.env.PERSIST_STORAGE_MODE = "on";
-      const model = await buildModel();
-      expect(await runTotal(model)).toBe(0);
-   });
-
-   it("falls back to live when the serve shape cannot compile (bad connection)", async () => {
-      process.env.PERSIST_STORAGE_MODE = "on";
-      const model = await buildModel();
-      // A binding on a connection the config doesn't have — the serve-shape
-      // compile throws, and the query must still succeed, served live.
-      model.setServeBindings([{ ...BINDING, connectionName: "missing_conn" }]);
-      expect(await runTotal(model)).toBe(0);
-   });
-
-   it("a binding for a source this model does not define does not break serving the model's own sources", async () => {
-      // Bindings are pushed to EVERY model in the package, so a model can receive
-      // a binding for a source it doesn't define (defined in a sibling model).
-      // That source has no field list here, so its narrowed schema is empty — the
-      // serve shape must simply omit it, not emit an empty `type: {}` that fails
-      // the whole shape model and drops storage serving for X too.
-      process.env.PERSIST_STORAGE_MODE = "on";
-      const model = await buildModel();
-      model.setServeBindings([
-         BINDING,
-         {
-            sourceName: "OtherModelSource", // not defined in this model
-            connectionName: "duckdb",
-            virtualHandle: "h2",
-            tablePath: "mz_real",
-            schema: [{ name: "total", type: "BIGINT" }],
-         },
-      ]);
-      expect(await runTotal(model)).toBe(60);
-   });
-
-   it("does not serve an except:-hidden column from storage (vector A)", async () => {
-      process.env.PERSIST_STORAGE_MODE = "on";
-      // X hides `secret` via except:, but the built table still carries it (the
-      // build's getSQL projected it) and the binding schema captured it.
-      const model = await buildModel({
-         originalText:
-            `source: X is duckdb.sql("SELECT 0 AS total, 'live' AS secret") ` +
-            `extend { except: secret }`,
-         storageSQL:
-            "CREATE OR REPLACE TABLE mz_real AS " +
-            "SELECT 60 AS total, 'from_storage' AS secret",
-      });
-      model.setServeBindings([
-         {
-            ...BINDING,
-            schema: [
-               { name: "total", type: "BIGINT" },
-               { name: "secret", type: "VARCHAR" },
-            ],
-         },
-      ]);
-      // The public column still serves from storage (narrowing didn't break it).
-      expect(await runTotal(model)).toBe(60);
-      // A query on the hidden column must NOT be served from storage: the shape
-      // no longer declares `secret`, so it falls back to live, where X's
-      // `except:` makes `secret` undefined → the query is refused. (Before the
-      // fix this returned 'from_storage' — the leak.)
-      await expect(
-         model.getQueryResults(
-            undefined,
-            undefined,
-            "run: X -> { group_by: secret }",
-            {},
-            true,
-         ),
-      ).rejects.toThrow();
-   });
-});