@malloy-publisher/server 0.0.199 → 0.0.201

package/src/service/model.ts

@@ -35,15 +35,17 @@ import type {
    SerializedNotebookCell,
 } from "../package_load/protocol";
 import {
-   MODEL_FILE_SUFFIX,
-   NOTEBOOK_FILE_SUFFIX,
-   ROW_LIMIT,
-} from "../constants";
+   getDefaultQueryRowLimit,
+   getMaxQueryRows,
+   getMaxResponseBytes,
+} from "../config";
+import { MODEL_FILE_SUFFIX, NOTEBOOK_FILE_SUFFIX } from "../constants";
 import { HackyDataStylesAccumulator } from "../data_styles";
 import {
    BadRequestError,
    ModelCompilationError,
    ModelNotFoundError,
+   PayloadTooLargeError,
 } from "../errors";
 import { logger } from "../logger";
 import { BuildManifest } from "../storage/DatabaseInterface";
@@ -57,6 +59,10 @@ import {
    type FilterParams,
 } from "./filter";
 import { malloyGivenToApi, type MalloyGiven } from "./given";
+import {
+   assertWithinModelResponseLimits,
+   resolveModelQueryRowLimit,
+} from "./model_limits";
 
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];
@@ -160,6 +166,21 @@ export class Model {
       this.modelInfo =
          modelInfo ??
          (this.modelDef ? modelDefToModelInfo(this.modelDef) : undefined);
+
+      // One-time deprecation notice per Model instance. Surfaces only when
+      // the model declares `#(filter)` annotations so operators migrating
+      // toward `given:` see a clear pointer in the server log without
+      // spamming for models that have already moved over.
+      if (this.filterMap.size > 0) {
+         logger.warn(
+            `Model "${packageName}/${modelPath}" uses deprecated #(filter) annotations. Migrate to given: — see https://github.com/malloydata/publisher/blob/main/docs/givens.md`,
+            {
+               packageName,
+               modelPath,
+               filterSourceCount: this.filterMap.size,
+            },
+         );
+      }
    }
 
    /**
@@ -504,6 +525,14 @@ export class Model {
       filterParams?: FilterParams,
       bypassFilters?: boolean,
       givens?: Record<string, GivenValue>,
+      // Optional caller-supplied abort signal. Plumbed straight into
+      // `runnable.run` so a publisher-issued query timeout (see
+      // `runWithQueryTimeout`) actually cancels the work in flight
+      // instead of just unblocking the awaiter. Pass `undefined` to
+      // keep the legacy "no timeout" behavior — useful for
+      // background callers (materialization, tests) that own their
+      // own deadline.
+      abortSignal?: AbortSignal,
    ): Promise<{
       result: Malloy.Result;
       compactResult: QueryData;
@@ -597,15 +626,18 @@ export class Model {
          throw new BadRequestError(`Invalid query: ${errorMessage}`);
       }
 
-      const rowLimit =
-         (await runnable.getPreparedResult({ givens })).resultExplore.limit ||
-         ROW_LIMIT;
+      const maxRows = getMaxQueryRows();
+      const maxBytes = getMaxResponseBytes();
+      const rowLimit = resolveModelQueryRowLimit(
+         (await runnable.getPreparedResult({ givens })).resultExplore.limit,
+         { defaultLimit: getDefaultQueryRowLimit(), maxRows },
+      );
       const endTime = performance.now();
       const executionTime = endTime - startTime;
 
       let queryResults;
       try {
-         queryResults = await runnable.run({ rowLimit, givens });
+         queryResults = await runnable.run({ rowLimit, givens, abortSignal });
       } catch (error) {
          // Record error metrics
          const errorEndTime = performance.now();
@@ -638,6 +670,24 @@ export class Model {
          throw new BadRequestError(`Query execution failed: ${errorMessage}`);
       }
 
+      const wrappedResult = API.util.wrapResult(queryResults);
+      // Best-effort byte check: we've already buffered `queryResults` and
+      // built `wrappedResult` by the time we get here, so this surfaces
+      // oversize responses with a clean HTTP 413 instead of letting the
+      // controller transmit a half-megabyte payload — it is not OOM
+      // prevention. True prevention requires streaming `Result`
+      // construction, which is out of scope for this step. The row cap
+      // above is the primary OOM defense.
+      const serializedBytes =
+         maxBytes > 0
+            ? Buffer.byteLength(JSON.stringify(wrappedResult), "utf8")
+            : 0;
+      assertWithinModelResponseLimits(
+         queryResults.totalRows,
+         serializedBytes,
+         { maxRows, maxBytes },
+         "model_query",
+      );
       this.queryExecutionHistogram.record(executionTime, {
          "malloy.model.path": this.modelPath,
          "malloy.model.query.name": queryName,
@@ -649,7 +699,7 @@ export class Model {
          "malloy.model.query.status": "success",
       });
       return {
-         result: API.util.wrapResult(queryResults),
+         result: wrappedResult,
          compactResult: queryResults.data.value,
          modelInfo: this.modelInfo,
          dataStyles: this.dataStyles,
@@ -682,7 +732,6 @@ export class Model {
       const notebookCells: ApiNotebookCell[] = (
          this.runnableNotebookCells as RunnableNotebookCell[]
       ).map((cell) => {
-         logger.debug("cell.queryInfo", cell.queryInfo);
          return {
             type: cell.type,
             text: cell.text,
@@ -732,6 +781,9 @@ export class Model {
       filterParams?: FilterParams,
       bypassFilters?: boolean,
       givens?: Record<string, GivenValue>,
+      // See `getQueryResults`: forwarded into `runnable.run` so the
+      // publisher's wall-clock timeout actually cancels the query.
+      abortSignal?: AbortSignal,
    ): Promise<{
       type: "code" | "markdown";
       text: string;
@@ -792,16 +844,40 @@ export class Model {
                }
             }
 
-            const rowLimit =
+            const cellMaxRows = getMaxQueryRows();
+            const cellMaxBytes = getMaxResponseBytes();
+            const rowLimit = resolveModelQueryRowLimit(
                (await runnableToExecute.getPreparedResult({ givens }))
-                  .resultExplore.limit || ROW_LIMIT;
-            const result = await runnableToExecute.run({ rowLimit, givens });
+                  .resultExplore.limit,
+               {
+                  defaultLimit: getDefaultQueryRowLimit(),
+                  maxRows: cellMaxRows,
+               },
+            );
+            const result = await runnableToExecute.run({
+               rowLimit,
+               givens,
+               abortSignal,
+            });
             const query = (await runnableToExecute.getPreparedQuery())._query;
             queryName = (query as NamedQueryDef).as || query.name;
             queryResult =
                result?._queryResult &&
                this.modelInfo &&
                JSON.stringify(API.util.wrapResult(result));
+            // Same caveat as `getQueryResults`: by the time we measure
+            // bytes the response has already been buffered and stringified,
+            // so this is loud-failure detection (clean 413 instead of
+            // partial transmission), not OOM prevention. The row cap above
+            // is the primary defense.
+            if (result?._queryResult && queryResult) {
+               assertWithinModelResponseLimits(
+                  result.totalRows,
+                  Buffer.byteLength(queryResult, "utf8"),
+                  { maxRows: cellMaxRows, maxBytes: cellMaxBytes },
+                  "notebook_cell",
+               );
+            }
          } catch (error) {
             if (error instanceof FilterValidationError) {
                throw new BadRequestError(error.message);
@@ -809,6 +885,12 @@ export class Model {
             if (error instanceof MalloyError) {
                throw error;
             }
+            // Surface PayloadTooLargeError as-is so the error middleware
+            // maps it to HTTP 413; without this it would get swallowed
+            // into a generic 400 BadRequestError below.
+            if (error instanceof PayloadTooLargeError) {
+               throw error;
+            }
             const errorMessage =
                error instanceof Error ? error.message : String(error);
             if (errorMessage.trim() === "Model has no queries.") {
@@ -819,7 +901,6 @@ export class Model {
             } else {
                logger.error("Error message: ", errorMessage);
             }
-            logger.debug("Cell content: ", cellIndex, cell.type, cell.text);
             throw new BadRequestError(`Cell execution failed: ${errorMessage}`);
          }
       }

package/src/service/model_limits.spec.ts

@@ -0,0 +1,181 @@
+import { describe, expect, it } from "bun:test";
+
+import { PayloadTooLargeError } from "../errors";
+import {
+   assertWithinModelResponseLimits,
+   resolveModelQueryRowLimit,
+} from "./model_limits";
+
+describe("resolveModelQueryRowLimit", () => {
+   it("uses the user's LIMIT when set and below the maxRows ceiling", () => {
+      expect(
+         resolveModelQueryRowLimit(500, {
+            defaultLimit: 1000,
+            maxRows: 100_000,
+         }),
+      ).toBe(500);
+   });
+
+   it("falls back to defaultLimit when the user's LIMIT is undefined", () => {
+      expect(
+         resolveModelQueryRowLimit(undefined, {
+            defaultLimit: 1000,
+            maxRows: 100_000,
+         }),
+      ).toBe(1000);
+   });
+
+   it("falls back to defaultLimit when the user's LIMIT is 0 (Malloy returns 0 for 'no limit')", () => {
+      // Malloy's PreparedResult returns 0 from `resultExplore.limit` when the
+      // query has no LIMIT clause; treat that as "no user limit".
+      expect(
+         resolveModelQueryRowLimit(0, {
+            defaultLimit: 1000,
+            maxRows: 100_000,
+         }),
+      ).toBe(1000);
+   });
+
+   it("clamps a too-high user LIMIT to the maxRows + 1 sentinel", () => {
+      expect(
+         resolveModelQueryRowLimit(1_000_000, {
+            defaultLimit: 1000,
+            maxRows: 100_000,
+         }),
+      ).toBe(100_001);
+   });
+
+   it("clamps a too-high defaultLimit to the maxRows + 1 sentinel", () => {
+      // Operator misconfigured DEFAULT > MAX; the hard cap still wins.
+      expect(
+         resolveModelQueryRowLimit(undefined, {
+            defaultLimit: 1_000_000,
+            maxRows: 50,
+         }),
+      ).toBe(51);
+   });
+
+   it("returns the requested limit unchanged when maxRows is 0 (cap disabled)", () => {
+      expect(
+         resolveModelQueryRowLimit(1_000_000, {
+            defaultLimit: 1000,
+            maxRows: 0,
+         }),
+      ).toBe(1_000_000);
+   });
+
+   it("returns the default unchanged when maxRows is 0 and no user limit", () => {
+      expect(
+         resolveModelQueryRowLimit(undefined, {
+            defaultLimit: 1000,
+            maxRows: 0,
+         }),
+      ).toBe(1000);
+   });
+
+   it("rejects negative user limits by treating them as 'no limit'", () => {
+      // Defensive — shouldn't happen in practice, but a -1 from a malformed
+      // PreparedResult shouldn't propagate as a negative rowLimit to the driver.
+      expect(
+         resolveModelQueryRowLimit(-1, {
+            defaultLimit: 1000,
+            maxRows: 100_000,
+         }),
+      ).toBe(1000);
+   });
+});
+
+describe("assertWithinModelResponseLimits", () => {
+   it("does not throw when both counts are below their caps", () => {
+      expect(() =>
+         assertWithinModelResponseLimits(
+            500,
+            1_000,
+            { maxRows: 1000, maxBytes: 10_000 },
+            "model_query",
+         ),
+      ).not.toThrow();
+   });
+
+   it("does not throw when row count equals the cap exactly (sentinel hasn't fired)", () => {
+      expect(() =>
+         assertWithinModelResponseLimits(
+            1000,
+            1_000,
+            { maxRows: 1000, maxBytes: 10_000 },
+            "model_query",
+         ),
+      ).not.toThrow();
+   });
+
+   it("throws PayloadTooLargeError with the row-cap message on row overflow", () => {
+      expect(() =>
+         assertWithinModelResponseLimits(
+            1001,
+            1_000,
+            { maxRows: 1000, maxBytes: 10_000 },
+            "model_query",
+         ),
+      ).toThrow(PayloadTooLargeError);
+      expect(() =>
+         assertWithinModelResponseLimits(
+            1001,
+            1_000,
+            { maxRows: 1000, maxBytes: 10_000 },
+            "model_query",
+         ),
+      ).toThrow("more than 1000 rows");
+   });
+
+   it("throws PayloadTooLargeError with the byte-cap message on byte overflow", () => {
+      expect(() =>
+         assertWithinModelResponseLimits(
+            10,
+            50_000,
+            { maxRows: 1000, maxBytes: 10_000 },
+            "model_query",
+         ),
+      ).toThrow(PayloadTooLargeError);
+      expect(() =>
+         assertWithinModelResponseLimits(
+            10,
+            50_000,
+            { maxRows: 1000, maxBytes: 10_000 },
+            "model_query",
+         ),
+      ).toThrow("exceeded 10000 bytes");
+   });
+
+   it("prefers the row-cap message when both caps would have fired (row check runs first)", () => {
+      expect(() =>
+         assertWithinModelResponseLimits(
+            2000,
+            50_000,
+            { maxRows: 1000, maxBytes: 10_000 },
+            "model_query",
+         ),
+      ).toThrow("more than 1000 rows");
+   });
+
+   it("disables row cap when maxRows is 0", () => {
+      expect(() =>
+         assertWithinModelResponseLimits(
+            1_000_000,
+            1_000,
+            { maxRows: 0, maxBytes: 10_000 },
+            "model_query",
+         ),
+      ).not.toThrow();
+   });
+
+   it("disables byte cap when maxBytes is 0", () => {
+      expect(() =>
+         assertWithinModelResponseLimits(
+            10,
+            1_000_000_000,
+            { maxRows: 1000, maxBytes: 0 },
+            "model_query",
+         ),
+      ).not.toThrow();
+   });
+});

package/src/service/model_limits.ts

@@ -0,0 +1,110 @@
+/**
+ * Memory guards for the Malloy model-query path (the `runnable.run`
+ * flow used by `getQueryResults` and notebook cell execution).
+ *
+ * Two layered defenses:
+ *
+ *   1. {@link resolveModelQueryRowLimit} — compute the effective
+ *      `rowLimit` to push down to `runnable.run`. The user's Malloy
+ *      `LIMIT` clause wins when present; otherwise the operator-
+ *      tunable default ({@link getDefaultQueryRowLimit}) fills in.
+ *      Either way the result is clamped to `maxRows + 1` so the
+ *      database itself stops producing rows when a user-supplied
+ *      `LIMIT 1_000_000` would otherwise blow up the process.
+ *
+ *   2. {@link assertWithinModelResponseLimits} — post-run overflow
+ *      detection. If the connector returned `maxRows + 1` rows
+ *      (the sentinel) or the JSON-serialized response exceeds the
+ *      byte cap, throw `PayloadTooLargeError` so the caller sees a
+ *      clean HTTP 413.
+ *
+ * Caveat on the byte cap: this path runs `runnable.run` (buffered),
+ * not `runStream`, so by the time we measure bytes the result has
+ * already been materialized in memory. The byte cap here is loud-
+ * failure detection — it surfaces oversize responses with a 413
+ * instead of letting the client receive a half-transmitted payload
+ * — not OOM prevention. True prevention requires streaming +
+ * `Result` reconstruction from `DataRecord`s, which is out of scope
+ * for this step (the model-query streaming path entangles with
+ * Malloy's `Result` schema metadata in non-trivial ways).
+ *
+ * Both helpers are pure so they can be unit-tested without spinning
+ * up a model runtime; the caller injects the env-derived limits.
+ */
+
+import { PayloadTooLargeError } from "../errors";
+import {
+   recordQueryCapExceeded,
+   type QueryCapSource,
+} from "../query_cap_metrics";
+
+export interface ResolveRowLimitConfig {
+   /**
+    * Result of {@link getDefaultQueryRowLimit}. Applied when the
+    * user's Malloy query doesn't carry a `LIMIT` clause.
+    */
+   defaultLimit: number;
+   /**
+    * Result of {@link getMaxQueryRows}. The effective row limit is
+    * clamped to `maxRows + 1` so a sentinel-count overflow check can
+    * distinguish "ran right up to the cap" from "would have
+    * overflowed". A value of `0` disables the cap.
+    */
+   maxRows: number;
+}
+
+/**
+ * Compute the `rowLimit` to pass to `runnable.run`. The +1 sentinel
+ * mirrors the Step 1 / Step 2 patterns on the connection-query path
+ * so behavior is uniform across all query surfaces.
+ */
+export function resolveModelQueryRowLimit(
+   userLimit: number | undefined,
+   { defaultLimit, maxRows }: ResolveRowLimitConfig,
+): number {
+   const requested = userLimit && userLimit > 0 ? userLimit : defaultLimit;
+   if (maxRows <= 0) return requested;
+   return Math.min(requested, maxRows + 1);
+}
+
+export interface ModelResponseLimitsConfig {
+   /** Result of {@link getMaxQueryRows}. `0` disables the row cap. */
+   maxRows: number;
+   /** Result of {@link getMaxResponseBytes}. `0` disables the byte cap. */
+   maxBytes: number;
+}
+
+/**
+ * Throw {@link PayloadTooLargeError} (HTTP 413) when a model-query
+ * response exceeds either configured cap. `rowCount` should be the
+ * raw row count Malloy actually fetched (typically
+ * `result._queryResult.data.rawData.length`); `serializedBytes`
+ * should be the byte length of the JSON-stringified response that
+ * would otherwise be returned to the client.
+ *
+ * Row check uses the `> maxRows` sentinel (not `>= maxRows`), since
+ * {@link resolveModelQueryRowLimit} asked the connector for
+ * `maxRows + 1` and we want to fail only when that sentinel fires.
+ */
+export function assertWithinModelResponseLimits(
+   rowCount: number,
+   serializedBytes: number,
+   { maxRows, maxBytes }: ModelResponseLimitsConfig,
+   source: QueryCapSource,
+): void {
+   if (maxRows > 0 && rowCount > maxRows) {
+      // Tick the counter *before* throwing so it reflects the
+      // event even if a downstream `catch` swallows the error
+      // (notebook handlers and MCP tools both do this in places).
+      recordQueryCapExceeded("rows", source);
+      throw new PayloadTooLargeError(
+         `Query returned more than ${maxRows} rows. Refine the query (add a LIMIT or more selective WHERE) or raise PUBLISHER_MAX_QUERY_ROWS.`,
+      );
+   }
+   if (maxBytes > 0 && serializedBytes > maxBytes) {
+      recordQueryCapExceeded("bytes", source);
+      throw new PayloadTooLargeError(
+         `Query response exceeded ${maxBytes} bytes (was ${serializedBytes}). Project fewer columns, add a LIMIT, or raise PUBLISHER_MAX_RESPONSE_BYTES.`,
+      );
+   }
+}

package/src/service/package.spec.ts

@@ -13,7 +13,7 @@ import { Package } from "./package";
 type PartialModel = Pick<Model, "getPath">;
 
 describe("service/package", () => {
-   const testPackageDirectory = "testPackage";
+   const testPackageDirectory = resolve("testPackage");
 
    beforeEach(async () => {
       await fs.mkdir(testPackageDirectory, { recursive: true });
@@ -100,11 +100,7 @@ describe("service/package", () => {
                   testPackageDirectory,
                   new Map(),
                ),
-            ).rejects.toThrowError(
-               new PackageNotFoundError(
-                  "Package manifest for testPackage does not exist.",
-               ),
-            );
+            ).rejects.toBeInstanceOf(PackageNotFoundError);
          });
          it(
             "should return a Package object if the package exists",

package/src/service/package.ts

@@ -28,6 +28,7 @@ import {
    ServiceUnavailableError,
 } from "../errors";
 import { formatDuration, logger } from "../logger";
+import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
 import { BuildManifest } from "../storage/DatabaseInterface";
 import { ignoreDotfiles } from "../utils";
 import { Model } from "./model";
@@ -90,6 +91,7 @@ export class Package {
       packagePath: string,
       environmentMalloyConfig: PackageConnectionInput,
    ): Promise<Package> {
+      assertSafeEnvironmentPath(packagePath);
       const startTime = performance.now();
       await Package.validatePackageManifestExistsOrThrowError(packagePath);
       const manifestValidationTime = performance.now();
@@ -515,7 +517,10 @@ export class Package {
    private static async validatePackageManifestExistsOrThrowError(
       packagePath: string,
    ) {
-      const packageConfigPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
+      const packageConfigPath = safeJoinUnderRoot(
+         packagePath,
+         PACKAGE_MANIFEST_NAME,
+      );
       try {
          await fs.stat(packageConfigPath);
       } catch {

package/src/service/path_injection.spec.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from "bun:test";
+import { BadRequestError } from "../errors";
+import { deleteDuckLakeConnectionFile } from "./connection";
+
+const TRAVERSAL_NAMES: ReadonlyArray<readonly [string, string]> = [
+   ["leading traversal", "../etc"],
+   ["embedded traversal", "foo/../../bar"],
+   ["slash in name", "foo/bar"],
+   ["backslash in name", "foo\\bar"],
+   ["leading dot", ".staging"],
+   ["bare dot-dot", ".."],
+   ["bare dot", "."],
+   ["empty", ""],
+   ["NUL byte", "foo\0bar"],
+   ["oversized", "a".repeat(256)],
+   ["absolute", "/etc/passwd"],
+] as const;
+
+describe("deleteDuckLakeConnectionFile path-injection guards", () => {
+   it.each(TRAVERSAL_NAMES)(
+      "rejects %s as connectionName (%p)",
+      async (_label, connectionName) => {
+         await expect(
+            deleteDuckLakeConnectionFile(connectionName, "/tmp/env"),
+         ).rejects.toBeInstanceOf(BadRequestError);
+      },
+   );
+
+   it.each([
+      ["relative", "relative/path"],
+      ["traversal", "/var/lib/../../etc"],
+      ["NUL byte", "/var/lib/env\0"],
+      ["bare dot-dot", ".."],
+   ])("rejects %s as environmentPath (%p)", async (_label, environmentPath) => {
+      await expect(
+         deleteDuckLakeConnectionFile("conn", environmentPath),
+      ).rejects.toBeInstanceOf(BadRequestError);
+   });
+});