@malloy-publisher/server 0.0.198 → 0.0.200

package/src/service/givens_integration.spec.ts

@@ -0,0 +1,192 @@
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import { Connection } from "@malloydata/malloy";
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { Model } from "./model";
+
+const TEST_DIR = path.join(os.tmpdir(), "givens-integration-tests");
+const TEST_DB_DIR = path.join(TEST_DIR, "db");
+const TEST_DB_PATH = path.join(TEST_DB_DIR, "test.duckdb");
+const TEST_PKG_DIR = path.join(TEST_DIR, "pkg");
+
+let duckdbConnection: DuckDBConnection;
+
+const SEED_SQL = `
+CREATE TABLE IF NOT EXISTS orders (
+   order_id INTEGER,
+   region VARCHAR,
+   order_date DATE
+);
+INSERT INTO orders VALUES
+   (1, 'US', '2024-01-15'),
+   (2, 'EU', '2024-02-10'),
+   (3, 'APAC', '2024-03-05');
+`;
+
+const MODEL_WITH_GIVENS = `
+##! experimental.givens
+
+given: region_filter :: string is 'US'
+given: cutoff_date :: date is @2024-02-01
+
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure: order_count is count()
+}
+`;
+
+const MODEL_WITHOUT_GIVENS = `
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure: order_count is count()
+}
+`;
+
+const MODEL_WITH_ANNOTATED_GIVEN = `
+##! experimental.givens
+
+#(doc) Region code, e.g. US, EU
+#(label) Region
+given: region_filter :: string is 'US'
+
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+}
+`;
+
+beforeAll(async () => {
+   await fs.mkdir(TEST_DB_DIR, { recursive: true });
+   await fs.mkdir(TEST_PKG_DIR, { recursive: true });
+   duckdbConnection = new DuckDBConnection("duckdb", TEST_DB_PATH, TEST_DB_DIR);
+   for (const stmt of SEED_SQL.trim().split(";").filter(Boolean)) {
+      await duckdbConnection.runSQL(stmt.trim() + ";");
+   }
+   // Each fixture lives in its own file. Tests share `beforeAll` for harness
+   // setup but never edit these files at runtime, so no `beforeEach` /
+   // `afterEach` cleanup is needed.
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "orders.malloy"),
+      MODEL_WITH_GIVENS,
+      "utf-8",
+   );
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "orders_no_givens.malloy"),
+      MODEL_WITHOUT_GIVENS,
+      "utf-8",
+   );
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "orders_annotated.malloy"),
+      MODEL_WITH_ANNOTATED_GIVEN,
+      "utf-8",
+   );
+});
+
+afterAll(async () => {
+   try {
+      await duckdbConnection.close();
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      await fs.rm(TEST_DIR, { recursive: true, force: true });
+   } catch {
+      // Ignore cleanup errors
+   }
+});
+
+function getConnections(): Map<string, Connection> {
+   const map = new Map<string, Connection>();
+   map.set("duckdb", duckdbConnection);
+   return map;
+}
+
+describe("givens introspection", () => {
+   it("surfaces declared givens on the compiled-model response", async () => {
+      const model = await Model.create(
+         "test-pkg",
+         TEST_PKG_DIR,
+         "orders.malloy",
+         getConnections(),
+      );
+
+      const compiledModel = await model.getModel();
+
+      expect(compiledModel.givens).toBeDefined();
+      expect(compiledModel.givens).toHaveLength(2);
+
+      const byName = new Map(
+         (compiledModel.givens ?? []).map((g) => [g.name, g]),
+      );
+      const region = byName.get("region_filter");
+      const cutoff = byName.get("cutoff_date");
+
+      expect(region).toBeDefined();
+      expect(region?.type).toBe("string");
+      expect(cutoff).toBeDefined();
+      expect(cutoff?.type).toBe("date");
+   });
+
+   it("attaches the model-level givens list to every source", async () => {
+      const model = await Model.create(
+         "test-pkg",
+         TEST_PKG_DIR,
+         "orders.malloy",
+         getConnections(),
+      );
+
+      const sources = model.getSources();
+      expect(sources).toBeDefined();
+      expect(sources).toHaveLength(1);
+
+      const ordersSource = sources?.[0];
+      expect(ordersSource?.name).toBe("orders");
+      expect(ordersSource?.givens).toBeDefined();
+      expect(ordersSource?.givens).toHaveLength(2);
+
+      const names = (ordersSource?.givens ?? []).map((g) => g.name).sort();
+      expect(names).toEqual(["cutoff_date", "region_filter"]);
+   });
+
+   it("returns undefined for givens when the model declares none", async () => {
+      const model = await Model.create(
+         "test-pkg",
+         TEST_PKG_DIR,
+         "orders_no_givens.malloy",
+         getConnections(),
+      );
+
+      const compiledModel = await model.getModel();
+
+      // Absent rather than empty: matches how `sources`/`queries` behave when
+      // there are none, and lets OpenAPI clients distinguish "feature
+      // unsupported" from "supported but no declarations."
+      expect(compiledModel.givens).toBeUndefined();
+      expect(model.getSources()?.[0]?.givens).toBeUndefined();
+   });
+
+   it("surfaces only `#(...)` annotations, not pragmas or doc comments", async () => {
+      const model = await Model.create(
+         "test-pkg",
+         TEST_PKG_DIR,
+         "orders_annotated.malloy",
+         getConnections(),
+      );
+
+      const compiledModel = await model.getModel();
+
+      expect(compiledModel.givens).toHaveLength(1);
+      const region = compiledModel.givens?.[0];
+      expect(region?.name).toBe("region_filter");
+
+      // The model declares two `#(...)` annotations plus a `##!` pragma.
+      // Only the `#(...)` lines should land on the wire.
+      const annotations = region?.annotations ?? [];
+      expect(annotations.length).toBeGreaterThanOrEqual(2);
+      for (const line of annotations) {
+         expect(line.startsWith("#(")).toBe(true);
+      }
+      // Negative assertion: no pragma leakage.
+      expect(annotations.some((a) => a.startsWith("##!"))).toBe(false);
+   });
+});

package/src/service/model.spec.ts

@@ -1,9 +1,13 @@
-import { MalloyError, Runtime } from "@malloydata/malloy";
-import { describe, expect, it } from "bun:test";
+import { API, MalloyError, Runtime } from "@malloydata/malloy";
+import { afterEach, describe, expect, it } from "bun:test";
 import fs from "fs/promises";
 import sinon from "sinon";
 
-import { BadRequestError, ModelNotFoundError } from "../errors";
+import {
+   BadRequestError,
+   ModelNotFoundError,
+   PayloadTooLargeError,
+} from "../errors";
 import { Model, ModelType } from "./model";
 
 describe("service/model", () => {
@@ -234,6 +238,297 @@ describe("service/model", () => {
 
             sinon.restore();
          });
+
+         it("forwards givens to runnable.getPreparedResult and .run", async () => {
+            const givensArg = { region: "EU" };
+            const preparedResultStub = sinon
+               .stub()
+               .resolves({ resultExplore: { limit: 10 } });
+            const runStub = sinon
+               .stub()
+               .rejects(new MalloyError("stub-stop", []));
+            const modelMaterializer = {
+               loadQuery: sinon.stub().returns({
+                  getPreparedResult: preparedResultStub,
+                  run: runStub,
+               }),
+            };
+
+            const model = new Model(
+               packageName,
+               mockModelPath,
+               {},
+               "model",
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               modelMaterializer as any,
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               { contents: {}, exports: [], queryList: [] } as any,
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+            );
+
+            await expect(
+               model.getQueryResults(
+                  undefined,
+                  undefined,
+                  "run: orders -> summary",
+                  undefined,
+                  undefined,
+                  givensArg,
+               ),
+            ).rejects.toThrow(MalloyError);
+
+            expect(preparedResultStub.calledOnce).toBe(true);
+            expect(preparedResultStub.firstCall.args[0]).toEqual({
+               givens: givensArg,
+            });
+            expect(runStub.firstCall.args[0]).toMatchObject({
+               givens: givensArg,
+            });
+
+            sinon.restore();
+         });
+
+         /**
+          * The row/byte caps live in `model_limits.ts` (unit-tested in
+          * `model_limits.spec.ts`); these tests just confirm the wiring —
+          * that `Model.getQueryResults` calls the helpers with the right
+          * values and that an overflow propagates as `PayloadTooLargeError`
+          * (HTTP 413), not the generic `BadRequestError` (HTTP 400).
+          */
+         describe("response caps", () => {
+            const originalRowsEnv = process.env.PUBLISHER_MAX_QUERY_ROWS;
+            const originalBytesEnv = process.env.PUBLISHER_MAX_RESPONSE_BYTES;
+            const originalDefaultEnv =
+               process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT;
+
+            afterEach(() => {
+               sinon.restore();
+               for (const [name, original] of [
+                  ["PUBLISHER_MAX_QUERY_ROWS", originalRowsEnv],
+                  ["PUBLISHER_MAX_RESPONSE_BYTES", originalBytesEnv],
+                  ["PUBLISHER_DEFAULT_QUERY_ROW_LIMIT", originalDefaultEnv],
+               ] as const) {
+                  if (original === undefined) {
+                     delete process.env[name];
+                  } else {
+                     process.env[name] = original;
+                  }
+               }
+            });
+
+            /**
+             * Build a Model whose `runnable.run` resolves to a fake Result
+             * with the given totalRows; stub `API.util.wrapResult` so we
+             * don't need to construct a real Malloy schema/queryResult.
+             */
+            function buildModelWithFakeRun(opts: {
+               userLimit?: number;
+               totalRows: number;
+               wrappedJson: object;
+            }): { model: Model; runStub: sinon.SinonStub } {
+               const preparedResultStub = sinon
+                  .stub()
+                  .resolves({ resultExplore: { limit: opts.userLimit ?? 0 } });
+               const fakeResult = {
+                  _queryResult: { data: { rawData: [] } },
+                  totalRows: opts.totalRows,
+                  data: { value: [] },
+                  connectionName: "fake",
+               };
+               const runStub = sinon.stub().resolves(fakeResult);
+               sinon
+                  .stub(API.util, "wrapResult")
+                  .returns(
+                     opts.wrappedJson as unknown as ReturnType<
+                        typeof API.util.wrapResult
+                     >,
+                  );
+               const modelMaterializer = {
+                  loadQuery: sinon.stub().returns({
+                     getPreparedResult: preparedResultStub,
+                     run: runStub,
+                  }),
+               };
+               const model = new Model(
+                  packageName,
+                  mockModelPath,
+                  {},
+                  "model",
+                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                  modelMaterializer as any,
+                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                  { contents: {}, exports: [], queryList: [] } as any,
+                  undefined,
+                  undefined,
+                  undefined,
+                  undefined,
+                  undefined,
+               );
+               return { model, runStub };
+            }
+
+            it("clamps user LIMIT to maxRows + 1 when the user requested more than the cap", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
+               const { model, runStub } = buildModelWithFakeRun({
+                  userLimit: 1_000_000,
+                  totalRows: 10,
+                  wrappedJson: { rows: [] },
+               });
+
+               await model.getQueryResults(
+                  undefined,
+                  undefined,
+                  "run: orders -> summary",
+               );
+
+               expect(runStub.firstCall.args[0].rowLimit).toBe(101);
+            });
+
+            it("passes user LIMIT through when below maxRows", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
+               const { model, runStub } = buildModelWithFakeRun({
+                  userLimit: 50,
+                  totalRows: 10,
+                  wrappedJson: { rows: [] },
+               });
+
+               await model.getQueryResults(
+                  undefined,
+                  undefined,
+                  "run: orders -> summary",
+               );
+
+               expect(runStub.firstCall.args[0].rowLimit).toBe(50);
+            });
+
+            it("falls back to PUBLISHER_DEFAULT_QUERY_ROW_LIMIT when the user query has no LIMIT", async () => {
+               process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "42";
+               delete process.env.PUBLISHER_MAX_QUERY_ROWS;
+               const { model, runStub } = buildModelWithFakeRun({
+                  userLimit: 0,
+                  totalRows: 10,
+                  wrappedJson: { rows: [] },
+               });
+
+               await model.getQueryResults(
+                  undefined,
+                  undefined,
+                  "run: orders -> summary",
+               );
+
+               expect(runStub.firstCall.args[0].rowLimit).toBe(42);
+            });
+
+            it("throws PayloadTooLargeError (not BadRequestError) when totalRows exceeds the cap", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
+               const { model } = buildModelWithFakeRun({
+                  userLimit: 1000,
+                  totalRows: 101,
+                  wrappedJson: { rows: [] },
+               });
+
+               await expect(
+                  model.getQueryResults(
+                     undefined,
+                     undefined,
+                     "run: orders -> summary",
+                  ),
+               ).rejects.toBeInstanceOf(PayloadTooLargeError);
+            });
+
+            it("throws PayloadTooLargeError when the wrapped response exceeds the byte cap", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "1000";
+               process.env.PUBLISHER_MAX_RESPONSE_BYTES = "100";
+               const huge = "x".repeat(500);
+               const { model } = buildModelWithFakeRun({
+                  userLimit: 10,
+                  totalRows: 10,
+                  wrappedJson: { rows: [{ s: huge }] },
+               });
+
+               await expect(
+                  model.getQueryResults(
+                     undefined,
+                     undefined,
+                     "run: orders -> summary",
+                  ),
+               ).rejects.toBeInstanceOf(PayloadTooLargeError);
+            });
+
+            it("does not throw when both counts are within their caps", async () => {
+               process.env.PUBLISHER_MAX_QUERY_ROWS = "1000";
+               process.env.PUBLISHER_MAX_RESPONSE_BYTES = "10000";
+               const { model } = buildModelWithFakeRun({
+                  userLimit: 10,
+                  totalRows: 10,
+                  wrappedJson: { rows: [{ a: 1 }] },
+               });
+
+               await expect(
+                  model.getQueryResults(
+                     undefined,
+                     undefined,
+                     "run: orders -> summary",
+                  ),
+               ).resolves.toBeDefined();
+            });
+         });
+      });
+
+      describe("executeNotebookCell", () => {
+         it("forwards givens to runnable.getPreparedResult and .run", async () => {
+            const givensArg = { target_code: "AA" };
+            const preparedResultStub = sinon
+               .stub()
+               .resolves({ resultExplore: { limit: 10 } });
+            const runStub = sinon
+               .stub()
+               .rejects(new MalloyError("stub-stop", []));
+            const cellRunnable = {
+               getPreparedResult: preparedResultStub,
+               run: runStub,
+            };
+            const runnableCells = [
+               {
+                  type: "code" as const,
+                  text: "run: orders -> by_code",
+                  runnable: cellRunnable,
+               },
+            ];
+
+            const model = new Model(
+               packageName,
+               "test.malloynb",
+               {},
+               "notebook",
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               runnableCells as any,
+               undefined,
+            );
+
+            await expect(
+               model.executeNotebookCell(0, undefined, undefined, givensArg),
+            ).rejects.toThrow(MalloyError);
+
+            expect(preparedResultStub.calledOnce).toBe(true);
+            expect(preparedResultStub.firstCall.args[0]).toEqual({
+               givens: givensArg,
+            });
+            expect(runStub.firstCall.args[0]).toMatchObject({
+               givens: givensArg,
+            });
+
+            sinon.restore();
+         });
       });
    });