@malloy-publisher/server 0.0.230 → 0.0.232

package/src/service/model_storage_serve.spec.ts

@@ -0,0 +1,193 @@
+// End-to-end routing contract for the `storage=` virtual-source serve path,
+// exercised through the real Model.getQueryResults against a real in-memory
+// DuckDB. The ORIGINAL source returns 0; the serve binding points at a table
+// returning 60 — so the value observed proves whether the query was routed
+// through the serve-shape transform (60) or served live from the original
+// model (0). This pins the mode gate, the binding gate, and the safe fallback.
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import {
+   FixedConnectionMap,
+   InMemoryURLReader,
+   modelDefToModelInfo,
+   Runtime,
+} from "@malloydata/malloy";
+import { afterEach, describe, expect, it } from "bun:test";
+import { Model } from "./model";
+import type { ServeBinding } from "./materialization_serve_transform";
+
+const ROOT = "file:///storage-serve-e2e/";
+const QUERY = "run: X -> { aggregate: t is total.sum() }";
+const BINDING: ServeBinding = {
+   sourceName: "X",
+   connectionName: "duckdb",
+   virtualHandle: "h",
+   tablePath: "mz_real",
+   schema: [{ name: "total", type: "BIGINT" }],
+};
+
+/**
+ * A Model whose original source `X` yields total=0, with a real DuckDB
+ * connection carrying `mz_real` (total=60) that the serve binding rebinds to.
+ */
+async function buildModel(opts?: {
+   originalText?: string;
+   storageSQL?: string;
+}): Promise<Model> {
+   const duckdb = new DuckDBConnection("duckdb", ":memory:");
+   // DuckDB shares a process-global :memory: db by connection name, so a prior
+   // test's table can linger — replace it.
+   await duckdb.runSQL(
+      opts?.storageSQL ??
+         "CREATE OR REPLACE TABLE mz_real AS SELECT 60 AS total",
+   );
+   const connMap = new Map<string, DuckDBConnection>([["duckdb", duckdb]]);
+   const originalText =
+      opts?.originalText ?? `source: X is duckdb.sql("SELECT 0 AS total")`;
+   const urlReader = new InMemoryURLReader(
+      new Map([[`${ROOT}m.malloy`, originalText]]),
+   );
+   const runtime = new Runtime({
+      urlReader,
+      connections: new FixedConnectionMap(connMap, "duckdb"),
+   });
+   const mm = runtime.loadModel(new URL(`${ROOT}m.malloy`), {
+      importBaseURL: new URL(ROOT),
+   });
+   const compiled = await mm.getModel();
+   // eslint-disable-next-line @typescript-eslint/no-explicit-any
+   const modelDef = (compiled as any)._modelDef;
+   const modelInfo = modelDefToModelInfo(modelDef);
+   const model = new Model(
+      "pkg",
+      "m.malloy",
+      {},
+      "model",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      mm as any,
+      modelDef,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      modelInfo,
+   );
+   model.setServeMalloyConfig(connMap);
+   return model;
+}
+
+async function runTotal(model: Model): Promise<number> {
+   const res = await model.getQueryResults(
+      undefined,
+      undefined,
+      QUERY,
+      {},
+      true,
+   );
+   // compactResult is the row value array: [{ t: <sum> }].
+   const rows = res.compactResult as unknown as { t: number }[];
+   return Number(rows[0].t);
+}
+
+describe("storage= serve routing (end-to-end)", () => {
+   afterEach(() => {
+      delete process.env.PERSIST_STORAGE_MODE;
+   });
+
+   it("mode=on + a binding routes the query to the materialized table", async () => {
+      process.env.PERSIST_STORAGE_MODE = "on";
+      const model = await buildModel();
+      model.setServeBindings([BINDING]);
+      expect(await runTotal(model)).toBe(60);
+   });
+
+   it("mode=off serves live even with a binding present (kill switch)", async () => {
+      process.env.PERSIST_STORAGE_MODE = "off";
+      const model = await buildModel();
+      model.setServeBindings([BINDING]);
+      expect(await runTotal(model)).toBe(0);
+   });
+
+   it("mode=write-only serves live (build-only rung)", async () => {
+      process.env.PERSIST_STORAGE_MODE = "write-only";
+      const model = await buildModel();
+      model.setServeBindings([BINDING]);
+      expect(await runTotal(model)).toBe(0);
+   });
+
+   it("mode=on with no bindings serves live", async () => {
+      process.env.PERSIST_STORAGE_MODE = "on";
+      const model = await buildModel();
+      expect(await runTotal(model)).toBe(0);
+   });
+
+   it("falls back to live when the serve shape cannot compile (bad connection)", async () => {
+      process.env.PERSIST_STORAGE_MODE = "on";
+      const model = await buildModel();
+      // A binding on a connection the config doesn't have — the serve-shape
+      // compile throws, and the query must still succeed, served live.
+      model.setServeBindings([{ ...BINDING, connectionName: "missing_conn" }]);
+      expect(await runTotal(model)).toBe(0);
+   });
+
+   it("a binding for a source this model does not define does not break serving the model's own sources", async () => {
+      // Bindings are pushed to EVERY model in the package, so a model can receive
+      // a binding for a source it doesn't define (defined in a sibling model).
+      // That source has no field list here, so its narrowed schema is empty — the
+      // serve shape must simply omit it, not emit an empty `type: {}` that fails
+      // the whole shape model and drops storage serving for X too.
+      process.env.PERSIST_STORAGE_MODE = "on";
+      const model = await buildModel();
+      model.setServeBindings([
+         BINDING,
+         {
+            sourceName: "OtherModelSource", // not defined in this model
+            connectionName: "duckdb",
+            virtualHandle: "h2",
+            tablePath: "mz_real",
+            schema: [{ name: "total", type: "BIGINT" }],
+         },
+      ]);
+      expect(await runTotal(model)).toBe(60);
+   });
+
+   it("does not serve an except:-hidden column from storage (vector A)", async () => {
+      process.env.PERSIST_STORAGE_MODE = "on";
+      // X hides `secret` via except:, but the built table still carries it (the
+      // build's getSQL projected it) and the binding schema captured it.
+      const model = await buildModel({
+         originalText:
+            `source: X is duckdb.sql("SELECT 0 AS total, 'live' AS secret") ` +
+            `extend { except: secret }`,
+         storageSQL:
+            "CREATE OR REPLACE TABLE mz_real AS " +
+            "SELECT 60 AS total, 'from_storage' AS secret",
+      });
+      model.setServeBindings([
+         {
+            ...BINDING,
+            schema: [
+               { name: "total", type: "BIGINT" },
+               { name: "secret", type: "VARCHAR" },
+            ],
+         },
+      ]);
+      // The public column still serves from storage (narrowing didn't break it).
+      expect(await runTotal(model)).toBe(60);
+      // A query on the hidden column must NOT be served from storage: the shape
+      // no longer declares `secret`, so it falls back to live, where X's
+      // `except:` makes `secret` undefined → the query is refused. (Before the
+      // fix this returned 'from_storage' — the leak.)
+      await expect(
+         model.getQueryResults(
+            undefined,
+            undefined,
+            "run: X -> { group_by: secret }",
+            {},
+            true,
+         ),
+      ).rejects.toThrow();
+   });
+});

package/src/service/model_storage_serve_joins.spec.ts

@@ -0,0 +1,193 @@
+// Model-level routing contract for JOINS and VIEWS in the `storage=` serve path,
+// through the real Model.getQueryResults against a real in-memory DuckDB. This
+// exercises the glue the transform-level tests can't: serveBindingsWithRefinements
+// reading a REAL compiled modelDef, mapping a join's `sourceID` to the joined
+// source name, and lifting join/view declarations verbatim from the on-disk
+// source file (as it does in production — hence a real temp file, not
+// InMemoryURLReader alone), plus the shape-compile escalation that keeps one
+// un-carriable refinement from disabling all storage serving.
+//
+// Signals: the live sources return region_name 'LIVE' and amount 10; the bound
+// (materialized) tables return 'STORE' and amount 99 — so the value observed
+// tells which path ran.
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import {
+   FixedConnectionMap,
+   InMemoryURLReader,
+   modelDefToModelInfo,
+   Runtime,
+} from "@malloydata/malloy";
+import { afterAll, afterEach, describe, expect, it } from "bun:test";
+import { mkdtempSync, rmSync, writeFileSync } from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { pathToFileURL } from "url";
+import { Model } from "./model";
+import type { ServeBinding } from "./materialization_serve_transform";
+
+const MODEL_SRC = `source: regions is duckdb.sql("SELECT 'r1' AS region_id, 'LIVE' AS region_name")
+source: orders is duckdb.sql("SELECT 10 AS amount, 'r1' AS region_id") extend {
+  join_one: regions is regions on region_id = regions.region_id
+  measure: total is amount.sum()
+  view: by_region is { group_by: regions.region_name; aggregate: total }
+}
+`;
+
+const ORDERS_BINDING: ServeBinding = {
+   sourceName: "orders",
+   connectionName: "duckdb",
+   virtualHandle: "orders_h",
+   tablePath: "orders_mz",
+   schema: [
+      { name: "amount", type: "BIGINT" },
+      { name: "region_id", type: "VARCHAR" },
+   ],
+};
+const REGIONS_BINDING: ServeBinding = {
+   sourceName: "regions",
+   connectionName: "duckdb",
+   virtualHandle: "regions_h",
+   tablePath: "regions_mz",
+   schema: [
+      { name: "region_id", type: "VARCHAR" },
+      { name: "region_name", type: "VARCHAR" },
+   ],
+};
+
+const tmpDirs: string[] = [];
+
+async function buildModel(): Promise<Model> {
+   const dir = mkdtempSync(join(tmpdir(), "mz-join-"));
+   tmpDirs.push(dir);
+   const file = join(dir, "m.malloy");
+   writeFileSync(file, MODEL_SRC);
+   const fileUrl = pathToFileURL(file).toString();
+
+   const duckdb = new DuckDBConnection("duckdb", ":memory:");
+   // Materialized tables carry distinct values (region_name 'STORE', amount 99)
+   // from the live sources ('LIVE', 10), so a result value proves which ran.
+   await duckdb.runSQL(
+      "CREATE OR REPLACE TABLE orders_mz AS SELECT 99 AS amount, 'r1' AS region_id",
+   );
+   await duckdb.runSQL(
+      "CREATE OR REPLACE TABLE regions_mz AS SELECT 'r1' AS region_id, 'STORE' AS region_name",
+   );
+   const connMap = new Map<string, DuckDBConnection>([["duckdb", duckdb]]);
+
+   // Register the source under its real file:// URL so the compiled modelDef's
+   // locations point at the file liftText reads back from disk.
+   const urlReader = new InMemoryURLReader(new Map([[fileUrl, MODEL_SRC]]));
+   const runtime = new Runtime({
+      urlReader,
+      connections: new FixedConnectionMap(connMap, "duckdb"),
+   });
+   const mm = runtime.loadModel(new URL(fileUrl), {
+      importBaseURL: new URL(".", fileUrl),
+   });
+   const compiled = await mm.getModel();
+   // eslint-disable-next-line @typescript-eslint/no-explicit-any
+   const modelDef = (compiled as any)._modelDef;
+   const modelInfo = modelDefToModelInfo(modelDef);
+   const model = new Model(
+      "pkg",
+      "m.malloy",
+      {},
+      "model",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      mm as any,
+      modelDef,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      modelInfo,
+   );
+   model.setServeMalloyConfig(connMap);
+   return model;
+}
+
+/** Run a query and return the first result row. */
+async function runRow<T>(model: Model, query: string): Promise<T> {
+   const res = await model.getQueryResults(
+      undefined,
+      undefined,
+      query,
+      {},
+      true,
+   );
+   return (res.compactResult as unknown as T[])[0];
+}
+
+const JOIN_QUERY =
+   "run: orders -> { group_by: regions.region_name; aggregate: t is amount.sum() }";
+const VIEW_QUERY = "run: orders -> by_region";
+const PLAIN_QUERY = "run: orders -> { aggregate: t is amount.sum() }";
+
+describe("storage= serve routing with joins and views (end-to-end)", () => {
+   afterEach(() => {
+      delete process.env.PERSIST_STORAGE_MODE;
+   });
+   afterAll(() => {
+      for (const d of tmpDirs) rmSync(d, { recursive: true, force: true });
+   });
+
+   it("serves a join query from the materialized tables when both sources are bound", async () => {
+      process.env.PERSIST_STORAGE_MODE = "on";
+      const model = await buildModel();
+      model.setServeBindings([ORDERS_BINDING, REGIONS_BINDING]);
+      const row = await runRow<{ region_name: string }>(model, JOIN_QUERY);
+      expect(row.region_name).toBe("STORE"); // join ran over the bound tables
+   });
+
+   it("serves a named view (grouping by a joined field) from the materialized tables", async () => {
+      process.env.PERSIST_STORAGE_MODE = "on";
+      const model = await buildModel();
+      model.setServeBindings([ORDERS_BINDING, REGIONS_BINDING]);
+      const row = await runRow<{ region_name: string; total: number }>(
+         model,
+         VIEW_QUERY,
+      );
+      expect(row.region_name).toBe("STORE");
+      expect(Number(row.total)).toBe(99); // amount from orders_mz, not live 10
+   });
+
+   it("falls back to live when the joined source is not materialized (the gate)", async () => {
+      process.env.PERSIST_STORAGE_MODE = "on";
+      const model = await buildModel();
+      // Only `orders` is bound; the join to `regions` is not carried, so a query
+      // traversing it cannot compile against the serve shape → live.
+      model.setServeBindings([ORDERS_BINDING]);
+      expect(
+         (await runRow<{ region_name: string }>(model, JOIN_QUERY)).region_name,
+      ).toBe("LIVE");
+   });
+
+   it("a view that reaches a non-materialized join does not disable base serving (escalation)", async () => {
+      process.env.PERSIST_STORAGE_MODE = "on";
+      const model = await buildModel();
+      // Only `orders` bound: the join is gated out, so the emitted `by_region`
+      // view references an absent join and the RICH shape fails to compile. The
+      // escalation must drop the view category and keep serving base + measures,
+      // rather than losing all storage serving. A plain aggregate must still be
+      // served from storage (amount 99), while the view query falls back to live.
+      model.setServeBindings([ORDERS_BINDING]);
+      expect(Number((await runRow<{ t: number }>(model, PLAIN_QUERY)).t)).toBe(
+         99,
+      );
+      expect(
+         (await runRow<{ region_name: string }>(model, VIEW_QUERY)).region_name,
+      ).toBe("LIVE");
+   });
+
+   it("mode=off serves live even with both bindings present", async () => {
+      process.env.PERSIST_STORAGE_MODE = "off";
+      const model = await buildModel();
+      model.setServeBindings([ORDERS_BINDING, REGIONS_BINDING]);
+      expect(
+         (await runRow<{ region_name: string }>(model, JOIN_QUERY)).region_name,
+      ).toBe("LIVE");
+   });
+});

package/src/service/package.spec.ts

@@ -1,5 +1,6 @@
 import { DuckDBConnection } from "@malloydata/db-duckdb";
 import "@malloydata/db-duckdb/native";
+import { MalloyConfig } from "@malloydata/malloy";
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import { Stats } from "fs";
 import fs from "fs/promises";
@@ -411,6 +412,201 @@ describe("service/package", () => {
                rowCount: 3,
             });
          });
+
+         it("should probe an xlsx database file", async () => {
+            // Real committed fixture: DuckDB's excel extension reads it via the
+            // same duckdb.table() replacement scan the csv path uses.
+            await fs.copyFile(
+               join("tests", "fixtures", "xlsx", "database.xlsx"),
+               join(testPackageDirectory, "database.xlsx"),
+            );
+            const conn = new DuckDBConnection("duckdb");
+
+            // Unlike parquet/csv (statically linked), excel is downloaded from
+            // extensions.duckdb.org on demand. Some CI runners (notably the
+            // hosted macOS fleet, per bake-duckdb-extensions.js) cannot reach
+            // that CDN, so skip the read there rather than fail; the Linux e2e
+            // and the extension-independent specs still cover this path.
+            try {
+               await conn.runSQL("INSTALL excel; LOAD excel;");
+            } catch {
+               console.warn(
+                  "Skipping xlsx probe test: excel extension could not be installed (offline runner)",
+               );
+               await conn.close();
+               return;
+            }
+
+            // @ts-expect-error Accessing private static method for testing
+            const info = await Package.getDatabaseInfo(
+               testPackageDirectory,
+               "database.xlsx",
+               conn,
+            );
+
+            expect(info).toEqual({
+               name: "database.xlsx",
+               columns: [
+                  { name: "Name", type: "string" },
+                  // xlsx stores every number as a double, so numeric columns
+                  // always surface as Malloy `number`.
+                  { name: "Value", type: "number" },
+               ],
+               rowCount: 3,
+            });
+         });
+      });
+
+      describe("getDatabasePaths", () => {
+         it("should list xlsx files alongside parquet and csv", async () => {
+            await fs.copyFile(
+               join("tests", "fixtures", "xlsx", "database.xlsx"),
+               join(testPackageDirectory, "database.xlsx"),
+            );
+
+            // @ts-expect-error Accessing private static method for testing
+            const paths = await Package.getDatabasePaths(testPackageDirectory);
+
+            expect(paths.sort()).toEqual(["database.csv", "database.xlsx"]);
+         });
+
+         it("should skip Excel owner files (~$name.xlsx)", async () => {
+            // Excel creates this sibling while a workbook is open; it is not a
+            // valid zip, so probing it would throw and drop the package.
+            await fs.writeFile(
+               join(testPackageDirectory, "~$database.xlsx"),
+               "not a real xlsx",
+            );
+
+            // @ts-expect-error Accessing private static method for testing
+            const paths = await Package.getDatabasePaths(testPackageDirectory);
+
+            expect(paths).toEqual(["database.csv"]);
+         });
+      });
+
+      describe("readDatabases", () => {
+         it("should report an unreadable database file instead of failing the package", async () => {
+            // A corrupt or partial spreadsheet must not take the whole package
+            // down, but it must not vanish silently either: it stays in the
+            // listing carrying `error` instead of `info`, the same way a model
+            // that fails to compile is reported. Extension-independent:
+            // broken.xlsx throws whether excel loads or not, so this holds on
+            // runners without the CDN too.
+            await fs.writeFile(
+               join(testPackageDirectory, "broken.xlsx"),
+               "not a real xlsx",
+            );
+            const conn = new DuckDBConnection("duckdb");
+            const malloyConfig = {
+               connections: { lookupConnection: async () => conn },
+            } as unknown as MalloyConfig;
+
+            const databases =
+               // @ts-expect-error Accessing private static method for testing
+               (await Package.readDatabases(
+                  testPackageDirectory,
+                  malloyConfig,
+               )) as {
+                  path: string;
+                  info?: unknown;
+                  error?: string;
+               }[];
+
+            // The valid file still loads with a full probe.
+            const good = databases.find((d) => d.path === "database.csv");
+            expect(good?.info).toBeDefined();
+            expect(good?.error).toBeUndefined();
+
+            // The broken one is visible over the API, with a reason.
+            const bad = databases.find((d) => d.path === "broken.xlsx");
+            expect(bad).toBeDefined();
+            expect(bad?.info).toBeUndefined();
+            expect(bad?.error).toBeTruthy();
+         });
+      });
+   });
+
+   // A host is authoritative about WHICH TABLE backs a source, never about
+   // WHETHER the source may be served frozen — that is decided by compiling the
+   // model. These assert the gate that enforces it, because the failure is silent:
+   // an admitted binding for a `given`-filtered source serves one caller's rows to
+   // everyone, and no error is raised. Scenarios
+   // `host-binding-honors-row-level-access` and `host-binding-of-unplanned-source`
+   // prove it end to end; these keep it from being refactored away.
+   describe("bindStorageServeBindings eligibility gate", () => {
+      /** A structurally complete manifest entry — the shape a host sends. */
+      const entry = (sourceName: string, table: string) => ({
+         sourceEntityId: `eid-${sourceName}`,
+         sourceName,
+         physicalTableName: table,
+         storageConnectionName: "lake",
+         schema: [{ name: "region", type: "VARCHAR" }],
+      });
+
+      const packageWith = (
+         eligibility:
+            | { eligible: string[]; refused: Record<string, string> }
+            | undefined,
+      ) => {
+         const pkg = new Package(
+            "testProject",
+            "testPackage",
+            testPackageDirectory,
+            { name: "testPackage" },
+            [],
+            new Map(),
+         );
+         // Private by design: its only writer is the load path, which needs a
+         // compiled package. Set directly to isolate the filter.
+         (
+            pkg as unknown as { sourceEligibility: typeof eligibility }
+         ).sourceEligibility = eligibility;
+         return pkg;
+      };
+
+      const boundSources = (pkg: Package) =>
+         (pkg.getPackageMetadata().storageServeBindings ?? []).map(
+            (b) => b.sourceName,
+         );
+
+      it("drops a refused source's binding and keeps its eligible neighbour", () => {
+         const pkg = packageWith({
+            eligible: ["all_rollup"],
+            refused: { scoped_rollup: "references a given" },
+         });
+
+         pkg.bindStorageServeBindings({
+            a: entry("all_rollup", "t_all"),
+            b: entry("scoped_rollup", "t_all"),
+         });
+
+         expect(boundSources(pkg)).toEqual(["all_rollup"]);
+      });
+
+      it("refuses every binding when the build plan could not be computed", () => {
+         // The plan compute does live schema RPCs, so a warehouse blip during load
+         // is the ordinary way this happens — and the package still serves. An
+         // unexamined binding must not read as an eligible one.
+         const pkg = packageWith(undefined);
+
+         pkg.bindStorageServeBindings({ a: entry("all_rollup", "t_all") });
+
+         expect(boundSources(pkg)).toEqual([]);
+      });
+
+      it("refuses a source the plan never contained", () => {
+         // Malloy admits only query-shaped sources as build roots, so `#@ persist`
+         // on a filtered pass-through produces no plan entry — neither eligible nor
+         // refused. Absence is not eligibility.
+         const pkg = packageWith({ eligible: ["all_rows"], refused: {} });
+
+         pkg.bindStorageServeBindings({
+            a: entry("all_rows", "t_all"),
+            b: entry("scoped", "t_all"),
+         });
+
+         expect(boundSources(pkg)).toEqual(["all_rows"]);
       });
    });
 });