@malloy-publisher/server 0.0.235 → 0.0.237

package/src/service/materialization_build_session.spec.ts

@@ -1,435 +0,0 @@
-import { describe, expect, it } from "bun:test";
-import { DuckDBConnection } from "@malloydata/db-duckdb";
-import { mkdtempSync, rmSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { BadRequestError } from "../errors";
-import type { components } from "../api";
-import {
-   assertStorageServeShapeCompiles,
-   buildDownstreamIntoStorage,
-   buildSourceIntoStorage,
-   createIsolatedBuildSession,
-   createTableAndDescribe,
-   dropStorageTable,
-   dropStorageTableSql,
-   passthroughSourceType,
-   wrapPassthrough,
-} from "./materialization_build_session";
-import {
-   buildChainedStorageBuildModel,
-   buildVirtualMap,
-   type ServeBinding,
-} from "./materialization_serve_transform";
-
-type ApiConnection = components["schemas"]["Connection"];
-
-describe("createIsolatedBuildSession (per-build instance isolation)", () => {
-   // Multi-tenant safety regression guard. @malloydata/db-duckdb pools :memory:
-   // DuckDB instances in a process-global cache keyed by a share key that
-   // EXCLUDES the connection name, so build sessions built with identical config
-   // would otherwise be pooled onto ONE shared instance and collide on their
-   // transient ATTACH aliases (cross-tenant read/write). createIsolatedBuildSession
-   // gives each session a unique workingDirectory so it lands its OWN instance.
-   // If a library change ever re-pools them (drops workingDirectory from the
-   // share key), this test fails LOUD rather than letting a silent cross-tenant
-   // leak ship. No warehouse needed: a plain table proves catalog isolation.
-   it("gives each session its own catalog (a table in one is invisible to another)", async () => {
-      const a = createIsolatedBuildSession("build_lake");
-      const b = createIsolatedBuildSession("build_lake");
-      try {
-         await a.session.runSQL("CREATE TABLE isolated_probe(x INT)");
-         await a.session.runSQL("INSERT INTO isolated_probe VALUES (1)");
-         // b must NOT see a's table — separate instances, separate catalogs.
-         await expect(
-            b.session.runSQL("SELECT * FROM isolated_probe"),
-         ).rejects.toThrow();
-      } finally {
-         await a.dispose();
-         await b.dispose();
-      }
-   });
-
-   it("stays in-memory (dispose leaves no primary DB file) and is idempotent", async () => {
-      const { session, dispose } = createIsolatedBuildSession("build_lake");
-      await session.runSQL("CREATE TABLE t(x INT)");
-      await dispose();
-      // Second dispose must not throw (idempotent teardown).
-      await dispose();
-   });
-});
-
-describe("assertStorageServeShapeCompiles (build-time servability gate)", () => {
-   it("passes for a well-formed captured schema (compiles the serve shape in DuckDB)", async () => {
-      await expect(
-         assertStorageServeShapeCompiles({
-            destinationName: "lake",
-            sourceName: "daily_orders",
-            virtualHandle: "se_1",
-            physicalTableName: "daily_orders__mabc123",
-            schema: [
-               { name: "order_date", type: "DATE" },
-               { name: "order_count", type: "BIGINT" },
-               { name: "total_amount", type: "DOUBLE" },
-            ],
-         }),
-      ).resolves.toBeUndefined();
-   });
-});
-
-describe("serve-shape gate shares one session across builds", () => {
-   // The gate compiles against ONE process-wide session rather than a fresh
-   // DuckDB instance per build (~5.9MB/build of unreclaimed RSS on the
-   // production image). Sharing is safe on paper — the gate does no ATTACH and
-   // establishes no credentials — but a shared session means a shared CATALOG,
-   // and every compile declares a virtual source into it.
-   //
-   // The failure that would matter is a false PASS: a later shape satisfied by
-   // declarations an earlier compile left behind. No end-to-end scenario reaches
-   // it, because each hammer scenario gets a fresh publisher and so never
-   // compiles many shapes through one session.
-   const valid = (i: number) => ({
-      destinationName: "lake",
-      sourceName: `daily_${i}`,
-      virtualHandle: `se_${i}`,
-      physicalTableName: `daily_${i}__mabc`,
-      schema: [
-         { name: "order_date", type: "DATE" },
-         { name: "total_amount", type: "DOUBLE" },
-      ],
-   });
-
-   it("still refuses an unservable shape after many successful compiles", async () => {
-      for (let i = 0; i < 25; i++) {
-         await assertStorageServeShapeCompiles(valid(i));
-      }
-      await expect(
-         assertStorageServeShapeCompiles({ ...valid(99), schema: [] }),
-      ).rejects.toThrow(/cannot be served/i);
-      await expect(
-         assertStorageServeShapeCompiles({
-            ...valid(98),
-            schema: [{ name: "a", type: "NOT_A_REAL_TYPE" }],
-         }),
-      ).rejects.toThrow(/cannot be served/i);
-   });
-
-   it("a refusal does not poison the session for later valid shapes", async () => {
-      await expect(
-         assertStorageServeShapeCompiles({ ...valid(31), schema: [] }),
-      ).rejects.toThrow(/cannot be served/i);
-      await expect(
-         assertStorageServeShapeCompiles(valid(32)),
-      ).resolves.toBeUndefined();
-   });
-
-   it("reusing one handle with a DIFFERENT schema sees the new schema", async () => {
-      // The sharpest false-pass shape: if the first declaration lingered, the
-      // second would compile against stale columns and wrongly pass.
-      await assertStorageServeShapeCompiles({
-         ...valid(41),
-         schema: [{ name: "kept", type: "BIGINT" }],
-      });
-      await expect(
-         assertStorageServeShapeCompiles({ ...valid(41), schema: [] }),
-      ).rejects.toThrow(/cannot be served/i);
-   });
-});
-
-describe("dropStorageTableSql", () => {
-   it("drops the content-addressed table, catalog-qualified and quoted", () => {
-      expect(dropStorageTableSql("lake", "daily__mabc123")).toBe(
-         'DROP TABLE IF EXISTS "lake"."daily__mabc123"',
-      );
-   });
-   it("qualifies a dotted schema.table name", () => {
-      expect(dropStorageTableSql("lake", "analytics.daily__mabc123")).toBe(
-         'DROP TABLE IF EXISTS "lake"."analytics"."daily__mabc123"',
-      );
-   });
-});
-
-describe("dropStorageTable gating (no session I/O before the gate)", () => {
-   it("refuses a non-DuckDB-family destination before creating a session", async () => {
-      await expect(
-         dropStorageTable({
-            destinationName: "bq_dest",
-            destinationConnection: {
-               name: "bq_dest",
-               type: "bigquery",
-            } as ApiConnection,
-            physicalTableName: "t__mabc",
-            environmentPath: "/tmp/env",
-         }),
-      ).rejects.toThrow(/must be one of/i);
-   });
-});
-
-describe("wrapPassthrough", () => {
-   const SQL = "SELECT a, b FROM t WHERE s = 'x'";
-
-   it("bigquery: bigquery_query(handle, sql)", () => {
-      expect(wrapPassthrough("bigquery", "my-project", SQL)).toBe(
-         "SELECT * FROM bigquery_query('my-project', 'SELECT a, b FROM t WHERE s = ''x''')",
-      );
-   });
-
-   it("postgres: postgres_query(handle, sql)", () => {
-      expect(wrapPassthrough("postgres", "src_pg", SQL)).toBe(
-         "SELECT * FROM postgres_query('src_pg', 'SELECT a, b FROM t WHERE s = ''x''')",
-      );
-   });
-
-   it("snowflake: snowflake_query(sql, handle) — SQL FIRST", () => {
-      expect(wrapPassthrough("snowflake", "secret_snowflake_src", SQL)).toBe(
-         "SELECT * FROM snowflake_query('SELECT a, b FROM t WHERE s = ''x''', 'secret_snowflake_src')",
-      );
-   });
-
-   it("escapes single quotes in both the handle and the inner SQL", () => {
-      expect(wrapPassthrough("postgres", "a'b", "x'y")).toBe(
-         "SELECT * FROM postgres_query('a''b', 'x''y')",
-      );
-   });
-});
-
-describe("passthroughSourceType", () => {
-   it.each(["bigquery", "snowflake", "postgres"])("accepts %s", (type) => {
-      expect(
-         passthroughSourceType({ name: "s", type } as ApiConnection) as string,
-      ).toBe(type);
-   });
-
-   it.each(["duckdb", "ducklake", "mysql", "trino"])(
-      "refuses %s with a clean 422",
-      (type) => {
-         expect(() =>
-            passthroughSourceType({ name: "s", type } as ApiConnection),
-         ).toThrow(BadRequestError);
-      },
-   );
-});
-
-describe("buildSourceIntoStorage gating (no session I/O before the gates)", () => {
-   const okSource = { name: "wh", type: "postgres" } as ApiConnection;
-
-   it("refuses a non-DuckDB-family destination before creating a session", async () => {
-      await expect(
-         buildSourceIntoStorage({
-            destinationName: "bq_dest",
-            destinationConnection: {
-               name: "bq_dest",
-               type: "bigquery",
-            } as ApiConnection,
-            sourceConnection: okSource,
-            buildSQL: "SELECT 1",
-            physicalTableName: "t",
-            environmentPath: "/tmp/env",
-         }),
-      ).rejects.toThrow(/must be one of/i);
-   });
-
-   it("refuses an unsupported source type before creating a session", async () => {
-      await expect(
-         buildSourceIntoStorage({
-            destinationName: "lake",
-            destinationConnection: {
-               name: "lake",
-               type: "duckdb",
-            } as ApiConnection,
-            sourceConnection: {
-               name: "md",
-               type: "motherduck",
-            } as ApiConnection,
-            buildSQL: "SELECT 1",
-            physicalTableName: "t",
-            environmentPath: "/tmp/env",
-         }),
-      ).rejects.toThrow(/query-passthrough build supports/i);
-   });
-});
-
-// Skipped on Windows: these tests seed the parent table via one DuckDBConnection
-// to a plain-DuckDB *file*, then buildDownstreamIntoStorage ATTACHes the SAME
-// file in the same process. On Windows, DuckDB keeps an exclusive/cached handle
-// on a file database even after close(), so the re-open fails ("file is already
-// open in this process") — a same-process file-reopen limitation of the dev-only
-// duckdb-file destination (production materializes into DuckLake, a Postgres
-// catalog, with no file reopen). POSIX releases the handle promptly. The
-// stack-on-parent rebind mechanism's cross-platform correctness is pinned in-memory (no file) by
-// the "runs the downstream over the parent's stored rows" test in
-// materialization_serve_transform.spec.ts.
-describe.skipIf(process.platform === "win32")(
-   "buildDownstreamIntoStorage (stack-on-parent, real DuckDB)",
-   () => {
-      const parent: ServeBinding = {
-         sourceName: "daily_orders",
-         connectionName: "lake",
-         virtualHandle: "daily_h",
-         tablePath: "lake.daily_orders__mabc",
-         schema: [
-            { name: "order_date", type: "DATE" },
-            { name: "total", type: "DOUBLE" },
-         ],
-      };
-      const downstreamDefText =
-         "monthly_orders is daily_orders -> {\n" +
-         "  group_by: order_month is order_date.month\n" +
-         "  aggregate: monthly_total is total.sum()\n}";
-
-      /** Seed a plain-DuckDB "lake" file with the parent's content-addressed table. */
-      async function seedLake(dir: string): Promise<void> {
-         const seed = new DuckDBConnection("lake", join(dir, "lake.duckdb"));
-         try {
-            await seed.runSQL(
-               'CREATE TABLE "daily_orders__mabc" AS ' +
-                  "SELECT CAST('2026-01-01' AS DATE) AS order_date, 150.0 AS total " +
-                  "UNION ALL SELECT CAST('2026-01-02' AS DATE), 225.0 " +
-                  "UNION ALL SELECT CAST('2026-02-01' AS DATE), 99.0",
-            );
-         } finally {
-            await seed.close();
-         }
-      }
-
-      it("compiles the rebind model, runs the downstream over the parent, and captures the rolled-up schema", async () => {
-         // Drives the full build-session path against a real plain-DuckDB file
-         // destination: attach read-write → compile the transient rebind model →
-         // getSQL over the rebound parent → CTAS into the lake → DESCRIBE. A clean
-         // return proves the downstream's SQL compiled AND executed against the
-         // attached parent table, and the captured schema is exactly the roll-up's
-         // output columns (the parent's raw columns are gone — it aggregated them).
-         // Row VALUES are pinned in-memory in the serve-transform spec (a file
-         // reopened by a second DuckDB instance is not reliably visible under load;
-         // production serves via a transactional catalog, not a file reopen).
-         const dir = mkdtempSync(join(tmpdir(), "t3-build-"));
-         try {
-            await seedLake(dir);
-            const result = await buildDownstreamIntoStorage({
-               destinationName: "lake",
-               destinationConnection: {
-                  name: "lake",
-                  type: "duckdb",
-               } as ApiConnection,
-               transientModel: buildChainedStorageBuildModel({
-                  upstreams: [parent],
-                  downstreamName: "monthly_orders",
-                  downstreamDefText,
-                  destinationName: "lake",
-               }),
-               downstreamName: "monthly_orders",
-               virtualMap: buildVirtualMap([parent]),
-               physicalTableName: "monthly_orders__mdef456",
-               environmentPath: dir,
-            });
-
-            expect(result.storageConnectionName).toBe("lake");
-            expect(result.schema.map((c) => c.name).sort()).toEqual([
-               "monthly_total",
-               "order_month",
-            ]);
-         } finally {
-            rmSync(dir, { recursive: true, force: true });
-         }
-      });
-
-      it("throws (⇒ caller falls back) when the downstream references a source the rebind model does not provide", async () => {
-         const dir = mkdtempSync(join(tmpdir(), "t3-miss-"));
-         try {
-            await seedLake(dir);
-            await expect(
-               buildDownstreamIntoStorage({
-                  destinationName: "lake",
-                  destinationConnection: {
-                     name: "lake",
-                     type: "duckdb",
-                  } as ApiConnection,
-                  // The downstream reads `weekly_orders`, which is NOT rebound here —
-                  // the transient model can't compile, mirroring the mixed-engine /
-                  // uncarried-parent case that must fall back to recompute-from-raw.
-                  transientModel: buildChainedStorageBuildModel({
-                     upstreams: [parent],
-                     downstreamName: "monthly_orders",
-                     downstreamDefText:
-                        "monthly_orders is weekly_orders -> { group_by: x is 1 }",
-                     destinationName: "lake",
-                  }),
-                  downstreamName: "monthly_orders",
-                  virtualMap: buildVirtualMap([parent]),
-                  physicalTableName: "monthly_orders__mdef456",
-                  environmentPath: dir,
-               }),
-            ).rejects.toThrow();
-         } finally {
-            rmSync(dir, { recursive: true, force: true });
-         }
-      });
-   },
-);
-
-describe("createTableAndDescribe (schema read-back window)", () => {
-   /** A session that CTASs fine and fails the read-back, recording every stmt. */
-   const sessionThatFailsDescribe = (opts: { dropFails?: boolean } = {}) => {
-      const issued: string[] = [];
-      const session = {
-         runSQL: async (sql: string) => {
-            issued.push(sql);
-            if (/^DESCRIBE/i.test(sql.trim())) throw new Error("describe boom");
-            if (/^DROP TABLE/i.test(sql.trim()) && opts.dropFails) {
-               throw new Error("drop boom");
-            }
-            return { rows: [] };
-         },
-      } as unknown as DuckDBConnection;
-      return { session, issued };
-   };
-
-   it("drops the just-created table when the schema read-back fails", async () => {
-      // The CTAS has committed by now and the caller records nothing until this
-      // returns — so without the drop the table is reachable by neither the
-      // failed-run reclaim nor manifest GC, which only drop names they recorded.
-      const { session, issued } = sessionThatFailsDescribe();
-
-      await expect(
-         createTableAndDescribe(session, `"lake"."t"`, "SELECT 1"),
-      ).rejects.toThrow(/describe boom/);
-
-      expect(issued.some((s) => /^CREATE OR REPLACE TABLE/i.test(s))).toBe(
-         true,
-      );
-      expect(
-         issued.some((s) => /^DROP TABLE IF EXISTS "lake"\."t"/i.test(s)),
-      ).toBe(true);
-   });
-
-   it("surfaces the read-back failure, not a failure of the cleanup drop", async () => {
-      // The DESCRIBE error is the diagnosis; a drop that also fails must not
-      // replace it (the leak is then logged and still surfaced by the warning).
-      const { session } = sessionThatFailsDescribe({ dropFails: true });
-
-      await expect(
-         createTableAndDescribe(session, `"lake"."t"`, "SELECT 1"),
-      ).rejects.toThrow(/describe boom/);
-   });
-
-   it("issues no drop on the happy path", async () => {
-      const issued: string[] = [];
-      const session = {
-         runSQL: async (sql: string) => {
-            issued.push(sql);
-            return {
-               rows: [{ column_name: "a", column_type: "VARCHAR" }],
-            };
-         },
-      } as unknown as DuckDBConnection;
-
-      const schema = await createTableAndDescribe(
-         session,
-         `"lake"."t"`,
-         "SELECT 1",
-      );
-
-      expect(schema).toEqual([{ name: "a", type: "VARCHAR" }]);
-      expect(issued.some((s) => /^DROP TABLE/i.test(s))).toBe(false);
-   });
-});