@malloy-publisher/server 0.0.203 → 0.0.204

package/src/service/materialization_service.ts

@@ -27,7 +27,7 @@ import {
    liveTableKey,
 } from "./materialized_table_gc";
 import { Model } from "./model";
-import { quoteTablePath, splitTablePath } from "./quoting";
+import { splitTablePath } from "./quoting";
 import { resolveEnvironmentId } from "./resolve_environment";
 
 /**
@@ -91,12 +91,17 @@ export function manifestTableKey(
  * running a zero-row SELECT. Returns `true` if the table resolves,
  * `false` if the query fails (assumed "table not found").
  */
+/**
+ * `tableName` is interpolated verbatim into the probe SQL — the caller
+ * supplies it already quoted for the dialect (the `#@ persist name=…`
+ * contract), matching how Malloy substitutes the name on the read side.
+ */
 export async function tablePhysicallyExists(
    connection: MalloyConnection,
-   quotedTableName: string,
+   tableName: string,
 ): Promise<boolean> {
    try {
-      await connection.runSQL(`SELECT 1 FROM ${quotedTableName} WHERE 1=0`);
+      await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
       return true;
    } catch {
       return false;
@@ -731,7 +736,7 @@ export class MaterializationService {
 
          // getBuildPlan() throws if the tag is missing, so check first to
          // keep plain models in the same package buildable.
-         const modelTag = malloyModel.tagParse({ prefix: /^##! / }).tag;
+         const modelTag = malloyModel.annotations.parseAsTag("!").tag;
          if (!modelTag.has("experimental", "persistence")) {
             logger.debug(
                "Model has no ##! experimental.persistence tag, skipping",
@@ -775,7 +780,7 @@ export class MaterializationService {
       const tableOwners = new Map<string, string>();
       for (const [sourceID, source] of Object.entries(allSources)) {
          const tableName =
-            source.tagParse({ prefix: /^#@ / }).tag.text("name") || source.name;
+            source.annotations.parseAsTag("@").tag.text("name") || source.name;
          const key = `${source.connectionName}::${tableName}`;
          const existing = tableOwners.get(key);
          if (existing) {
@@ -844,12 +849,16 @@ export class MaterializationService {
 
       const connectionName = persistSource.connectionName;
       const tableName =
-         persistSource.tagParse({ prefix: /^#@ / }).tag.text("name") ||
+         persistSource.annotations.parseAsTag("@").tag.text("name") ||
          persistSource.name;
-      const { schemaPrefix, bareName } = splitTablePath(tableName);
-      const stagingTableName = `${schemaPrefix}${bareName}${stagingSuffix(buildId)}`;
-      const dialect = persistSource.dialect;
-      const quoted = (p: string) => quoteTablePath(p, dialect);
+      const { bareName } = splitTablePath(tableName);
+      const stagingTableName = `${tableName}${stagingSuffix(buildId)}`;
+
+      // Table names go into DDL verbatim. Malloy assumes a table name handed
+      // to it (here, via the build manifest) is already quoted for the
+      // dialect and substitutes it into generated SQL as-is; our DDL has to
+      // match that exact identifier or the CREATE and the read diverge. The
+      // model author owns quoting the `#@ persist name=...` value.
 
       // Guard: refuse to overwrite a pre-existing table that was not
       // created by a previous materialization build. Without this check a
@@ -858,7 +867,7 @@ export class MaterializationService {
       // DROP TABLE below would silently destroy it.
       const tableKey = manifestTableKey(connectionName, tableName);
       if (!knownMaterializedTables.has(tableKey)) {
-         if (await tablePhysicallyExists(connection, quoted(tableName))) {
+         if (await tablePhysicallyExists(connection, tableName)) {
             throw new BadRequestError(
                `Refusing to materialize source '${persistSource.name}': ` +
                   `target table '${tableName}' already exists on connection ` +
@@ -877,26 +886,22 @@ export class MaterializationService {
 
       const startTime = performance.now();
 
-      await connection.runSQL(
-         `DROP TABLE IF EXISTS ${quoted(stagingTableName)}`,
-      );
+      await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
 
       // If any step after CREATE throws we must best-effort drop the
       // staging table, else it orphans under a name that GC will never
       // find (no manifest row is written for a failed build).
       try {
          await connection.runSQL(
-            `CREATE TABLE ${quoted(stagingTableName)} AS (${buildSQL})`,
+            `CREATE TABLE ${stagingTableName} AS (${buildSQL})`,
          );
-         await connection.runSQL(`DROP TABLE IF EXISTS ${quoted(tableName)}`);
+         await connection.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
          await connection.runSQL(
-            `ALTER TABLE ${quoted(stagingTableName)} RENAME TO ${dialect.quoteTablePath(bareName)}`,
+            `ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`,
          );
       } catch (err) {
          try {
-            await connection.runSQL(
-               `DROP TABLE IF EXISTS ${quoted(stagingTableName)}`,
-            );
+            await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
          } catch (cleanupErr) {
             logger.warn(
                "Build: failed to clean up staging table after a failed rebuild; physical leak",

package/src/service/materialized_table_gc.spec.ts

@@ -174,7 +174,7 @@ describe("dropManifestEntries", () => {
       );
    });
 
-   it("skips DROP and records an error when the dialect is unknown", async () => {
+   it("drops regardless of connection dialect — DDL uses table names verbatim", async () => {
       const ctx2 = makeCtx("martian-sql");
       const entry = makeEntry();
 
@@ -184,10 +184,11 @@ describe("dropManifestEntries", () => {
          environmentId: "proj-1",
       });
 
-      expect(result.dropped).toHaveLength(0);
-      expect(result.errors).toHaveLength(1);
-      expect(result.errors[0].error).toMatch(/martian-sql/);
-      expect(ctx2.conn.runSQL.called).toBe(false);
+      // GC no longer needs a per-dialect quoter: table names are used
+      // verbatim, so an unrecognized dialect is no longer a special case.
+      expect(result.dropped).toHaveLength(1);
+      expect(result.errors).toHaveLength(0);
+      expect(ctx2.conn.runSQL.callCount).toBe(2);
    });
 
    it("dryRun lists what would drop without issuing SQL or deleting rows", async () => {

package/src/service/materialized_table_gc.ts

@@ -1,37 +1,8 @@
 import type { Connection } from "@malloydata/malloy";
-import {
-   DatabricksDialect,
-   DuckDBDialect,
-   MySQLDialect,
-   PostgresDialect,
-   SnowflakeDialect,
-   StandardSQLDialect,
-   TrinoDialect,
-} from "@malloydata/malloy";
 import { logger } from "../logger";
 import { ManifestEntry } from "../storage/DatabaseInterface";
 import { ManifestService } from "./manifest_service";
 import { stagingSuffix } from "./materialization_service";
-import { type Quoter, quoteTablePath } from "./quoting";
-
-/**
- * Registry of built-in dialects keyed by `Connection.dialectName`. Malloy's
- * internal `getDialect` helper isn't part of the package's public exports,
- * so we assemble our own registry from the exported dialect classes.
- *
- * Note: `presto` (extends `TrinoDialect`) is not re-exported publicly and
- * is niche enough to omit; if/when it ships as a publisher connection type,
- * add it here.
- */
-const DIALECTS: Readonly<Record<string, Quoter>> = Object.freeze({
-   duckdb: new DuckDBDialect(),
-   standardsql: new StandardSQLDialect(),
-   trino: new TrinoDialect(),
-   postgres: new PostgresDialect(),
-   snowflake: new SnowflakeDialect(),
-   mysql: new MySQLDialect(),
-   databricks: new DatabricksDialect(),
-});
 
 /** Build a stable key for a `(connectionName, tableName)` tuple. */
 export function liveTableKey(
@@ -154,19 +125,6 @@ async function processOneEntry(
       };
    }
 
-   // ── Unknown dialect ───────────────────────────────────────────
-   const dialect = DIALECTS[connection.dialectName];
-   if (!dialect) {
-      return {
-         error: {
-            buildId: entry.buildId,
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            error: `No dialect registered for '${connection.dialectName}'`,
-         },
-      };
-   }
-
    // ── Dry run ───────────────────────────────────────────────────
    if (ctx.dryRun) {
       return {
@@ -181,8 +139,6 @@ async function processOneEntry(
    }
 
    // ── Live run: delete manifest row first ───────────────────────
-   const quoted = (p: string) => quoteTablePath(p, dialect);
-
    try {
       await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
    } catch (err) {
@@ -214,9 +170,7 @@ async function processOneEntry(
       );
    } else {
       try {
-         await connection.runSQL(
-            `DROP TABLE IF EXISTS ${quoted(entry.tableName)}`,
-         );
+         await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
       } catch (err) {
          logger.warn(
             "GC: deleted manifest row but failed to drop materialized table (orphaned)",
@@ -231,9 +185,7 @@ async function processOneEntry(
 
    // ── Best-effort drop staging companion ────────────────────────
    try {
-      await connection.runSQL(
-         `DROP TABLE IF EXISTS ${quoted(stagingTableName)}`,
-      );
+      await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
    } catch (err) {
       logger.warn("GC: failed to drop staging table (best-effort)", {
          stagingTableName,

package/src/service/model.spec.ts

@@ -191,6 +191,84 @@ describe("service/model", () => {
 
             sinon.restore();
          });
+
+         it("embeds model-level givens in each newSources SourceInfo", async () => {
+            const sourceInfo = {
+               name: "carriers",
+               schema: { fields: [] },
+            };
+            const givens = [
+               {
+                  name: "region",
+                  type: "string",
+                  annotations: ["#(doc) Region"],
+               },
+            ];
+            const model = new Model(
+               packageName,
+               "test.malloynb",
+               {},
+               "notebook",
+               undefined, // modelMaterializer
+               undefined, // modelDef
+               undefined, // sources
+               undefined, // queries
+               undefined, // sourceInfos
+               [
+                  {
+                     type: "code",
+                     text: "import 'carriers.malloy'",
+                     newSources: [sourceInfo],
+                  },
+               ], // runnableNotebookCells
+               undefined, // compilationError
+               undefined, // filterMap
+               givens, // givens
+            );
+
+            const notebook = await model.getNotebook();
+            expect(notebook.notebookCells).toHaveLength(1);
+            const parsed = JSON.parse(
+               notebook.notebookCells![0].newSources![0],
+            );
+            expect(parsed.name).toBe("carriers");
+            // SourceInfo fields are preserved untouched.
+            expect(parsed.schema).toEqual({ fields: [] });
+            // Givens ride along verbatim — no second getModel round-trip needed.
+            expect(parsed.givens).toEqual(givens);
+         });
+
+         it("omits givens from newSources when the model declares none", async () => {
+            const sourceInfo = { name: "carriers", schema: { fields: [] } };
+            const model = new Model(
+               packageName,
+               "test.malloynb",
+               {},
+               "notebook",
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+               [
+                  {
+                     type: "code",
+                     text: "import 'carriers.malloy'",
+                     newSources: [sourceInfo],
+                  },
+               ],
+               undefined,
+               undefined,
+               undefined, // no givens
+            );
+
+            const notebook = await model.getNotebook();
+            const parsed = JSON.parse(
+               notebook.notebookCells![0].newSources![0],
+            );
+            expect(parsed.name).toBe("carriers");
+            expect(parsed).not.toHaveProperty("givens");
+         });
       });
 
       describe("getQueryResults", () => {
@@ -239,6 +317,78 @@ describe("service/model", () => {
             sinon.restore();
          });
 
+         // Both caller-driven compile paths — the free-form `query` text and the
+         // `run: source->view` string built from `sourceName`/`queryName` — must
+         // go through restricted mode. The trusted `loadQuery` is reserved for
+         // author-curated content (notebook cells) and must never be reached from
+         // `getQueryResults`. These tests pin the dispatch so a regression that
+         // re-routes either path back to `loadQuery` is caught.
+         describe("compile dispatch", () => {
+            function buildDispatchModel(): {
+               model: Model;
+               loadQuery: sinon.SinonStub;
+               loadRestrictedQuery: sinon.SinonStub;
+            } {
+               // getPreparedResult rejects so execution stops right after the
+               // loader call; we only assert which loader was invoked.
+               const runnableStub = {
+                  getPreparedResult: sinon
+                     .stub()
+                     .rejects(new MalloyError("stub-stop", [])),
+                  run: sinon.stub().rejects(new MalloyError("stub-stop", [])),
+               };
+               const loadQuery = sinon.stub().returns(runnableStub);
+               const loadRestrictedQuery = sinon.stub().returns(runnableStub);
+               const modelMaterializer = { loadQuery, loadRestrictedQuery };
+               const model = new Model(
+                  packageName,
+                  mockModelPath,
+                  {},
+                  "model",
+                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                  modelMaterializer as any,
+                  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                  { contents: {}, exports: [], queryList: [] } as any,
+                  undefined,
+                  undefined,
+                  undefined,
+                  undefined,
+                  undefined,
+               );
+               return { model, loadQuery, loadRestrictedQuery };
+            }
+
+            afterEach(() => sinon.restore());
+
+            it("compiles ad-hoc query text in restricted mode, never trusted loadQuery", async () => {
+               const { model, loadQuery, loadRestrictedQuery } =
+                  buildDispatchModel();
+
+               await expect(
+                  model.getQueryResults(
+                     undefined,
+                     undefined,
+                     "run: orders -> { aggregate: c is count() }",
+                  ),
+               ).rejects.toThrow(MalloyError);
+
+               expect(loadRestrictedQuery.calledOnce).toBe(true);
+               expect(loadQuery.called).toBe(false);
+            });
+
+            it("compiles the named source/view path in restricted mode, never trusted loadQuery", async () => {
+               const { model, loadQuery, loadRestrictedQuery } =
+                  buildDispatchModel();
+
+               await expect(
+                  model.getQueryResults("orders", "summary"),
+               ).rejects.toThrow(MalloyError);
+
+               expect(loadRestrictedQuery.calledOnce).toBe(true);
+               expect(loadQuery.called).toBe(false);
+            });
+         });
+
          it("forwards givens to runnable.getPreparedResult and .run", async () => {
             const givensArg = { region: "EU" };
             const preparedResultStub = sinon
@@ -247,11 +397,13 @@ describe("service/model", () => {
             const runStub = sinon
                .stub()
                .rejects(new MalloyError("stub-stop", []));
+            const runnableStub = {
+               getPreparedResult: preparedResultStub,
+               run: runStub,
+            };
             const modelMaterializer = {
-               loadQuery: sinon.stub().returns({
-                  getPreparedResult: preparedResultStub,
-                  run: runStub,
-               }),
+               loadQuery: sinon.stub().returns(runnableStub),
+               loadRestrictedQuery: sinon.stub().returns(runnableStub),
             };
 
             const model = new Model(
@@ -347,11 +499,13 @@ describe("service/model", () => {
                         typeof API.util.wrapResult
                      >,
                   );
+               const runnableStub = {
+                  getPreparedResult: preparedResultStub,
+                  run: runStub,
+               };
                const modelMaterializer = {
-                  loadQuery: sinon.stub().returns({
-                     getPreparedResult: preparedResultStub,
-                     run: runStub,
-                  }),
+                  loadQuery: sinon.stub().returns(runnableStub),
+                  loadRestrictedQuery: sinon.stub().returns(runnableStub),
                };
                const model = new Model(
                   packageName,
@@ -529,6 +683,47 @@ describe("service/model", () => {
 
             sinon.restore();
          });
+
+         it("embeds model-level givens in executed cell newSources", async () => {
+            const sourceInfo = { name: "carriers", schema: { fields: [] } };
+            const givens = [
+               {
+                  name: "region",
+                  type: "string",
+                  annotations: ["#(doc) Region"],
+               },
+            ];
+            // A source-only code cell (no runnable) still emits newSources.
+            const runnableCells = [
+               {
+                  type: "code" as const,
+                  text: "import 'carriers.malloy'",
+                  newSources: [sourceInfo],
+               },
+            ];
+
+            const model = new Model(
+               packageName,
+               "test.malloynb",
+               {},
+               "notebook",
+               undefined, // modelMaterializer
+               undefined, // modelDef
+               undefined, // sources
+               undefined, // queries
+               undefined, // sourceInfos
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               runnableCells as any, // runnableNotebookCells
+               undefined, // compilationError
+               undefined, // filterMap
+               givens, // givens
+            );
+
+            const result = await model.executeNotebookCell(0);
+            const parsed = JSON.parse(result.newSources![0]);
+            expect(parsed.name).toBe("carriers");
+            expect(parsed.givens).toEqual(givens);
+         });
       });
    });