@malloy-publisher/server 0.0.211 → 0.0.213

package/dist/server.mjs

@@ -230137,7 +230137,10 @@ async function listTablesForSchema(connection, schemaName, malloyConnection, tab
 async function listTablesForBigQuery(connection, schemaName, malloyConnection, tableNames) {
   try {
     const bigquery = createBigQueryClient(connection);
-    const dataset = bigquery.dataset(schemaName);
+    const lastDot = schemaName.lastIndexOf(".");
+    const dataset = lastDot === -1 ? bigquery.dataset(schemaName) : bigquery.dataset(schemaName.slice(lastDot + 1), {
+      projectId: schemaName.slice(0, lastDot)
+    });
     const [tables] = await dataset.getTables();
     let names = tables.map((table) => table.id).filter((id) => id !== undefined);
     if (tableNames) {
@@ -239287,11 +239290,21 @@ function flattenDependsOn(node) {
   return node.dependsOn.map((d) => d.sourceID);
 }
 function* iterGraphSources(graph, sources) {
+  const seen = new Set;
+  function* visit(node) {
+    if (seen.has(node.sourceID))
+      return;
+    for (const dep of node.dependsOn) {
+      yield* visit(dep);
+    }
+    seen.add(node.sourceID);
+    const source = sources[node.sourceID];
+    if (source)
+      yield source;
+  }
   for (const level of graph.nodes) {
     for (const node of level) {
-      const source = sources[node.sourceID];
-      if (source)
-        yield source;
+      yield* visit(node);
     }
   }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.211",
+   "version": "0.0.213",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/service/build_plan.spec.ts

@@ -45,6 +45,74 @@ describe("iterGraphSources", () => {
       );
       expect(names).toEqual(["a", "b"]);
    });
+
+   it("walks each root's nested dependsOn tree, deps before dependents", () => {
+      // root -> mid -> leaf, with only `root` at the graph's node level — this
+      // mirrors malloy getBuildPlan(): terminal persist sources are the nodes,
+      // and every transitive persist dependency is nested in dependsOn. All
+      // three must be yielded (so all get built), leaf-first so a downstream
+      // build reads its upstream's freshly materialized table.
+      const root = fakeSource({ name: "root", buildId: "br" });
+      const mid = fakeSource({ name: "mid", buildId: "bm" });
+      const leaf = fakeSource({ name: "leaf", buildId: "bl" });
+      const graph = {
+         connectionName: "duckdb",
+         nodes: [
+            [
+               {
+                  sourceID: "root@m",
+                  dependsOn: [
+                     {
+                        sourceID: "mid@m",
+                        dependsOn: [{ sourceID: "leaf@m", dependsOn: [] }],
+                     },
+                  ],
+               },
+            ],
+         ],
+      } as unknown as MalloyBuildGraph;
+
+      const names = [
+         ...iterGraphSources(graph, {
+            "root@m": root,
+            "mid@m": mid,
+            "leaf@m": leaf,
+         }),
+      ].map((s) => s.name);
+      expect(names).toEqual(["leaf", "mid", "root"]);
+   });
+
+   it("deduplicates a shared (diamond) dependency across roots", () => {
+      // r1 and r2 both depend on `shared`; it must be yielded exactly once and
+      // before both dependents.
+      const r1 = fakeSource({ name: "r1", buildId: "b1" });
+      const r2 = fakeSource({ name: "r2", buildId: "b2" });
+      const shared = fakeSource({ name: "shared", buildId: "bs" });
+      const graph = {
+         connectionName: "duckdb",
+         nodes: [
+            [
+               {
+                  sourceID: "r1@m",
+                  dependsOn: [{ sourceID: "shared@m", dependsOn: [] }],
+               },
+               {
+                  sourceID: "r2@m",
+                  dependsOn: [{ sourceID: "shared@m", dependsOn: [] }],
+               },
+            ],
+         ],
+      } as unknown as MalloyBuildGraph;
+
+      const names = [
+         ...iterGraphSources(graph, {
+            "r1@m": r1,
+            "r2@m": r2,
+            "shared@m": shared,
+         }),
+      ].map((s) => s.name);
+      expect(names).toEqual(["shared", "r1", "r2"]);
+   });
 });
 
 describe("deriveAnnotationFields", () => {

package/src/service/build_plan.ts

@@ -1,6 +1,7 @@
 import type {
    AtomicField,
    BuildGraph as MalloyBuildGraph,
+   BuildNode,
    MalloyConfig,
    Connection as MalloyConnection,
    PersistSource,
@@ -94,19 +95,46 @@ export function flattenDependsOn(node: {
 }
 
 /**
- * Yield each dependency-ordered persist source of one graph, resolving the
- * node's sourceID against the sources map and skipping nodes whose source is
- * absent. Centralizes the graph→levels→nodes→source walk shared by the
- * planning and build loops.
+ * Yield every persist source of one graph in dependency order — each source's
+ * persist dependencies before the source itself — resolving each node's
+ * sourceID against the sources map and skipping nodes whose source is absent.
+ * Centralizes the graph→source walk shared by the planning and build loops.
+ *
+ * <p>Malloy's {@code getBuildPlan()} puts only the <em>root</em> persist sources
+ * (the terminals nothing else consumes) in {@code graph.nodes}; every transitive
+ * persist dependency is nested under a root in its recursive {@code dependsOn}
+ * tree (and present in {@code sources}). Walking only {@code graph.nodes} —
+ * which this used to do — therefore silently skips every intermediate persist
+ * source, so it never gets materialized (it only got a table by coincidence when
+ * it shared a buildId with a root). We post-order DFS the {@code dependsOn} tree
+ * so dependencies are built first (a downstream build can then read its upstream
+ * source's freshly materialized table), deduplicating shared (diamond)
+ * dependencies by sourceID so each is yielded once. This mirrors the canonical
+ * malloy-cli reference build loop (its {@code flattenBuildNodes} +
+ * dedup-by-sourceID; see malloydata/malloy-cli src/malloy/build_graph.ts).
  */
 export function* iterGraphSources(
    graph: MalloyBuildGraph,
    sources: Record<string, PersistSource>,
 ): Iterable<PersistSource> {
+   const seen = new Set<string>();
+
+   function* visit(node: BuildNode): Iterable<PersistSource> {
+      if (seen.has(node.sourceID)) return;
+      // Dependencies first: a source built later in the run resolves its
+      // upstream references against the tables built earlier (see the Manifest
+      // threading in executeInstructedBuild).
+      for (const dep of node.dependsOn) {
+         yield* visit(dep);
+      }
+      seen.add(node.sourceID);
+      const source = sources[node.sourceID];
+      if (source) yield source;
+   }
+
    for (const level of graph.nodes) {
       for (const node of level) {
-         const source = sources[node.sourceID];
-         if (source) yield source;
+         yield* visit(node);
       }
    }
 }

package/src/service/db_utils.spec.ts

@@ -7,8 +7,16 @@ mock.module("@azure/identity", () => ({
 mock.module("@azure/storage-blob", () => ({
    ContainerClient: class {},
 }));
+// Records bigquery.dataset() calls so tests can assert how a (possibly
+// project-qualified) schema name is split into datasetId + projectId.
+const bqDatasetCalls: Array<{ id: string; options?: unknown }> = [];
 mock.module("@google-cloud/bigquery", () => ({
-   BigQuery: class {},
+   BigQuery: class {
+      dataset(id: string, options?: unknown) {
+         bqDatasetCalls.push({ id, options });
+         return { getTables: async () => [[]] };
+      }
+   },
 }));
 
 import { Connection } from "@malloydata/malloy";
@@ -1031,3 +1039,47 @@ describe("publisher proxy introspection", () => {
       );
    });
 });
+
+// ---------------------------------------------------------------------------
+// listTablesForSchema — bigquery (project-qualified dataset)
+// ---------------------------------------------------------------------------
+describe("listTablesForSchema bigquery", () => {
+   const bqConnection = {
+      name: "bq",
+      type: "bigquery",
+      bigqueryConnection: {
+         defaultProjectId: "default-proj",
+         serviceAccountKeyJson: JSON.stringify({ project_id: "default-proj" }),
+      },
+   } as unknown as ApiConnection;
+   const malloyConn = {} as unknown as Connection;
+
+   it("splits a project-qualified schema into a bare dataset id + projectId", async () => {
+      bqDatasetCalls.length = 0;
+      await listTablesForSchema(bqConnection, "myproj.myds", malloyConn);
+      expect(bqDatasetCalls).toHaveLength(1);
+      expect(bqDatasetCalls[0]?.id).toBe("myds");
+      expect(bqDatasetCalls[0]?.options).toEqual({ projectId: "myproj" });
+   });
+
+   it("splits on the last dot so domain-scoped project ids survive", async () => {
+      bqDatasetCalls.length = 0;
+      await listTablesForSchema(
+         bqConnection,
+         "domain.com:proj.myds",
+         malloyConn,
+      );
+      expect(bqDatasetCalls[0]?.id).toBe("myds");
+      expect(bqDatasetCalls[0]?.options).toEqual({
+         projectId: "domain.com:proj",
+      });
+   });
+
+   it("passes a bare dataset name through unqualified", async () => {
+      bqDatasetCalls.length = 0;
+      await listTablesForSchema(bqConnection, "myds", malloyConn);
+      expect(bqDatasetCalls).toHaveLength(1);
+      expect(bqDatasetCalls[0]?.id).toBe("myds");
+      expect(bqDatasetCalls[0]?.options).toBeUndefined();
+   });
+});

package/src/service/db_utils.ts

@@ -1077,7 +1077,20 @@ async function listTablesForBigQuery(
 ): Promise<ApiTable[]> {
    try {
       const bigquery = createBigQueryClient(connection);
-      const dataset = bigquery.dataset(schemaName);
+      // A 3-segment table reference ("project.dataset.table") reaches here with a
+      // project-qualified schema ("project.dataset"). bigquery.dataset() takes a
+      // BARE dataset id plus an optional projectId, so passing "project.dataset" as
+      // the id resolves to a non-existent dataset in the client's default project —
+      // the dataset's tables never list, so the orphan sweep can't see (and reclaim)
+      // those tables. Split the project off the last dot. (Domain-scoped project ids
+      // like "domain.com:project" keep their dots/colon since we split on the last.)
+      const lastDot = schemaName.lastIndexOf(".");
+      const dataset =
+         lastDot === -1
+            ? bigquery.dataset(schemaName)
+            : bigquery.dataset(schemaName.slice(lastDot + 1), {
+                 projectId: schemaName.slice(0, lastDot),
+              });
       const [tables] = await dataset.getTables();
 
       let names = tables

package/src/service/materialization_service.spec.ts

@@ -718,6 +718,66 @@ describe("executeInstructedBuild", () => {
       expect(entries["carried0"].physicalTableName).toBe("carried_tbl");
    });
 
+   it("builds an intermediate persist source nested under a root node", async () => {
+      // Mirrors malloy getBuildPlan(): only the terminal `root` is a graph
+      // node; its persist dependency `mid` is nested in dependsOn (not a node).
+      // Before the iterGraphSources fix, `mid` was silently never built even
+      // though the caller instructed it. Both must now build, `mid` first.
+      const runSQL = sinon.stub().resolves();
+      const connection = { runSQL } as unknown as MalloyConnection;
+      const root = fakeSource({ name: "root", buildId: "brootaaaaaaaaaa" });
+      const mid = fakeSource({ name: "mid", buildId: "bmidbbbbbbbbbbb" });
+      const compiled = {
+         graphs: [
+            {
+               connectionName: "duckdb",
+               nodes: [
+                  [
+                     {
+                        sourceID: "root",
+                        dependsOn: [{ sourceID: "mid", dependsOn: [] }],
+                     },
+                  ],
+               ],
+            },
+         ],
+         sources: { root, mid },
+         connectionDigests: { duckdb: "dig" },
+         connections: new Map([["duckdb", connection]]),
+      };
+
+      const entries = await callExecute(
+         compiled,
+         [
+            {
+               buildId: "brootaaaaaaaaaa",
+               materializedTableId: "mt-r",
+               physicalTableName: "root_v1",
+               realization: "COPY",
+            },
+            {
+               buildId: "bmidbbbbbbbbbbb",
+               materializedTableId: "mt-m",
+               physicalTableName: "mid_v1",
+               realization: "COPY",
+            },
+         ],
+         {},
+      );
+
+      expect(entries["bmidbbbbbbbbbbb"].physicalTableName).toBe("mid_v1");
+      expect(entries["brootaaaaaaaaaa"].physicalTableName).toBe("root_v1");
+      // The dependency's CREATE precedes its dependent's (dependency order).
+      const creates = runSQL
+         .getCalls()
+         .map((c) => c.args[0] as string)
+         .filter((s) => s.startsWith("CREATE TABLE"));
+      const midIdx = creates.findIndex((s) => s.includes("mid_v1"));
+      const rootIdx = creates.findIndex((s) => s.includes("root_v1"));
+      expect(midIdx).toBeGreaterThanOrEqual(0);
+      expect(rootIdx).toBeGreaterThan(midIdx);
+   });
+
    it("throws when an instructed graph's connection is missing", async () => {
       const s1 = fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" });
       const compiled = compiledWith({ s1 }, [["s1"]], new Map());

package/src/service/quoting.spec.ts

@@ -30,6 +30,35 @@ describe("quoteIdentifier", () => {
    });
 });
 
+// Cross-repo conformance (PR ms2data/service#5272 review item 1). The publisher
+// keys identifier quoting off the Malloy `dialectName`; the control plane keys
+// the SAME fact off the publisher *connection type* in
+// `PhysicalTableName.BACKTICK_TYPES` ({bigquery, mysql, databricks}). The two
+// must stay byte-compatible — a name the publisher CREATEs with backticks the CP
+// must DROP with backticks — but they live in different repos under different key
+// vocabularies (`standardsql` the dialect vs `bigquery` the connection type), so
+// drift produces malformed DDL with no compile error. This table pins the full
+// connection-type → dialect → quote-char correspondence; if it changes, update
+// `PhysicalTableName.BACKTICK_TYPES` in the control plane in lockstep.
+describe("dialect/connection-type quoting conformance", () => {
+   const QUOTING = [
+      { connectionType: "bigquery", dialect: "standardsql", quote: "`" },
+      { connectionType: "mysql", dialect: "mysql", quote: "`" },
+      { connectionType: "databricks", dialect: "databricks", quote: "`" },
+      { connectionType: "postgres", dialect: "postgres", quote: '"' },
+      { connectionType: "snowflake", dialect: "snowflake", quote: '"' },
+      { connectionType: "trino", dialect: "trino", quote: '"' },
+      { connectionType: "duckdb", dialect: "duckdb", quote: '"' },
+      { connectionType: "motherduck", dialect: "duckdb", quote: '"' },
+   ] as const;
+
+   for (const { connectionType, dialect, quote } of QUOTING) {
+      it(`${connectionType} (${dialect}) quotes with ${quote}`, () => {
+         expect(quoteIdentifier("x", dialect)).toBe(`${quote}x${quote}`);
+      });
+   }
+});
+
 describe("quoteTablePath", () => {
    it("quotes each segment of a container path independently", () => {
       expect(

package/src/service/quoting.ts

@@ -10,7 +10,10 @@ export function bareTableName(tableName: string): string {
 }
 
 // Dialects whose identifier quote character is a backtick; everything else uses
-// the SQL-standard double quote. Keyed by Malloy `dialectName`.
+// the SQL-standard double quote. Keyed by Malloy `dialectName`. The control
+// plane encodes the same fact keyed by connection type
+// (`PhysicalTableName.BACKTICK_TYPES` = {bigquery, mysql, databricks}); the two
+// must stay byte-compatible. See the conformance table in quoting.spec.ts.
 const BACKTICK_DIALECTS = new Set(["standardsql", "mysql", "databricks"]);
 
 /**