@malloy-publisher/server 0.0.211 → 0.0.212

package/dist/server.mjs

@@ -239287,11 +239287,21 @@ function flattenDependsOn(node) {
   return node.dependsOn.map((d) => d.sourceID);
 }
 function* iterGraphSources(graph, sources) {
+  const seen = new Set;
+  function* visit(node) {
+    if (seen.has(node.sourceID))
+      return;
+    for (const dep of node.dependsOn) {
+      yield* visit(dep);
+    }
+    seen.add(node.sourceID);
+    const source = sources[node.sourceID];
+    if (source)
+      yield source;
+  }
   for (const level of graph.nodes) {
     for (const node of level) {
-      const source = sources[node.sourceID];
-      if (source)
-        yield source;
+      yield* visit(node);
     }
   }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.211",
+   "version": "0.0.212",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/service/build_plan.spec.ts

@@ -45,6 +45,74 @@ describe("iterGraphSources", () => {
       );
       expect(names).toEqual(["a", "b"]);
    });
+
+   it("walks each root's nested dependsOn tree, deps before dependents", () => {
+      // root -> mid -> leaf, with only `root` at the graph's node level — this
+      // mirrors malloy getBuildPlan(): terminal persist sources are the nodes,
+      // and every transitive persist dependency is nested in dependsOn. All
+      // three must be yielded (so all get built), leaf-first so a downstream
+      // build reads its upstream's freshly materialized table.
+      const root = fakeSource({ name: "root", buildId: "br" });
+      const mid = fakeSource({ name: "mid", buildId: "bm" });
+      const leaf = fakeSource({ name: "leaf", buildId: "bl" });
+      const graph = {
+         connectionName: "duckdb",
+         nodes: [
+            [
+               {
+                  sourceID: "root@m",
+                  dependsOn: [
+                     {
+                        sourceID: "mid@m",
+                        dependsOn: [{ sourceID: "leaf@m", dependsOn: [] }],
+                     },
+                  ],
+               },
+            ],
+         ],
+      } as unknown as MalloyBuildGraph;
+
+      const names = [
+         ...iterGraphSources(graph, {
+            "root@m": root,
+            "mid@m": mid,
+            "leaf@m": leaf,
+         }),
+      ].map((s) => s.name);
+      expect(names).toEqual(["leaf", "mid", "root"]);
+   });
+
+   it("deduplicates a shared (diamond) dependency across roots", () => {
+      // r1 and r2 both depend on `shared`; it must be yielded exactly once and
+      // before both dependents.
+      const r1 = fakeSource({ name: "r1", buildId: "b1" });
+      const r2 = fakeSource({ name: "r2", buildId: "b2" });
+      const shared = fakeSource({ name: "shared", buildId: "bs" });
+      const graph = {
+         connectionName: "duckdb",
+         nodes: [
+            [
+               {
+                  sourceID: "r1@m",
+                  dependsOn: [{ sourceID: "shared@m", dependsOn: [] }],
+               },
+               {
+                  sourceID: "r2@m",
+                  dependsOn: [{ sourceID: "shared@m", dependsOn: [] }],
+               },
+            ],
+         ],
+      } as unknown as MalloyBuildGraph;
+
+      const names = [
+         ...iterGraphSources(graph, {
+            "r1@m": r1,
+            "r2@m": r2,
+            "shared@m": shared,
+         }),
+      ].map((s) => s.name);
+      expect(names).toEqual(["shared", "r1", "r2"]);
+   });
 });
 
 describe("deriveAnnotationFields", () => {

package/src/service/build_plan.ts

@@ -1,6 +1,7 @@
 import type {
    AtomicField,
    BuildGraph as MalloyBuildGraph,
+   BuildNode,
    MalloyConfig,
    Connection as MalloyConnection,
    PersistSource,
@@ -94,19 +95,46 @@ export function flattenDependsOn(node: {
 }
 
 /**
- * Yield each dependency-ordered persist source of one graph, resolving the
- * node's sourceID against the sources map and skipping nodes whose source is
- * absent. Centralizes the graph→levels→nodes→source walk shared by the
- * planning and build loops.
+ * Yield every persist source of one graph in dependency order — each source's
+ * persist dependencies before the source itself — resolving each node's
+ * sourceID against the sources map and skipping nodes whose source is absent.
+ * Centralizes the graph→source walk shared by the planning and build loops.
+ *
+ * <p>Malloy's {@code getBuildPlan()} puts only the <em>root</em> persist sources
+ * (the terminals nothing else consumes) in {@code graph.nodes}; every transitive
+ * persist dependency is nested under a root in its recursive {@code dependsOn}
+ * tree (and present in {@code sources}). Walking only {@code graph.nodes} —
+ * which this used to do — therefore silently skips every intermediate persist
+ * source, so it never gets materialized (it only got a table by coincidence when
+ * it shared a buildId with a root). We post-order DFS the {@code dependsOn} tree
+ * so dependencies are built first (a downstream build can then read its upstream
+ * source's freshly materialized table), deduplicating shared (diamond)
+ * dependencies by sourceID so each is yielded once. This mirrors the canonical
+ * malloy-cli reference build loop (its {@code flattenBuildNodes} +
+ * dedup-by-sourceID; see malloydata/malloy-cli src/malloy/build_graph.ts).
  */
 export function* iterGraphSources(
    graph: MalloyBuildGraph,
    sources: Record<string, PersistSource>,
 ): Iterable<PersistSource> {
+   const seen = new Set<string>();
+
+   function* visit(node: BuildNode): Iterable<PersistSource> {
+      if (seen.has(node.sourceID)) return;
+      // Dependencies first: a source built later in the run resolves its
+      // upstream references against the tables built earlier (see the Manifest
+      // threading in executeInstructedBuild).
+      for (const dep of node.dependsOn) {
+         yield* visit(dep);
+      }
+      seen.add(node.sourceID);
+      const source = sources[node.sourceID];
+      if (source) yield source;
+   }
+
    for (const level of graph.nodes) {
       for (const node of level) {
-         const source = sources[node.sourceID];
-         if (source) yield source;
+         yield* visit(node);
       }
    }
 }

package/src/service/materialization_service.spec.ts

@@ -718,6 +718,66 @@ describe("executeInstructedBuild", () => {
       expect(entries["carried0"].physicalTableName).toBe("carried_tbl");
    });
 
+   it("builds an intermediate persist source nested under a root node", async () => {
+      // Mirrors malloy getBuildPlan(): only the terminal `root` is a graph
+      // node; its persist dependency `mid` is nested in dependsOn (not a node).
+      // Before the iterGraphSources fix, `mid` was silently never built even
+      // though the caller instructed it. Both must now build, `mid` first.
+      const runSQL = sinon.stub().resolves();
+      const connection = { runSQL } as unknown as MalloyConnection;
+      const root = fakeSource({ name: "root", buildId: "brootaaaaaaaaaa" });
+      const mid = fakeSource({ name: "mid", buildId: "bmidbbbbbbbbbbb" });
+      const compiled = {
+         graphs: [
+            {
+               connectionName: "duckdb",
+               nodes: [
+                  [
+                     {
+                        sourceID: "root",
+                        dependsOn: [{ sourceID: "mid", dependsOn: [] }],
+                     },
+                  ],
+               ],
+            },
+         ],
+         sources: { root, mid },
+         connectionDigests: { duckdb: "dig" },
+         connections: new Map([["duckdb", connection]]),
+      };
+
+      const entries = await callExecute(
+         compiled,
+         [
+            {
+               buildId: "brootaaaaaaaaaa",
+               materializedTableId: "mt-r",
+               physicalTableName: "root_v1",
+               realization: "COPY",
+            },
+            {
+               buildId: "bmidbbbbbbbbbbb",
+               materializedTableId: "mt-m",
+               physicalTableName: "mid_v1",
+               realization: "COPY",
+            },
+         ],
+         {},
+      );
+
+      expect(entries["bmidbbbbbbbbbbb"].physicalTableName).toBe("mid_v1");
+      expect(entries["brootaaaaaaaaaa"].physicalTableName).toBe("root_v1");
+      // The dependency's CREATE precedes its dependent's (dependency order).
+      const creates = runSQL
+         .getCalls()
+         .map((c) => c.args[0] as string)
+         .filter((s) => s.startsWith("CREATE TABLE"));
+      const midIdx = creates.findIndex((s) => s.includes("mid_v1"));
+      const rootIdx = creates.findIndex((s) => s.includes("root_v1"));
+      expect(midIdx).toBeGreaterThanOrEqual(0);
+      expect(rootIdx).toBeGreaterThan(midIdx);
+   });
+
    it("throws when an instructed graph's connection is missing", async () => {
       const s1 = fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" });
       const compiled = compiledWith({ s1 }, [["s1"]], new Map());

package/src/service/quoting.spec.ts

@@ -30,6 +30,35 @@ describe("quoteIdentifier", () => {
    });
 });
 
+// Cross-repo conformance (PR ms2data/service#5272 review item 1). The publisher
+// keys identifier quoting off the Malloy `dialectName`; the control plane keys
+// the SAME fact off the publisher *connection type* in
+// `PhysicalTableName.BACKTICK_TYPES` ({bigquery, mysql, databricks}). The two
+// must stay byte-compatible — a name the publisher CREATEs with backticks the CP
+// must DROP with backticks — but they live in different repos under different key
+// vocabularies (`standardsql` the dialect vs `bigquery` the connection type), so
+// drift produces malformed DDL with no compile error. This table pins the full
+// connection-type → dialect → quote-char correspondence; if it changes, update
+// `PhysicalTableName.BACKTICK_TYPES` in the control plane in lockstep.
+describe("dialect/connection-type quoting conformance", () => {
+   const QUOTING = [
+      { connectionType: "bigquery", dialect: "standardsql", quote: "`" },
+      { connectionType: "mysql", dialect: "mysql", quote: "`" },
+      { connectionType: "databricks", dialect: "databricks", quote: "`" },
+      { connectionType: "postgres", dialect: "postgres", quote: '"' },
+      { connectionType: "snowflake", dialect: "snowflake", quote: '"' },
+      { connectionType: "trino", dialect: "trino", quote: '"' },
+      { connectionType: "duckdb", dialect: "duckdb", quote: '"' },
+      { connectionType: "motherduck", dialect: "duckdb", quote: '"' },
+   ] as const;
+
+   for (const { connectionType, dialect, quote } of QUOTING) {
+      it(`${connectionType} (${dialect}) quotes with ${quote}`, () => {
+         expect(quoteIdentifier("x", dialect)).toBe(`${quote}x${quote}`);
+      });
+   }
+});
+
 describe("quoteTablePath", () => {
    it("quotes each segment of a container path independently", () => {
       expect(

package/src/service/quoting.ts

@@ -10,7 +10,10 @@ export function bareTableName(tableName: string): string {
 }
 
 // Dialects whose identifier quote character is a backtick; everything else uses
-// the SQL-standard double quote. Keyed by Malloy `dialectName`.
+// the SQL-standard double quote. Keyed by Malloy `dialectName`. The control
+// plane encodes the same fact keyed by connection type
+// (`PhysicalTableName.BACKTICK_TYPES` = {bigquery, mysql, databricks}); the two
+// must stay byte-compatible. See the conformance table in quoting.spec.ts.
 const BACKTICK_DIALECTS = new Set(["standardsql", "mysql", "databricks"]);
 
 /**