@malloy-publisher/server 0.0.210 → 0.0.212

package/src/materialization_metrics.ts

@@ -7,8 +7,8 @@
  * the run-duration histogram carries `mode` so a dashboard can render auto-run
  * vs orchestrated build latency side by side.
  *
- * Lazy init for the same reason as {@link ./query_cap_metrics}: instruments
- * created before `setGlobalMeterProvider` bind to a NoOp meter
+ * Instruments are created lazily for the same reason as {@link ./query_cap_metrics}:
+ * one created before `setGlobalMeterProvider` binds to a NoOp meter
  * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
  */
 
@@ -19,49 +19,83 @@ export type MaterializationOutcome = "success" | "failed" | "cancelled";
 /** Manifest bind outcome: timeout is split out from generic failure on purpose. */
 export type ManifestBindOutcome = "success" | "failure" | "timeout";
 
-let runCounter: Counter | null = null;
-let runDuration: Histogram | null = null;
-let sourcesCounter: Counter | null = null;
-let buildPlanComputeDuration: Histogram | null = null;
-let autoLoadCounter: Counter | null = null;
-let connectionDigestSkipCounter: Counter | null = null;
-let manifestBindCounter: Counter | null = null;
-let sourceBuildDuration: Histogram | null = null;
-let dropTablesCounter: Counter | null = null;
+const resetHooks: (() => void)[] = [];
 
-function ensureTelemetry(): { counter: Counter; duration: Histogram } {
-   if (runCounter && runDuration) {
-      return { counter: runCounter, duration: runDuration };
-   }
-   const meter = metrics.getMeter("publisher");
-   if (!runCounter) {
-      runCounter = meter.createCounter("publisher_materialization_runs_total", {
-         description:
-            "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').",
-      });
-   }
-   if (!runDuration) {
-      runDuration = meter.createHistogram(
-         "publisher_materialization_run_duration_ms",
-         {
-            description:
-               "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
-            unit: "ms",
-         },
-      );
-   }
-   return { counter: runCounter, duration: runDuration };
+/**
+ * A lazily-created instrument: built from the global meter on first use (so it
+ * binds to the real MeterProvider, not the NoOp default) and memoized. Each
+ * registers a reset hook so a test can swap in a fresh MeterProvider.
+ */
+function lazyCounter(name: string, description: string): () => Counter {
+   let instrument: Counter | null = null;
+   resetHooks.push(() => (instrument = null));
+   return () =>
+      (instrument ??= metrics
+         .getMeter("publisher")
+         .createCounter(name, { description }));
 }
 
+function lazyHistogram(
+   name: string,
+   description: string,
+   unit: string,
+): () => Histogram {
+   let instrument: Histogram | null = null;
+   resetHooks.push(() => (instrument = null));
+   return () =>
+      (instrument ??= metrics
+         .getMeter("publisher")
+         .createHistogram(name, { description, unit }));
+}
+
+const runCounter = lazyCounter(
+   "publisher_materialization_runs_total",
+   "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').",
+);
+const runDuration = lazyHistogram(
+   "publisher_materialization_run_duration_ms",
+   "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
+   "ms",
+);
+const sourcesCounter = lazyCounter(
+   "publisher_materialization_sources_total",
+   "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').",
+);
+const buildPlanComputeDuration = lazyHistogram(
+   "publisher_materialization_build_plan_compute_duration_ms",
+   "Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
+   "ms",
+);
+const autoLoadCounter = lazyCounter(
+   "publisher_materialization_auto_load_total",
+   "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').",
+);
+const connectionDigestSkipCounter = lazyCounter(
+   "publisher_materialization_connection_digest_skipped_total",
+   "Connection digests skipped during build-plan compile because the connection did not resolve.",
+);
+const manifestBindCounter = lazyCounter(
+   "publisher_materialization_manifest_bind_total",
+   "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
+);
+const sourceBuildDuration = lazyHistogram(
+   "publisher_materialization_source_build_duration_ms",
+   "Wall-clock duration of building a single persist source.",
+   "ms",
+);
+const dropTablesCounter = lazyCounter(
+   "publisher_materialization_drop_tables_total",
+   "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
+);
+
 /** Record the outcome and duration of a materialization build. */
 export function recordMaterializationRun(
    mode: MaterializationMode,
    outcome: MaterializationOutcome,
    durationMs: number,
 ): void {
-   const { counter, duration } = ensureTelemetry();
-   counter.add(1, { mode, outcome });
-   duration.record(durationMs, { mode });
+   runCounter().add(1, { mode, outcome });
+   runDuration().record(durationMs, { mode });
 }
 
 /**
@@ -74,15 +108,7 @@ export function recordSourcesOutcome(
    count: number,
 ): void {
    if (count <= 0) return;
-   if (!sourcesCounter) {
-      sourcesCounter = metrics
-         .getMeter("publisher")
-         .createCounter("publisher_materialization_sources_total", {
-            description:
-               "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').",
-         });
-   }
-   sourcesCounter.add(count, { outcome });
+   sourcesCounter().add(count, { outcome });
 }
 
 /**
@@ -91,19 +117,7 @@ export function recordSourcesOutcome(
  * tracking as a discrete cost separate from the build itself.
  */
 export function recordBuildPlanComputeDuration(durationMs: number): void {
-   if (!buildPlanComputeDuration) {
-      buildPlanComputeDuration = metrics
-         .getMeter("publisher")
-         .createHistogram(
-            "publisher_materialization_build_plan_compute_duration_ms",
-            {
-               description:
-                  "Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
-               unit: "ms",
-            },
-         );
-   }
-   buildPlanComputeDuration.record(durationMs);
+   buildPlanComputeDuration().record(durationMs);
 }
 
 /**
@@ -112,15 +126,7 @@ export function recordBuildPlanComputeDuration(durationMs: number): void {
  * the run is MANIFEST_FILE_READY but queries still resolve to the old tables.
  */
 export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
-   if (!autoLoadCounter) {
-      autoLoadCounter = metrics
-         .getMeter("publisher")
-         .createCounter("publisher_materialization_auto_load_total", {
-            description:
-               "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').",
-         });
-   }
-   autoLoadCounter.add(1, { outcome });
+   autoLoadCounter().add(1, { outcome });
 }
 
 /**
@@ -129,18 +135,7 @@ export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
  * are computed without a digest, so this is a correctness signal, not just noise.
  */
 export function recordConnectionDigestSkipped(): void {
-   if (!connectionDigestSkipCounter) {
-      connectionDigestSkipCounter = metrics
-         .getMeter("publisher")
-         .createCounter(
-            "publisher_materialization_connection_digest_skipped_total",
-            {
-               description:
-                  "Connection digests skipped during build-plan compile because the connection did not resolve.",
-            },
-         );
-   }
-   connectionDigestSkipCounter.add(1);
+   connectionDigestSkipCounter().add(1);
 }
 
 /**
@@ -149,56 +144,20 @@ export function recordConnectionDigestSkipped(): void {
  * tell an unreachable/slow manifest store from a malformed manifest.
  */
 export function recordManifestBind(outcome: ManifestBindOutcome): void {
-   if (!manifestBindCounter) {
-      manifestBindCounter = metrics
-         .getMeter("publisher")
-         .createCounter("publisher_materialization_manifest_bind_total", {
-            description:
-               "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
-         });
-   }
-   manifestBindCounter.add(1, { outcome });
+   manifestBindCounter().add(1, { outcome });
 }
 
 /** Record the wall-clock duration of building one persist source's table. */
 export function recordSourceBuildDuration(durationMs: number): void {
-   if (!sourceBuildDuration) {
-      sourceBuildDuration = metrics
-         .getMeter("publisher")
-         .createHistogram(
-            "publisher_materialization_source_build_duration_ms",
-            {
-               description:
-                  "Wall-clock duration of building a single persist source.",
-               unit: "ms",
-            },
-         );
-   }
-   sourceBuildDuration.record(durationMs);
+   sourceBuildDuration().record(durationMs);
 }
 
 /** Record a best-effort physical-table drop on materialization delete. */
 export function recordDropTables(outcome: "success" | "failure"): void {
-   if (!dropTablesCounter) {
-      dropTablesCounter = metrics
-         .getMeter("publisher")
-         .createCounter("publisher_materialization_drop_tables_total", {
-            description:
-               "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
-         });
-   }
-   dropTablesCounter.add(1, { outcome });
+   dropTablesCounter().add(1, { outcome });
 }
 
 /** Visible for tests. Drops cached instruments so a fresh MeterProvider can capture emissions. */
 export function resetMaterializationTelemetryForTesting(): void {
-   runCounter = null;
-   runDuration = null;
-   sourcesCounter = null;
-   buildPlanComputeDuration = null;
-   autoLoadCounter = null;
-   connectionDigestSkipCounter = null;
-   manifestBindCounter = null;
-   sourceBuildDuration = null;
-   dropTablesCounter = null;
+   for (const reset of resetHooks) reset();
 }

package/src/service/build_plan.spec.ts

@@ -1,14 +1,19 @@
 import type { PersistSource } from "@malloydata/malloy";
 import { describe, expect, it } from "bun:test";
 import * as sinon from "sinon";
+import type { BuildGraph as MalloyBuildGraph } from "@malloydata/malloy";
 import {
+   compilePackageBuildPlan,
    computeBuildId,
    computePackageBuildPlan,
+   deriveAnnotationFields,
    deriveBuildPlan,
    flattenDependsOn,
+   iterGraphSources,
    resolvePackageConnections,
 } from "./build_plan";
 import { fakeSource } from "./materialization_test_fixtures";
+import { Model } from "./model";
 
 describe("flattenDependsOn", () => {
    it("maps nested dependsOn entries to a flat sourceID list", () => {
@@ -20,6 +25,130 @@ describe("flattenDependsOn", () => {
    });
 });
 
+describe("iterGraphSources", () => {
+   it("yields resolvable sources in dependency order, skipping missing ones", () => {
+      const a = fakeSource({ name: "a", buildId: "ba" });
+      const b = fakeSource({ name: "b", buildId: "bb" });
+      const graph = {
+         connectionName: "duckdb",
+         nodes: [
+            [{ sourceID: "a@m", dependsOn: [] }],
+            [
+               { sourceID: "missing@m", dependsOn: [] },
+               { sourceID: "b@m", dependsOn: [] },
+            ],
+         ],
+      } as unknown as MalloyBuildGraph;
+
+      const names = [...iterGraphSources(graph, { "a@m": a, "b@m": b })].map(
+         (s) => s.name,
+      );
+      expect(names).toEqual(["a", "b"]);
+   });
+
+   it("walks each root's nested dependsOn tree, deps before dependents", () => {
+      // root -> mid -> leaf, with only `root` at the graph's node level — this
+      // mirrors malloy getBuildPlan(): terminal persist sources are the nodes,
+      // and every transitive persist dependency is nested in dependsOn. All
+      // three must be yielded (so all get built), leaf-first so a downstream
+      // build reads its upstream's freshly materialized table.
+      const root = fakeSource({ name: "root", buildId: "br" });
+      const mid = fakeSource({ name: "mid", buildId: "bm" });
+      const leaf = fakeSource({ name: "leaf", buildId: "bl" });
+      const graph = {
+         connectionName: "duckdb",
+         nodes: [
+            [
+               {
+                  sourceID: "root@m",
+                  dependsOn: [
+                     {
+                        sourceID: "mid@m",
+                        dependsOn: [{ sourceID: "leaf@m", dependsOn: [] }],
+                     },
+                  ],
+               },
+            ],
+         ],
+      } as unknown as MalloyBuildGraph;
+
+      const names = [
+         ...iterGraphSources(graph, {
+            "root@m": root,
+            "mid@m": mid,
+            "leaf@m": leaf,
+         }),
+      ].map((s) => s.name);
+      expect(names).toEqual(["leaf", "mid", "root"]);
+   });
+
+   it("deduplicates a shared (diamond) dependency across roots", () => {
+      // r1 and r2 both depend on `shared`; it must be yielded exactly once and
+      // before both dependents.
+      const r1 = fakeSource({ name: "r1", buildId: "b1" });
+      const r2 = fakeSource({ name: "r2", buildId: "b2" });
+      const shared = fakeSource({ name: "shared", buildId: "bs" });
+      const graph = {
+         connectionName: "duckdb",
+         nodes: [
+            [
+               {
+                  sourceID: "r1@m",
+                  dependsOn: [{ sourceID: "shared@m", dependsOn: [] }],
+               },
+               {
+                  sourceID: "r2@m",
+                  dependsOn: [{ sourceID: "shared@m", dependsOn: [] }],
+               },
+            ],
+         ],
+      } as unknown as MalloyBuildGraph;
+
+      const names = [
+         ...iterGraphSources(graph, {
+            "r1@m": r1,
+            "r2@m": r2,
+            "shared@m": shared,
+         }),
+      ].map((s) => s.name);
+      expect(names).toEqual(["shared", "r1", "r2"]);
+   });
+});
+
+describe("deriveAnnotationFields", () => {
+   it("returns all key=value fields of the #@ persist annotation", () => {
+      const source = {
+         annotations: {
+            parseAsTag: () => ({
+               tag: {
+                  *entries() {
+                     yield ["name", { text: () => "engaged_events" }];
+                     yield ["realization", { text: () => "COPY" }];
+                  },
+               },
+            }),
+         },
+      } as unknown as PersistSource;
+
+      expect(deriveAnnotationFields(source)).toEqual({
+         name: "engaged_events",
+         realization: "COPY",
+      });
+   });
+
+   it("degrades to {} when the annotation is absent or unparseable", () => {
+      const source = {
+         annotations: {
+            parseAsTag: () => {
+               throw new Error("no @ annotation");
+            },
+         },
+      } as unknown as PersistSource;
+
+      expect(deriveAnnotationFields(source)).toEqual({});
+   });
+});
+
 describe("computeBuildId", () => {
    it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
       const makeBuildId = sinon.stub().returns("computed-id");
@@ -102,6 +231,29 @@ describe("deriveBuildPlan", () => {
    });
 });
 
+describe("compilePackageBuildPlan", () => {
+   it("skips .malloynb notebooks without compiling them", async () => {
+      // A notebook would throw on its `>>>` cell delimiter if compiled as a
+      // flat model, aborting the whole package plan; it must be skipped.
+      const getModelRuntime = sinon.stub(Model, "getModelRuntime");
+      try {
+         const pkg = {
+            getModelPaths: () => ["notes.malloynb"],
+            getPackagePath: () => "/test",
+            getMalloyConfig: () => ({}),
+            getMalloyConnection: async () => ({}),
+         } as unknown as Parameters<typeof compilePackageBuildPlan>[0];
+
+         const compiled = await compilePackageBuildPlan(pkg);
+
+         expect(compiled.graphs).toEqual([]);
+         expect(getModelRuntime.called).toBe(false);
+      } finally {
+         getModelRuntime.restore();
+      }
+   });
+});
+
 describe("computePackageBuildPlan", () => {
    it("returns null when the package declares no persist sources", async () => {
       const pkg = {

package/src/service/build_plan.ts

@@ -1,13 +1,16 @@
 import type {
    AtomicField,
    BuildGraph as MalloyBuildGraph,
+   BuildNode,
    MalloyConfig,
    Connection as MalloyConnection,
    PersistSource,
 } from "@malloydata/malloy";
 import { components } from "../api";
+import { MODEL_FILE_SUFFIX } from "../constants";
 import { logger } from "../logger";
 import { recordConnectionDigestSkipped } from "../materialization_metrics";
+import { errMessage } from "../utils";
 import { Model } from "./model";
 
 type WireBuildGraph = components["schemas"]["BuildGraph"];
@@ -53,12 +56,37 @@ export function deriveColumns(persistSource: PersistSource): WireColumn[] {
    } catch (err) {
       logger.warn("Failed to derive columns for persist source", {
          sourceID: persistSource.sourceID,
-         error: err instanceof Error ? err.message : String(err),
+         error: errMessage(err),
       });
       return [];
    }
 }
 
+/**
+ * All key=value fields of a source's `#@ persist` annotation (e.g.
+ * `{ name: "engaged_events", realization: "COPY" }`), degrading to {} if the
+ * annotation is absent or unparseable. The control plane consumes these — most
+ * importantly `name`, which it uses as the materialized table name (and which
+ * may carry a dialect-style container path like `dataset.table`). Returning the
+ * full set rather than a fixed subset means new persist directives flow through
+ * without a publisher change.
+ */
+export function deriveAnnotationFields(
+   persistSource: PersistSource,
+): Record<string, string> {
+   const out: Record<string, string> = {};
+   try {
+      const tag = persistSource.annotations.parseAsTag("@").tag;
+      for (const [key, value] of tag.entries()) {
+         const text = value.text();
+         if (text !== undefined) out[key] = text;
+      }
+   } catch {
+      // Degrade to {} — mirrors deriveColumns / selfAssignTableName.
+   }
+   return out;
+}
+
 /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
 export function flattenDependsOn(node: {
    dependsOn: { sourceID: string }[];
@@ -66,6 +94,51 @@ export function flattenDependsOn(node: {
    return node.dependsOn.map((d) => d.sourceID);
 }
 
+/**
+ * Yield every persist source of one graph in dependency order — each source's
+ * persist dependencies before the source itself — resolving each node's
+ * sourceID against the sources map and skipping nodes whose source is absent.
+ * Centralizes the graph→source walk shared by the planning and build loops.
+ *
+ * <p>Malloy's {@code getBuildPlan()} puts only the <em>root</em> persist sources
+ * (the terminals nothing else consumes) in {@code graph.nodes}; every transitive
+ * persist dependency is nested under a root in its recursive {@code dependsOn}
+ * tree (and present in {@code sources}). Walking only {@code graph.nodes} —
+ * which this used to do — therefore silently skips every intermediate persist
+ * source, so it never gets materialized (it only got a table by coincidence when
+ * it shared a buildId with a root). We post-order DFS the {@code dependsOn} tree
+ * so dependencies are built first (a downstream build can then read its upstream
+ * source's freshly materialized table), deduplicating shared (diamond)
+ * dependencies by sourceID so each is yielded once. This mirrors the canonical
+ * malloy-cli reference build loop (its {@code flattenBuildNodes} +
+ * dedup-by-sourceID; see malloydata/malloy-cli src/malloy/build_graph.ts).
+ */
+export function* iterGraphSources(
+   graph: MalloyBuildGraph,
+   sources: Record<string, PersistSource>,
+): Iterable<PersistSource> {
+   const seen = new Set<string>();
+
+   function* visit(node: BuildNode): Iterable<PersistSource> {
+      if (seen.has(node.sourceID)) return;
+      // Dependencies first: a source built later in the run resolves its
+      // upstream references against the tables built earlier (see the Manifest
+      // threading in executeInstructedBuild).
+      for (const dep of node.dependsOn) {
+         yield* visit(dep);
+      }
+      seen.add(node.sourceID);
+      const source = sources[node.sourceID];
+      if (source) yield source;
+   }
+
+   for (const level of graph.nodes) {
+      for (const node of level) {
+         yield* visit(node);
+      }
+   }
+}
+
 /**
  * The buildId for a persist source: a stable digest of its connection identity
  * and canonical SQL. Centralizes the (source, connectionDigests) call shape so
@@ -100,7 +173,7 @@ export async function resolvePackageConnections(
          map.set(name, await pkg.getMalloyConnection(name));
       } catch (err) {
          logger.warn(`Failed to resolve connection ${name}`, {
-            error: err instanceof Error ? err.message : String(err),
+            error: errMessage(err),
          });
       }
    }
@@ -121,6 +194,11 @@ export async function compilePackageBuildPlan(
    const allSources: Record<string, PersistSource> = {};
 
    for (const modelPath of pkg.getModelPaths()) {
+      // Only `.malloy` models declare persist sources. Skip `.malloynb`
+      // notebooks: getModel() parses a model file as a flat model and throws on
+      // the notebook's `>>>` cell delimiter, which would abort the entire
+      // package build plan and silently drop every persist source in it.
+      if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) continue;
       if (signal?.aborted) throw new Error("Build cancelled");
 
       const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(
@@ -211,6 +289,7 @@ export function deriveBuildPlan(
          buildId: computeBuildId(source, connectionDigests),
          sql: source.getSQL(),
          columns: deriveColumns(source),
+         annotationFields: deriveAnnotationFields(source),
       };
    }