@malloy-publisher/server 0.0.209 → 0.0.211

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.209",
+   "version": "0.0.211",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/materialization_metrics.spec.ts

@@ -0,0 +1,84 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+
+import {
+   recordConnectionDigestSkipped,
+   recordDropTables,
+   recordMaterializationRun,
+   recordSourcesOutcome,
+   resetMaterializationTelemetryForTesting,
+} from "./materialization_metrics";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "./test_helpers/metrics_harness";
+
+describe("materialization_metrics", () => {
+   let harness: MetricsHarness;
+
+   beforeEach(async () => {
+      harness = await startMetricsHarness();
+      // Drop cached instruments so they re-bind to this test's provider.
+      resetMaterializationTelemetryForTesting();
+   });
+
+   afterEach(async () => {
+      resetMaterializationTelemetryForTesting();
+      await harness.shutdown();
+   });
+
+   it("counts runs labeled by mode and outcome", async () => {
+      recordMaterializationRun("auto", "success", 10);
+      recordMaterializationRun("auto", "success", 20);
+      recordMaterializationRun("orchestrated", "failed", 5);
+
+      expect(
+         await harness.collectCounter("publisher_materialization_runs_total", {
+            mode: "auto",
+            outcome: "success",
+         }),
+      ).toBe(2);
+      expect(
+         await harness.collectCounter("publisher_materialization_runs_total", {
+            mode: "orchestrated",
+            outcome: "failed",
+         }),
+      ).toBe(1);
+   });
+
+   it("adds the source count labeled by outcome, ignoring non-positive counts", async () => {
+      recordSourcesOutcome("built", 3);
+      recordSourcesOutcome("reused", 2);
+      recordSourcesOutcome("built", 0); // guarded: no emission
+
+      expect(
+         await harness.collectCounter(
+            "publisher_materialization_sources_total",
+            { outcome: "built" },
+         ),
+      ).toBe(3);
+      expect(
+         await harness.collectCounter(
+            "publisher_materialization_sources_total",
+            { outcome: "reused" },
+         ),
+      ).toBe(2);
+   });
+
+   it("counts drop-table outcomes and connection-digest skips", async () => {
+      recordDropTables("success");
+      recordDropTables("failure");
+      recordConnectionDigestSkipped();
+
+      expect(
+         await harness.collectCounter(
+            "publisher_materialization_drop_tables_total",
+            { outcome: "failure" },
+         ),
+      ).toBe(1);
+      expect(
+         await harness.collectCounter(
+            "publisher_materialization_connection_digest_skipped_total",
+         ),
+      ).toBe(1);
+   });
+});

package/src/materialization_metrics.ts

@@ -7,8 +7,8 @@
  * the run-duration histogram carries `mode` so a dashboard can render auto-run
  * vs orchestrated build latency side by side.
  *
- * Lazy init for the same reason as {@link ./query_cap_metrics}: instruments
- * created before `setGlobalMeterProvider` bind to a NoOp meter
+ * Instruments are created lazily for the same reason as {@link ./query_cap_metrics}:
+ * one created before `setGlobalMeterProvider` binds to a NoOp meter
  * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
  */
 
@@ -19,49 +19,83 @@ export type MaterializationOutcome = "success" | "failed" | "cancelled";
 /** Manifest bind outcome: timeout is split out from generic failure on purpose. */
 export type ManifestBindOutcome = "success" | "failure" | "timeout";
 
-let runCounter: Counter | null = null;
-let runDuration: Histogram | null = null;
-let sourcesCounter: Counter | null = null;
-let buildPlanComputeDuration: Histogram | null = null;
-let autoLoadCounter: Counter | null = null;
-let connectionDigestSkipCounter: Counter | null = null;
-let manifestBindCounter: Counter | null = null;
-let sourceBuildDuration: Histogram | null = null;
-let dropTablesCounter: Counter | null = null;
+const resetHooks: (() => void)[] = [];
 
-function ensureTelemetry(): { counter: Counter; duration: Histogram } {
-   if (runCounter && runDuration) {
-      return { counter: runCounter, duration: runDuration };
-   }
-   const meter = metrics.getMeter("publisher");
-   if (!runCounter) {
-      runCounter = meter.createCounter("publisher_materialization_runs_total", {
-         description:
-            "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').",
-      });
-   }
-   if (!runDuration) {
-      runDuration = meter.createHistogram(
-         "publisher_materialization_run_duration_ms",
-         {
-            description:
-               "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
-            unit: "ms",
-         },
-      );
-   }
-   return { counter: runCounter, duration: runDuration };
+/**
+ * A lazily-created instrument: built from the global meter on first use (so it
+ * binds to the real MeterProvider, not the NoOp default) and memoized. Each
+ * registers a reset hook so a test can swap in a fresh MeterProvider.
+ */
+function lazyCounter(name: string, description: string): () => Counter {
+   let instrument: Counter | null = null;
+   resetHooks.push(() => (instrument = null));
+   return () =>
+      (instrument ??= metrics
+         .getMeter("publisher")
+         .createCounter(name, { description }));
 }
 
+function lazyHistogram(
+   name: string,
+   description: string,
+   unit: string,
+): () => Histogram {
+   let instrument: Histogram | null = null;
+   resetHooks.push(() => (instrument = null));
+   return () =>
+      (instrument ??= metrics
+         .getMeter("publisher")
+         .createHistogram(name, { description, unit }));
+}
+
+const runCounter = lazyCounter(
+   "publisher_materialization_runs_total",
+   "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').",
+);
+const runDuration = lazyHistogram(
+   "publisher_materialization_run_duration_ms",
+   "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
+   "ms",
+);
+const sourcesCounter = lazyCounter(
+   "publisher_materialization_sources_total",
+   "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').",
+);
+const buildPlanComputeDuration = lazyHistogram(
+   "publisher_materialization_build_plan_compute_duration_ms",
+   "Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
+   "ms",
+);
+const autoLoadCounter = lazyCounter(
+   "publisher_materialization_auto_load_total",
+   "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').",
+);
+const connectionDigestSkipCounter = lazyCounter(
+   "publisher_materialization_connection_digest_skipped_total",
+   "Connection digests skipped during build-plan compile because the connection did not resolve.",
+);
+const manifestBindCounter = lazyCounter(
+   "publisher_materialization_manifest_bind_total",
+   "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
+);
+const sourceBuildDuration = lazyHistogram(
+   "publisher_materialization_source_build_duration_ms",
+   "Wall-clock duration of building a single persist source.",
+   "ms",
+);
+const dropTablesCounter = lazyCounter(
+   "publisher_materialization_drop_tables_total",
+   "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
+);
+
 /** Record the outcome and duration of a materialization build. */
 export function recordMaterializationRun(
    mode: MaterializationMode,
    outcome: MaterializationOutcome,
    durationMs: number,
 ): void {
-   const { counter, duration } = ensureTelemetry();
-   counter.add(1, { mode, outcome });
-   duration.record(durationMs, { mode });
+   runCounter().add(1, { mode, outcome });
+   runDuration().record(durationMs, { mode });
 }
 
 /**
@@ -74,15 +108,7 @@ export function recordSourcesOutcome(
    count: number,
 ): void {
    if (count <= 0) return;
-   if (!sourcesCounter) {
-      sourcesCounter = metrics
-         .getMeter("publisher")
-         .createCounter("publisher_materialization_sources_total", {
-            description:
-               "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').",
-         });
-   }
-   sourcesCounter.add(count, { outcome });
+   sourcesCounter().add(count, { outcome });
 }
 
 /**
@@ -91,19 +117,7 @@ export function recordSourcesOutcome(
  * tracking as a discrete cost separate from the build itself.
  */
 export function recordBuildPlanComputeDuration(durationMs: number): void {
-   if (!buildPlanComputeDuration) {
-      buildPlanComputeDuration = metrics
-         .getMeter("publisher")
-         .createHistogram(
-            "publisher_materialization_build_plan_compute_duration_ms",
-            {
-               description:
-                  "Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
-               unit: "ms",
-            },
-         );
-   }
-   buildPlanComputeDuration.record(durationMs);
+   buildPlanComputeDuration().record(durationMs);
 }
 
 /**
@@ -112,15 +126,7 @@ export function recordBuildPlanComputeDuration(durationMs: number): void {
  * the run is MANIFEST_FILE_READY but queries still resolve to the old tables.
  */
 export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
-   if (!autoLoadCounter) {
-      autoLoadCounter = metrics
-         .getMeter("publisher")
-         .createCounter("publisher_materialization_auto_load_total", {
-            description:
-               "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').",
-         });
-   }
-   autoLoadCounter.add(1, { outcome });
+   autoLoadCounter().add(1, { outcome });
 }
 
 /**
@@ -129,18 +135,7 @@ export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
  * are computed without a digest, so this is a correctness signal, not just noise.
  */
 export function recordConnectionDigestSkipped(): void {
-   if (!connectionDigestSkipCounter) {
-      connectionDigestSkipCounter = metrics
-         .getMeter("publisher")
-         .createCounter(
-            "publisher_materialization_connection_digest_skipped_total",
-            {
-               description:
-                  "Connection digests skipped during build-plan compile because the connection did not resolve.",
-            },
-         );
-   }
-   connectionDigestSkipCounter.add(1);
+   connectionDigestSkipCounter().add(1);
 }
 
 /**
@@ -149,56 +144,20 @@ export function recordConnectionDigestSkipped(): void {
  * tell an unreachable/slow manifest store from a malformed manifest.
  */
 export function recordManifestBind(outcome: ManifestBindOutcome): void {
-   if (!manifestBindCounter) {
-      manifestBindCounter = metrics
-         .getMeter("publisher")
-         .createCounter("publisher_materialization_manifest_bind_total", {
-            description:
-               "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
-         });
-   }
-   manifestBindCounter.add(1, { outcome });
+   manifestBindCounter().add(1, { outcome });
 }
 
 /** Record the wall-clock duration of building one persist source's table. */
 export function recordSourceBuildDuration(durationMs: number): void {
-   if (!sourceBuildDuration) {
-      sourceBuildDuration = metrics
-         .getMeter("publisher")
-         .createHistogram(
-            "publisher_materialization_source_build_duration_ms",
-            {
-               description:
-                  "Wall-clock duration of building a single persist source.",
-               unit: "ms",
-            },
-         );
-   }
-   sourceBuildDuration.record(durationMs);
+   sourceBuildDuration().record(durationMs);
 }
 
 /** Record a best-effort physical-table drop on materialization delete. */
 export function recordDropTables(outcome: "success" | "failure"): void {
-   if (!dropTablesCounter) {
-      dropTablesCounter = metrics
-         .getMeter("publisher")
-         .createCounter("publisher_materialization_drop_tables_total", {
-            description:
-               "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
-         });
-   }
-   dropTablesCounter.add(1, { outcome });
+   dropTablesCounter().add(1, { outcome });
 }
 
 /** Visible for tests. Drops cached instruments so a fresh MeterProvider can capture emissions. */
 export function resetMaterializationTelemetryForTesting(): void {
-   runCounter = null;
-   runDuration = null;
-   sourcesCounter = null;
-   buildPlanComputeDuration = null;
-   autoLoadCounter = null;
-   connectionDigestSkipCounter = null;
-   manifestBindCounter = null;
-   sourceBuildDuration = null;
-   dropTablesCounter = null;
+   for (const reset of resetHooks) reset();
 }

package/src/service/build_plan.spec.ts

@@ -1,14 +1,19 @@
 import type { PersistSource } from "@malloydata/malloy";
 import { describe, expect, it } from "bun:test";
 import * as sinon from "sinon";
+import type { BuildGraph as MalloyBuildGraph } from "@malloydata/malloy";
 import {
+   compilePackageBuildPlan,
    computeBuildId,
    computePackageBuildPlan,
+   deriveAnnotationFields,
    deriveBuildPlan,
    flattenDependsOn,
+   iterGraphSources,
    resolvePackageConnections,
 } from "./build_plan";
 import { fakeSource } from "./materialization_test_fixtures";
+import { Model } from "./model";
 
 describe("flattenDependsOn", () => {
    it("maps nested dependsOn entries to a flat sourceID list", () => {
@@ -20,6 +25,62 @@ describe("flattenDependsOn", () => {
    });
 });
 
+describe("iterGraphSources", () => {
+   it("yields resolvable sources in dependency order, skipping missing ones", () => {
+      const a = fakeSource({ name: "a", buildId: "ba" });
+      const b = fakeSource({ name: "b", buildId: "bb" });
+      const graph = {
+         connectionName: "duckdb",
+         nodes: [
+            [{ sourceID: "a@m", dependsOn: [] }],
+            [
+               { sourceID: "missing@m", dependsOn: [] },
+               { sourceID: "b@m", dependsOn: [] },
+            ],
+         ],
+      } as unknown as MalloyBuildGraph;
+
+      const names = [...iterGraphSources(graph, { "a@m": a, "b@m": b })].map(
+         (s) => s.name,
+      );
+      expect(names).toEqual(["a", "b"]);
+   });
+});
+
+describe("deriveAnnotationFields", () => {
+   it("returns all key=value fields of the #@ persist annotation", () => {
+      const source = {
+         annotations: {
+            parseAsTag: () => ({
+               tag: {
+                  *entries() {
+                     yield ["name", { text: () => "engaged_events" }];
+                     yield ["realization", { text: () => "COPY" }];
+                  },
+               },
+            }),
+         },
+      } as unknown as PersistSource;
+
+      expect(deriveAnnotationFields(source)).toEqual({
+         name: "engaged_events",
+         realization: "COPY",
+      });
+   });
+
+   it("degrades to {} when the annotation is absent or unparseable", () => {
+      const source = {
+         annotations: {
+            parseAsTag: () => {
+               throw new Error("no @ annotation");
+            },
+         },
+      } as unknown as PersistSource;
+
+      expect(deriveAnnotationFields(source)).toEqual({});
+   });
+});
+
 describe("computeBuildId", () => {
    it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
       const makeBuildId = sinon.stub().returns("computed-id");
@@ -102,6 +163,29 @@ describe("deriveBuildPlan", () => {
    });
 });
 
+describe("compilePackageBuildPlan", () => {
+   it("skips .malloynb notebooks without compiling them", async () => {
+      // A notebook would throw on its `>>>` cell delimiter if compiled as a
+      // flat model, aborting the whole package plan; it must be skipped.
+      const getModelRuntime = sinon.stub(Model, "getModelRuntime");
+      try {
+         const pkg = {
+            getModelPaths: () => ["notes.malloynb"],
+            getPackagePath: () => "/test",
+            getMalloyConfig: () => ({}),
+            getMalloyConnection: async () => ({}),
+         } as unknown as Parameters<typeof compilePackageBuildPlan>[0];
+
+         const compiled = await compilePackageBuildPlan(pkg);
+
+         expect(compiled.graphs).toEqual([]);
+         expect(getModelRuntime.called).toBe(false);
+      } finally {
+         getModelRuntime.restore();
+      }
+   });
+});
+
 describe("computePackageBuildPlan", () => {
    it("returns null when the package declares no persist sources", async () => {
       const pkg = {

package/src/service/build_plan.ts

@@ -6,8 +6,10 @@ import type {
    PersistSource,
 } from "@malloydata/malloy";
 import { components } from "../api";
+import { MODEL_FILE_SUFFIX } from "../constants";
 import { logger } from "../logger";
 import { recordConnectionDigestSkipped } from "../materialization_metrics";
+import { errMessage } from "../utils";
 import { Model } from "./model";
 
 type WireBuildGraph = components["schemas"]["BuildGraph"];
@@ -53,12 +55,37 @@ export function deriveColumns(persistSource: PersistSource): WireColumn[] {
    } catch (err) {
       logger.warn("Failed to derive columns for persist source", {
          sourceID: persistSource.sourceID,
-         error: err instanceof Error ? err.message : String(err),
+         error: errMessage(err),
       });
       return [];
    }
 }
 
+/**
+ * All key=value fields of a source's `#@ persist` annotation (e.g.
+ * `{ name: "engaged_events", realization: "COPY" }`), degrading to {} if the
+ * annotation is absent or unparseable. The control plane consumes these — most
+ * importantly `name`, which it uses as the materialized table name (and which
+ * may carry a dialect-style container path like `dataset.table`). Returning the
+ * full set rather than a fixed subset means new persist directives flow through
+ * without a publisher change.
+ */
+export function deriveAnnotationFields(
+   persistSource: PersistSource,
+): Record<string, string> {
+   const out: Record<string, string> = {};
+   try {
+      const tag = persistSource.annotations.parseAsTag("@").tag;
+      for (const [key, value] of tag.entries()) {
+         const text = value.text();
+         if (text !== undefined) out[key] = text;
+      }
+   } catch {
+      // Degrade to {} — mirrors deriveColumns / selfAssignTableName.
+   }
+   return out;
+}
+
 /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
 export function flattenDependsOn(node: {
    dependsOn: { sourceID: string }[];
@@ -66,6 +93,24 @@ export function flattenDependsOn(node: {
    return node.dependsOn.map((d) => d.sourceID);
 }
 
+/**
+ * Yield each dependency-ordered persist source of one graph, resolving the
+ * node's sourceID against the sources map and skipping nodes whose source is
+ * absent. Centralizes the graph→levels→nodes→source walk shared by the
+ * planning and build loops.
+ */
+export function* iterGraphSources(
+   graph: MalloyBuildGraph,
+   sources: Record<string, PersistSource>,
+): Iterable<PersistSource> {
+   for (const level of graph.nodes) {
+      for (const node of level) {
+         const source = sources[node.sourceID];
+         if (source) yield source;
+      }
+   }
+}
+
 /**
  * The buildId for a persist source: a stable digest of its connection identity
  * and canonical SQL. Centralizes the (source, connectionDigests) call shape so
@@ -100,7 +145,7 @@ export async function resolvePackageConnections(
          map.set(name, await pkg.getMalloyConnection(name));
       } catch (err) {
          logger.warn(`Failed to resolve connection ${name}`, {
-            error: err instanceof Error ? err.message : String(err),
+            error: errMessage(err),
          });
       }
    }
@@ -121,6 +166,11 @@ export async function compilePackageBuildPlan(
    const allSources: Record<string, PersistSource> = {};
 
    for (const modelPath of pkg.getModelPaths()) {
+      // Only `.malloy` models declare persist sources. Skip `.malloynb`
+      // notebooks: getModel() parses a model file as a flat model and throws on
+      // the notebook's `>>>` cell delimiter, which would abort the entire
+      // package build plan and silently drop every persist source in it.
+      if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) continue;
       if (signal?.aborted) throw new Error("Build cancelled");
 
       const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(
@@ -211,6 +261,7 @@ export function deriveBuildPlan(
          buildId: computeBuildId(source, connectionDigests),
          sql: source.getSQL(),
          columns: deriveColumns(source),
+         annotationFields: deriveAnnotationFields(source),
       };
    }