@malloy-publisher/server 0.0.208 → 0.0.210

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.208",
+   "version": "0.0.210",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -41,6 +41,7 @@
       "@malloydata/db-duckdb": "^0.0.415",
       "@malloydata/db-mysql": "^0.0.415",
       "@malloydata/db-postgres": "^0.0.415",
+      "@malloydata/db-publisher": "^0.0.415",
       "@malloydata/db-snowflake": "^0.0.415",
       "@malloydata/db-trino": "^0.0.415",
       "@malloydata/malloy": "^0.0.415",

package/src/controller/materialization.controller.spec.ts

@@ -0,0 +1,125 @@
+import { describe, expect, it } from "bun:test";
+import * as sinon from "sinon";
+import { BadRequestError } from "../errors";
+import type { MaterializationService } from "../service/materialization_service";
+import { MaterializationController } from "./materialization.controller";
+
+/**
+ * Unit tests for {@link MaterializationController.createMaterialization}'s body
+ * validation. The service is stubbed so each test asserts the parsed options
+ * the controller forwards (or the rejection for a malformed body).
+ */
+function build() {
+   const createMaterialization = sinon.stub().resolves({ id: "m1" });
+   const service = {
+      createMaterialization,
+   } as unknown as MaterializationService;
+   const controller = new MaterializationController(service);
+   return { controller, createMaterialization };
+}
+
+/** Parsed options the controller forwards for a given request body. */
+async function parse(body: Record<string, unknown>) {
+   const { controller, createMaterialization } = build();
+   await controller.createMaterialization("env", "pkg", body);
+   return createMaterialization.firstCall.args[2];
+}
+
+describe("MaterializationController.createMaterialization validation", () => {
+   it("forwards an empty body as empty options", async () => {
+      expect(await parse({})).toEqual({});
+   });
+
+   it("passes through forceRefresh and sourceNames", async () => {
+      expect(
+         await parse({ forceRefresh: true, sourceNames: ["a", "b"] }),
+      ).toEqual({ forceRefresh: true, sourceNames: ["a", "b"] });
+   });
+
+   it("rejects a non-boolean forceRefresh", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            forceRefresh: "yes",
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+
+   it("rejects sourceNames that is not an array of strings", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            sourceNames: [1, 2],
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+
+   it("flattens buildInstructions.sources into the instruction list", async () => {
+      const parsed = await parse({
+         buildInstructions: {
+            sources: [
+               {
+                  buildId: "b1",
+                  sourceID: "orders@m",
+                  materializedTableId: "mt-1",
+                  physicalTableName: "orders_v1",
+                  realization: "COPY",
+               },
+            ],
+         },
+      });
+      expect(parsed).toEqual({
+         buildInstructions: [
+            {
+               buildId: "b1",
+               sourceID: "orders@m",
+               materializedTableId: "mt-1",
+               physicalTableName: "orders_v1",
+               realization: "COPY",
+            },
+         ],
+      });
+   });
+
+   it("treats null buildInstructions as absent (auto-run)", async () => {
+      expect(await parse({ buildInstructions: null })).toEqual({});
+   });
+
+   it("rejects buildInstructions without a non-empty sources array", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            buildInstructions: { sources: [] },
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+
+   it("rejects an instruction missing a required field", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            buildInstructions: {
+               sources: [{ buildId: "b1", materializedTableId: "mt-1" }],
+            },
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+
+   it("rejects an unrecognized realization", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            buildInstructions: {
+               sources: [
+                  {
+                     buildId: "b1",
+                     materializedTableId: "mt-1",
+                     physicalTableName: "orders_v1",
+                     realization: "MERGE",
+                  },
+               ],
+            },
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+});

package/src/controller/materialization.controller.ts

@@ -20,18 +20,20 @@ export class MaterializationController {
    private validateCreateBody(body: Record<string, unknown>): {
       forceRefresh?: boolean;
       sourceNames?: string[];
-      pauseBetweenPhases?: boolean;
+      buildInstructions?: BuildInstruction[];
    } {
       const result: {
          forceRefresh?: boolean;
          sourceNames?: string[];
-         pauseBetweenPhases?: boolean;
+         buildInstructions?: BuildInstruction[];
       } = {};
-      if (body.pauseBetweenPhases !== undefined) {
-         if (typeof body.pauseBetweenPhases !== "boolean") {
-            throw new BadRequestError("pauseBetweenPhases must be a boolean");
-         }
-         result.pauseBetweenPhases = body.pauseBetweenPhases;
+      if (
+         body.buildInstructions !== undefined &&
+         body.buildInstructions !== null
+      ) {
+         result.buildInstructions = this.validateBuildInstructions(
+            body.buildInstructions,
+         );
       }
       if (body.forceRefresh !== undefined) {
          if (typeof body.forceRefresh !== "boolean") {
@@ -53,30 +55,24 @@ export class MaterializationController {
       return result;
    }
 
-   async buildMaterialization(
-      environmentName: string,
-      packageName: string,
-      materializationId: string,
-      body: Record<string, unknown>,
-   ) {
-      return this.materializationService.buildMaterialization(
-         environmentName,
-         packageName,
-         materializationId,
-         this.validateBuildBody(body),
-      );
-   }
-
-   private validateBuildBody(
-      body: Record<string, unknown>,
-   ): BuildInstruction[] {
-      const sources = body.sources;
+   /**
+    * Validate the orchestrated `buildInstructions` payload (BuildInstructions:
+    * `{ sources: BuildInstruction[] }`) and flatten it to the instruction list
+    * the service consumes.
+    */
+   private validateBuildInstructions(raw: unknown): BuildInstruction[] {
+      if (typeof raw !== "object" || raw === null) {
+         throw new BadRequestError("buildInstructions must be an object");
+      }
+      const sources = (raw as Record<string, unknown>).sources;
       if (!Array.isArray(sources) || sources.length === 0) {
          throw new BadRequestError(
-            "build requires a non-empty 'sources' array of BuildInstruction",
+            "buildInstructions requires a non-empty 'sources' array of BuildInstruction",
          );
       }
-      return sources.map((raw) => this.validateInstruction(raw));
+      return sources.map((instruction) =>
+         this.validateInstruction(instruction),
+      );
    }
 
    private validateInstruction(raw: unknown): BuildInstruction {

package/src/materialization_metrics.ts

@@ -1,12 +1,11 @@
 /**
- * Centralized telemetry for the two-round materialization protocol.
+ * Centralized telemetry for materialization builds.
  *
- * Operators need to answer "are builds completing, how long do the two
- * rounds take, and where are they failing?" without grepping logs. The
- * counter carries `round` (`round1`|`round2`) and `outcome`
- * (`success`|`failed`|`cancelled`); the histogram carries `round` so a
- * dashboard can render Round 1 (compile+plan) vs Round 2 (build) latency
- * side by side.
+ * Operators need to answer "are builds completing, how long do they take, and
+ * where are they failing?" without grepping logs. The run counter carries
+ * `mode` (`auto`|`orchestrated`) and `outcome` (`success`|`failed`|`cancelled`);
+ * the run-duration histogram carries `mode` so a dashboard can render auto-run
+ * vs orchestrated build latency side by side.
  *
  * Lazy init for the same reason as {@link ./query_cap_metrics}: instruments
  * created before `setGlobalMeterProvider` bind to a NoOp meter
@@ -15,58 +14,138 @@
 
 import { metrics, type Counter, type Histogram } from "@opentelemetry/api";
 
-export type MaterializationRound = "round1" | "round2" | "auto";
+export type MaterializationMode = "auto" | "orchestrated";
 export type MaterializationOutcome = "success" | "failed" | "cancelled";
 /** Manifest bind outcome: timeout is split out from generic failure on purpose. */
 export type ManifestBindOutcome = "success" | "failure" | "timeout";
 
-let roundCounter: Counter | null = null;
-let roundDuration: Histogram | null = null;
+let runCounter: Counter | null = null;
+let runDuration: Histogram | null = null;
+let sourcesCounter: Counter | null = null;
+let buildPlanComputeDuration: Histogram | null = null;
+let autoLoadCounter: Counter | null = null;
+let connectionDigestSkipCounter: Counter | null = null;
 let manifestBindCounter: Counter | null = null;
 let sourceBuildDuration: Histogram | null = null;
 let dropTablesCounter: Counter | null = null;
 
 function ensureTelemetry(): { counter: Counter; duration: Histogram } {
-   if (roundCounter && roundDuration) {
-      return { counter: roundCounter, duration: roundDuration };
+   if (runCounter && runDuration) {
+      return { counter: runCounter, duration: runDuration };
    }
    const meter = metrics.getMeter("publisher");
-   if (!roundCounter) {
-      roundCounter = meter.createCounter(
-         "publisher_materialization_rounds_total",
-         {
-            description:
-               "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled').",
-         },
-      );
+   if (!runCounter) {
+      runCounter = meter.createCounter("publisher_materialization_runs_total", {
+         description:
+            "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').",
+      });
    }
-   if (!roundDuration) {
-      roundDuration = meter.createHistogram(
-         "publisher_materialization_round_duration_ms",
+   if (!runDuration) {
+      runDuration = meter.createHistogram(
+         "publisher_materialization_run_duration_ms",
          {
             description:
-               "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
+               "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
             unit: "ms",
          },
       );
    }
-   return { counter: roundCounter, duration: roundDuration };
+   return { counter: runCounter, duration: runDuration };
 }
 
-/** Record the outcome and duration of a materialization round. */
-export function recordMaterializationRound(
-   round: MaterializationRound,
+/** Record the outcome and duration of a materialization build. */
+export function recordMaterializationRun(
+   mode: MaterializationMode,
    outcome: MaterializationOutcome,
    durationMs: number,
 ): void {
    const { counter, duration } = ensureTelemetry();
-   counter.add(1, { round, outcome });
-   duration.record(durationMs, { round });
+   counter.add(1, { mode, outcome });
+   duration.record(durationMs, { mode });
+}
+
+/**
+ * Record how many persist sources a run built vs. reused (carried forward
+ * unchanged via skip-if-unchanged). Lets a dashboard show the reuse ratio,
+ * the main lever on materialization cost.
+ */
+export function recordSourcesOutcome(
+   outcome: "built" | "reused",
+   count: number,
+): void {
+   if (count <= 0) return;
+   if (!sourcesCounter) {
+      sourcesCounter = metrics
+         .getMeter("publisher")
+         .createCounter("publisher_materialization_sources_total", {
+            description:
+               "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').",
+         });
+   }
+   sourcesCounter.add(count, { outcome });
+}
+
+/**
+ * Record the wall-clock cost of compiling a package's build plan
+ * (`Package.buildPlan`). This recompiles models at load, so it is worth
+ * tracking as a discrete cost separate from the build itself.
+ */
+export function recordBuildPlanComputeDuration(durationMs: number): void {
+   if (!buildPlanComputeDuration) {
+      buildPlanComputeDuration = metrics
+         .getMeter("publisher")
+         .createHistogram(
+            "publisher_materialization_build_plan_compute_duration_ms",
+            {
+               description:
+                  "Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
+               unit: "ms",
+            },
+         );
+   }
+   buildPlanComputeDuration.record(durationMs);
+}
+
+/**
+ * Record an auto-run manifest auto-load attempt. Auto-load is best-effort (a
+ * failure does not fail the run), so a failure here is otherwise invisible:
+ * the run is MANIFEST_FILE_READY but queries still resolve to the old tables.
+ */
+export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
+   if (!autoLoadCounter) {
+      autoLoadCounter = metrics
+         .getMeter("publisher")
+         .createCounter("publisher_materialization_auto_load_total", {
+            description:
+               "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').",
+         });
+   }
+   autoLoadCounter.add(1, { outcome });
+}
+
+/**
+ * Record that a connection digest could not be computed during build-plan
+ * compilation because the connection failed to resolve. The resulting buildIds
+ * are computed without a digest, so this is a correctness signal, not just noise.
+ */
+export function recordConnectionDigestSkipped(): void {
+   if (!connectionDigestSkipCounter) {
+      connectionDigestSkipCounter = metrics
+         .getMeter("publisher")
+         .createCounter(
+            "publisher_materialization_connection_digest_skipped_total",
+            {
+               description:
+                  "Connection digests skipped during build-plan compile because the connection did not resolve.",
+            },
+         );
+   }
+   connectionDigestSkipCounter.add(1);
 }
 
 /**
- * Record a manifest-bind attempt (publisher binding a control-plane manifest to
- * a package at load). `timeout` is distinguished from `failure` so operators can
+ * Record a manifest-bind attempt (publisher binding a configured manifest to a
+ * package at load). `timeout` is distinguished from `failure` so operators can
  * tell an unreachable/slow manifest store from a malformed manifest.
  */
 export function recordManifestBind(outcome: ManifestBindOutcome): void {
@@ -113,8 +192,12 @@ export function recordDropTables(outcome: "success" | "failure"): void {
 
 /** Visible for tests. Drops cached instruments so a fresh MeterProvider can capture emissions. */
 export function resetMaterializationTelemetryForTesting(): void {
-   roundCounter = null;
-   roundDuration = null;
+   runCounter = null;
+   runDuration = null;
+   sourcesCounter = null;
+   buildPlanComputeDuration = null;
+   autoLoadCounter = null;
+   connectionDigestSkipCounter = null;
    manifestBindCounter = null;
    sourceBuildDuration = null;
    dropTablesCounter = null;

package/src/server-old.ts

@@ -944,16 +944,7 @@ export function registerLegacyRoutes(
       async (req, res) => {
          try {
             const action = req.query.action;
-            if (action === "build") {
-               const build =
-                  await materializationController.buildMaterialization(
-                     req.params.projectName,
-                     req.params.packageName,
-                     req.params.materializationId,
-                     req.body || {},
-                  );
-               res.status(202).json(remapMaterializationResponse(build));
-            } else if (action === "stop") {
+            if (action === "stop") {
                const build =
                   await materializationController.stopMaterialization(
                      req.params.projectName,
@@ -963,7 +954,7 @@ export function registerLegacyRoutes(
                res.status(200).json(remapMaterializationResponse(build));
             } else {
                throw new BadRequestError(
-                  `Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`,
+                  `Unsupported action '${String(action ?? "")}'. Expected 'stop'.`,
                );
             }
          } catch (error) {

package/src/server.ts

@@ -1592,15 +1592,7 @@ app.post(
    async (req, res) => {
       try {
          const action = req.query.action;
-         if (action === "build") {
-            const build = await materializationController.buildMaterialization(
-               req.params.environmentName,
-               req.params.packageName,
-               req.params.materializationId,
-               req.body || {},
-            );
-            res.status(202).json(build);
-         } else if (action === "stop") {
+         if (action === "stop") {
             const build = await materializationController.stopMaterialization(
                req.params.environmentName,
                req.params.packageName,
@@ -1609,7 +1601,7 @@ app.post(
             res.status(200).json(build);
          } else {
             throw new BadRequestError(
-               `Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`,
+               `Unsupported action '${String(action ?? "")}'. Expected 'stop'.`,
             );
          }
       } catch (error) {

package/src/service/build_plan.spec.ts

@@ -0,0 +1,116 @@
+import type { PersistSource } from "@malloydata/malloy";
+import { describe, expect, it } from "bun:test";
+import * as sinon from "sinon";
+import {
+   computeBuildId,
+   computePackageBuildPlan,
+   deriveBuildPlan,
+   flattenDependsOn,
+   resolvePackageConnections,
+} from "./build_plan";
+import { fakeSource } from "./materialization_test_fixtures";
+
+describe("flattenDependsOn", () => {
+   it("maps nested dependsOn entries to a flat sourceID list", () => {
+      expect(
+         flattenDependsOn({
+            dependsOn: [{ sourceID: "a" }, { sourceID: "b" }],
+         }),
+      ).toEqual(["a", "b"]);
+   });
+});
+
+describe("computeBuildId", () => {
+   it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
+      const makeBuildId = sinon.stub().returns("computed-id");
+      const source = {
+         connectionName: "duckdb",
+         makeBuildId,
+         getSQL: () => "SELECT 7",
+      } as unknown as PersistSource;
+
+      const id = computeBuildId(source, { duckdb: "dig-1" });
+
+      expect(id).toBe("computed-id");
+      expect(makeBuildId.calledOnceWithExactly("dig-1", "SELECT 7")).toBe(true);
+   });
+});
+
+describe("resolvePackageConnections", () => {
+   it("resolves each unique name once and omits failures", async () => {
+      const getMalloyConnection = sinon.stub();
+      getMalloyConnection.withArgs("ok").resolves({ id: "ok-conn" });
+      getMalloyConnection.withArgs("bad").rejects(new Error("nope"));
+
+      const map = await resolvePackageConnections({ getMalloyConnection }, [
+         "ok",
+         "ok",
+         "bad",
+      ]);
+
+      expect(map.has("ok")).toBe(true);
+      expect(map.has("bad")).toBe(false);
+      // "ok" requested twice but resolved once (dedupe).
+      expect(getMalloyConnection.withArgs("ok").callCount).toBe(1);
+   });
+});
+
+describe("deriveBuildPlan", () => {
+   it("projects graphs and sources into the wire build plan", () => {
+      const orders = fakeSource({
+         name: "orders",
+         buildId: "bid-orders",
+         sql: "SELECT 1",
+      });
+      const plan = deriveBuildPlan(
+         [
+            {
+               connectionName: "duckdb",
+               nodes: [[{ sourceID: "orders@m", dependsOn: [] }]],
+            },
+         ] as unknown as Parameters<typeof deriveBuildPlan>[0],
+         { "orders@m": orders },
+         { duckdb: "dig" },
+      );
+
+      expect(plan.graphs[0].connectionName).toBe("duckdb");
+      expect(plan.sources["orders@m"]).toMatchObject({
+         name: "orders",
+         connectionName: "duckdb",
+         buildId: "bid-orders",
+         sql: "SELECT 1",
+         columns: [],
+      });
+   });
+
+   it("honors the sourceNames filter", () => {
+      const a = fakeSource({ name: "a", buildId: "bid-a" });
+      const b = fakeSource({ name: "b", buildId: "bid-b" });
+      const plan = deriveBuildPlan(
+         [
+            {
+               connectionName: "duckdb",
+               nodes: [[{ sourceID: "a@m", dependsOn: [] }]],
+            },
+         ] as unknown as Parameters<typeof deriveBuildPlan>[0],
+         { "a@m": a, "b@m": b },
+         { duckdb: "dig" },
+         ["a"],
+      );
+
+      expect(Object.keys(plan.sources)).toEqual(["a@m"]);
+   });
+});
+
+describe("computePackageBuildPlan", () => {
+   it("returns null when the package declares no persist sources", async () => {
+      const pkg = {
+         getModelPaths: () => [],
+         getPackagePath: () => "/test",
+         getMalloyConfig: () => ({}),
+         getMalloyConnection: async () => ({}),
+      } as unknown as Parameters<typeof computePackageBuildPlan>[0];
+
+      expect(await computePackageBuildPlan(pkg)).toBeNull();
+   });
+});