@malloy-publisher/server 0.0.208 → 0.0.209

package/src/service/materialization_test_fixtures.ts

@@ -0,0 +1,119 @@
+import type { PersistSource } from "@malloydata/malloy";
+import {
+   BuildInstruction,
+   BuildPlan,
+   Materialization,
+} from "../storage/DatabaseInterface";
+import { CompiledBuildPlan } from "./build_plan";
+
+/**
+ * Shared fixtures for the materialization unit specs (service, controller,
+ * build-plan). Centralizing the builders keeps the per-source stand-ins and
+ * record shapes consistent across files and avoids drift when the wire types
+ * change.
+ */
+
+/** A persisted materialization record with sensible PENDING defaults. */
+export function makeMaterialization(
+   overrides: Partial<Materialization> = {},
+): Materialization {
+   return {
+      id: "mat-1",
+      environmentId: "env-1",
+      packageName: "pkg",
+      status: "PENDING",
+      manifest: null,
+      startedAt: null,
+      completedAt: null,
+      error: null,
+      metadata: null,
+      createdAt: new Date("2026-01-01"),
+      updatedAt: new Date("2026-01-01"),
+      ...overrides,
+   };
+}
+
+/** A single-source wire BuildPlan (the `Package.buildPlan` artifact). */
+export function makeBuildPlan(overrides: Partial<BuildPlan> = {}): BuildPlan {
+   return {
+      graphs: [
+         {
+            connectionName: "duckdb",
+            nodes: [[{ sourceID: "orders@m.malloy", dependsOn: [] }]],
+         },
+      ],
+      sources: {
+         "orders@m.malloy": {
+            name: "orders",
+            sourceID: "orders@m.malloy",
+            connectionName: "duckdb",
+            dialect: "duckdb",
+            buildId: "build-orders",
+            sql: "SELECT 1",
+            columns: [],
+         },
+      },
+      ...overrides,
+   };
+}
+
+/** A caller-supplied build instruction matching {@link makeBuildPlan}. */
+export function makeInstruction(
+   overrides: Partial<BuildInstruction> = {},
+): BuildInstruction {
+   return {
+      buildId: "build-orders",
+      materializedTableId: "mt-1",
+      physicalTableName: '"orders_v1"',
+      realization: "COPY",
+      ...overrides,
+   };
+}
+
+/**
+ * A minimal stand-in for a Malloy {@link PersistSource} exposing only what the
+ * build internals touch (name/id, deterministic buildId, SQL, and the
+ * `#@ persist name=` annotation reader, defaulted to "unset").
+ */
+export function fakeSource(opts: {
+   name: string;
+   buildId: string;
+   sql?: string;
+   connectionName?: string;
+}): PersistSource {
+   return {
+      name: opts.name,
+      sourceID: opts.name,
+      connectionName: opts.connectionName ?? "duckdb",
+      makeBuildId: () => opts.buildId,
+      getSQL: () => opts.sql ?? "SELECT 1",
+      annotations: {
+         parseAsTag: () => ({ tag: { text: () => undefined } }),
+      },
+   } as unknown as PersistSource;
+}
+
+/**
+ * Assemble a {@link CompiledBuildPlan} from a sources map and dependency
+ * levels (one connection, "duckdb"). `connections` is supplied by the caller
+ * because build vs. plan-derivation tests need different connection mocks.
+ */
+export function compiledWith(
+   sources: Record<string, PersistSource>,
+   levels: string[][],
+   connections: CompiledBuildPlan["connections"] = new Map(),
+): CompiledBuildPlan {
+   return {
+      graphs: [
+         {
+            connectionName: "duckdb",
+            nodes: levels.map((level) =>
+               level.map((sourceID) => ({ sourceID, dependsOn: [] })),
+            ),
+         },
+      ] as unknown as CompiledBuildPlan["graphs"],
+      sources,
+      connectionDigests: { duckdb: "dig" },
+      connections,
+   };
+}

package/src/service/package.ts

@@ -28,9 +28,11 @@ import {
    ServiceUnavailableError,
 } from "../errors";
 import { formatDuration, logger } from "../logger";
+import { recordBuildPlanComputeDuration } from "../materialization_metrics";
 import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
-import { BuildManifest } from "../storage/DatabaseInterface";
+import { BuildManifest, BuildPlan } from "../storage/DatabaseInterface";
 import { ignoreDotfiles } from "../utils";
+import { computePackageBuildPlan } from "./build_plan";
 import { Model } from "./model";
 
 type ApiDatabase = components["schemas"]["Database"];
@@ -67,6 +69,13 @@ export class Package {
       "unbound";
    private manifestEntryCount = 0;
    private boundManifestUri: string | null = null;
+   // The package's persist build plan: a deterministic property of the compiled
+   // package (per-source buildId, columns, build SQL, dependency graphs),
+   // computed once at load from the live (unbound) models so it is stable for a
+   // given (package version, connection config). Null when the package declares
+   // no persist source. Surfaced read-only on getPackageMetadata() so a caller
+   // can derive build instructions without a separate plan round-trip.
+   private buildPlan: BuildPlan | null = null;
    private static meter = metrics.getMeter("publisher");
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -377,6 +386,26 @@ export class Package {
          malloyConfig,
       );
 
+      // Compute the persist build plan off the live (unbound) models, before the
+      // caller binds any configured manifest, so the surfaced plan reflects the
+      // canonical build (not the manifest-rewritten SQL). Best-effort: a plan
+      // failure is logged, not fatal — the package still serves; the plan is
+      // just absent. Recompiles the models (duplicate schema RPCs vs the worker
+      // compile); accepted for now.
+      try {
+         const buildPlanStart = Date.now();
+         pkg.buildPlan = await computePackageBuildPlan(pkg);
+         recordBuildPlanComputeDuration(Date.now() - buildPlanStart);
+      } catch (err) {
+         logger.warn(
+            `Failed to compute build plan for package ${packageName}`,
+            {
+               packageName,
+               error: err instanceof Error ? err.message : String(err),
+            },
+         );
+      }
+
       // Fail-safe at load: a bad explores entry doesn't fail the package
       // (its models still load and listModels hides the unmatched entry — it
       // never falls back to listing everything). Warn so the misconfig is
@@ -397,6 +426,16 @@ export class Package {
       return this.packageName;
    }
 
+   /**
+    * The package's persist build plan (per-source buildId, columns, build SQL,
+    * dependency graphs), or null when the package declares no persist source.
+    * A deterministic property of the compiled package; callers derive build
+    * instructions from it for an orchestrated materialization.
+    */
+   public getBuildPlan(): BuildPlan | null {
+      return this.buildPlan;
+   }
+
    public getPackageMetadata(): ApiPackage {
       // Overlay the server-computed fields onto the stored metadata: the
       // explores misconfig warnings (loading is fail-safe — the package still
@@ -418,6 +457,7 @@ export class Package {
          manifestBindingStatus: this.manifestBindingStatus,
          manifestEntryCount: this.manifestEntryCount,
          boundManifestUri: this.boundManifestUri,
+         buildPlan: this.buildPlan,
       };
       const warnings = this.exploreWarnings();
       if (warnings.length > 0) {

package/src/storage/DatabaseInterface.ts

@@ -108,8 +108,8 @@ export interface Connection {
    updatedAt: Date;
 }
 
-// Wire types for the two-round build protocol, kept in sync with the OpenAPI
-// spec via the generated `api.ts`.
+// Wire types for the build protocol, kept in sync with the OpenAPI spec via
+// the generated `api.ts`.
 export type MaterializationStatus =
    components["schemas"]["MaterializationStatus"];
 export type BuildPlan = components["schemas"]["BuildPlan"];
@@ -122,15 +122,8 @@ export interface Materialization {
    id: string;
    environmentId: string;
    packageName: string;
-   /**
-    * Echoes the create-time flag (read from metadata). False (default) =
-    * auto-run all phases; true = pause at BUILD_PLAN_READY for Round 2.
-    */
-   pauseBetweenPhases: boolean;
    status: MaterializationStatus;
-   /** Round 1 output. Null until status >= BUILD_PLAN_READY. */
-   buildPlan: BuildPlan | null;
-   /** Round 2 output. Null until status = MANIFEST_FILE_READY. */
+   /** Build output. Null until status = MANIFEST_FILE_READY. */
    manifest: BuildManifestResult | null;
    startedAt: Date | null;
    completedAt: Date | null;
@@ -142,7 +135,6 @@ export interface Materialization {
 
 export interface MaterializationUpdate {
    status?: MaterializationStatus;
-   buildPlan?: BuildPlan | null;
    manifest?: BuildManifestResult | null;
    startedAt?: Date;
    completedAt?: Date;
@@ -154,8 +146,8 @@ export interface MaterializationUpdate {
  * Malloy-facing build manifest: maps a buildId to the physical table backing
  * that persist source. This is the shape the Malloy runtime consumes when
  * (re)loading models so persist references resolve to materialized tables.
- * Distinct from {@link BuildManifestResult} (the wire/Round 2 output, which
- * also carries CP bookkeeping like materializedTableId and rowCount).
+ * Distinct from {@link BuildManifestResult} (the wire build output, which also
+ * carries caller bookkeeping like materializedTableId and rowCount).
  */
 export interface BuildManifestEntry {
    tableName: string;

package/src/storage/duckdb/MaterializationRepository.ts

@@ -1,6 +1,5 @@
 import {
    BuildManifestResult,
-   BuildPlan,
    Materialization,
    MaterializationStatus,
    MaterializationUpdate,
@@ -16,7 +15,6 @@ const TERMINAL_STATUSES: ReadonlySet<MaterializationStatus> = new Set([
 /** Non-terminal statuses count as "active" for the single-active guard. */
 const ACTIVE_STATUSES: readonly MaterializationStatus[] = [
    "PENDING",
-   "BUILD_PLAN_READY",
    "MANIFEST_ROWS_READY",
 ];
 
@@ -40,8 +38,9 @@ export class DuplicateActiveMaterializationError extends Error {
 /**
  * DuckDB-backed repository for package materializations.
  *
- * A Materialization tracks a single build run for an (environment, package) pair
- * through its lifecycle: PENDING -> RUNNING -> SUCCESS | FAILED | CANCELLED.
+ * A Materialization tracks a single build run for an (environment, package)
+ * pair through its lifecycle: PENDING -> MANIFEST_ROWS_READY ->
+ * MANIFEST_FILE_READY, or -> FAILED | CANCELLED.
  */
 export class MaterializationRepository {
    constructor(private db: DuckDBConnection) {}
@@ -114,8 +113,8 @@ export class MaterializationRepository {
 
       try {
          const rows = await this.db.all<Record<string, unknown>>(
-            `INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
-            VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
+            `INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, manifest, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)
             RETURNING *`,
             [
                id,
@@ -181,12 +180,6 @@ export class MaterializationRepository {
             updates.metadata ? JSON.stringify(updates.metadata) : null,
          );
       }
-      if (updates.buildPlan !== undefined) {
-         setClauses.push(`build_plan = ?`);
-         params.push(
-            updates.buildPlan ? JSON.stringify(updates.buildPlan) : null,
-         );
-      }
       if (updates.manifest !== undefined) {
          setClauses.push(`manifest = ?`);
          params.push(
@@ -237,9 +230,7 @@ export class MaterializationRepository {
          id: row.id as string,
          environmentId: row.environment_id as string,
          packageName: row.package_name as string,
-         pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
          status: row.status as MaterializationStatus,
-         buildPlan: parseJsonColumn<BuildPlan>(row.build_plan),
          manifest: parseJsonColumn<BuildManifestResult>(row.manifest),
          metadata,
          startedAt: row.started_at ? new Date(row.started_at as string) : null,

package/src/storage/duckdb/schema.ts

@@ -75,15 +75,15 @@ export async function initializeSchema(
 
    // Materializations table.
    //
-   // `active_key` enforces at-most-one active (PENDING or RUNNING)
-   // materialization per (environment, package) at the DB layer. It is set to
+   // `active_key` enforces at-most-one active (non-terminal) materialization
+   // per (environment, package) at the DB layer. It is set to
    // `{environment_id}|{package_name}` while the row is active and cleared
    // to NULL on transition to any terminal state. A unique index on
    // `active_key` (see below) makes the insert-then-check race impossible —
    // a second concurrent create fails with a constraint violation, which the
    // service layer translates to `MaterializationConflictError`.
-   // `build_plan` (Round 1) and `manifest` (Round 2) are JSON blobs holding
-   // the two-round protocol payloads returned inline on the resource.
+   // `manifest` is a JSON blob holding the build output returned inline on the
+   // resource.
    await db.run(`
     CREATE TABLE IF NOT EXISTS materializations (
       id VARCHAR PRIMARY KEY,
@@ -95,7 +95,6 @@ export async function initializeSchema(
       completed_at TIMESTAMP,
       error TEXT,
       metadata JSON,
-      build_plan JSON,
       manifest JSON,
       created_at TIMESTAMP NOT NULL,
       updated_at TIMESTAMP NOT NULL,