@malloy-publisher/server 0.0.205 → 0.0.207

package/src/service/materialization_service.ts

@@ -1,10 +1,12 @@
 import type {
-   BuildGraph,
+   AtomicField,
+   BuildGraph as MalloyBuildGraph,
    MalloyConfig,
    Connection as MalloyConnection,
    PersistSource,
 } from "@malloydata/malloy";
 import { Manifest } from "@malloydata/malloy";
+import { components } from "../api";
 import {
    BadRequestError,
    InvalidStateTransitionError,
@@ -12,48 +14,115 @@ import {
    MaterializationNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
-import type { ManifestEntry } from "../storage/DatabaseInterface";
 import {
+   MaterializationRound,
+   recordDropTables,
+   recordMaterializationRound,
+   recordSourceBuildDuration,
+} from "../materialization_metrics";
+import {
+   BuildInstruction,
+   BuildManifest,
+   BuildManifestResult,
+   BuildPlan,
    Materialization,
    MaterializationStatus,
+   MaterializationUpdate,
+   ManifestEntry,
    ResourceRepository,
 } from "../storage/DatabaseInterface";
 import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
 import { EnvironmentStore } from "./environment_store";
-import { ManifestService } from "./manifest_service";
-import {
-   dropManifestEntries,
-   GcResult,
-   liveTableKey,
-} from "./materialized_table_gc";
 import { Model } from "./model";
 import { splitTablePath } from "./quoting";
 import { resolveEnvironmentId } from "./resolve_environment";
 
+type WireBuildGraph = components["schemas"]["BuildGraph"];
+type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
+type WireColumn = components["schemas"]["Column"];
+
 /**
- * Length of the BuildID prefix used when synthesizing staging table names.
- * BuildID is a 64-char SHA-256 hex string; 12 hex chars is 48 bits of entropy
- * — plenty of uniqueness per source, and keeps the final identifier well
- * inside every dialect's limit (Postgres is the tightest at 63).
+ * Length of the buildId prefix used when synthesizing staging table names.
+ * buildId is a 64-char SHA-256 hex string; 12 hex chars is 48 bits of
+ * entropy, well inside every dialect's identifier limit (Postgres is the
+ * tightest at 63).
  */
 const STAGING_BUILD_ID_LEN = 12;
 
-/**
- * Return the staging suffix `_<truncatedBuildId>` appended to a table name
- * while it is being built. The suffix is fully derivable from a manifest
- * entry's `buildId`, which is how GC finds and drops orphaned staging tables.
- */
+/** Staging suffix appended to a table name while it is being built. */
 export function stagingSuffix(buildId: string): string {
    return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
 }
 
+/** Output columns of a persist source, degrading to [] if unavailable. */
+function deriveColumns(persistSource: PersistSource): WireColumn[] {
+   try {
+      return persistSource._explore.intrinsicFields
+         .filter((f) => f.isAtomicField())
+         .map((f) => ({
+            name: f.name,
+            type: String((f as AtomicField).type),
+         }));
+   } catch (err) {
+      logger.warn("Failed to derive columns for persist source", {
+         sourceID: persistSource.sourceID,
+         error: err instanceof Error ? err.message : String(err),
+      });
+      return [];
+   }
+}
+
+/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
+function flattenDependsOn(node: {
+   dependsOn: { sourceID: string }[];
+}): string[] {
+   return node.dependsOn.map((d) => d.sourceID);
+}
+
 /**
- * Resolve a Map<name, Connection> for just the names a materialization
- * step is about to touch. The package's MalloyConfig caches each lookup,
- * so subsequent calls with overlapping names are cheap. A failed lookup
- * is logged and the name is omitted from the result — downstream code
- * (`forceDeleteRowOnMissingConnection` in teardown, or the explicit
- * "connection X not found" check in build) handles a missing entry.
+ * The buildId for a persist source: a stable digest of its connection identity
+ * and canonical SQL. Centralizes the (source, connectionDigests) call shape so
+ * planning, self-instruction, and build all agree on the same id.
+ */
+function computeBuildId(
+   source: PersistSource,
+   connectionDigests: Record<string, string>,
+): string {
+   return source.makeBuildId(
+      connectionDigests[source.connectionName],
+      source.getSQL(),
+   );
+}
+
+/**
+ * Physical table name the publisher self-assigns in auto-run mode: the
+ * `#@ persist name=<table>` value if present, else the Malloy source name.
+ * The author owns quoting the `name=` value for the dialect.
+ */
+function selfAssignTableName(persistSource: PersistSource): string {
+   try {
+      return (
+         persistSource.annotations.parseAsTag("@").tag.text("name") ||
+         persistSource.name
+      );
+   } catch {
+      return persistSource.name;
+   }
+}
+
+/** Classify a thrown round error as cancelled (cooperative abort) or failed. */
+function outcomeFor(
+   _err: unknown,
+   signal: AbortSignal | undefined,
+): "failed" | "cancelled" {
+   return signal?.aborted ? "cancelled" : "failed";
+}
+
+/**
+ * Resolve a Map<name, Connection> for the names a step is about to touch.
+ * The package's MalloyConfig caches each lookup, so repeated calls are cheap.
+ * A failed lookup is logged and omitted; downstream code reports the missing
+ * connection explicitly.
  */
 async function resolvePackageConnections(
    pkg: { getMalloyConnection(name: string): Promise<MalloyConnection> },
@@ -76,87 +145,41 @@ async function resolvePackageConnections(
 }
 
 /**
- * Build a stable key for a `(connectionName, tableName)` pair.
- * Used to check whether a persist target was created by a previous build.
- */
-export function manifestTableKey(
-   connectionName: string,
-   tableName: string,
-): string {
-   return `${connectionName}::${tableName}`;
-}
-
-/**
- * Probe whether a table physically exists on the given connection by
- * running a zero-row SELECT. Returns `true` if the table resolves,
- * `false` if the query fails (assumed "table not found").
- */
-/**
- * `tableName` is interpolated verbatim into the probe SQL — the caller
- * supplies it already quoted for the dialect (the `#@ persist name=…`
- * contract), matching how Malloy substitutes the name on the read side.
- */
-export async function tablePhysicallyExists(
-   connection: MalloyConnection,
-   tableName: string,
-): Promise<boolean> {
-   try {
-      await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
-      return true;
-   } catch {
-      return false;
-   }
-}
-
-/**
- * Allowed execution status transitions. SUCCESS, FAILED, and CANCELLED are
- * terminal — once an execution reaches one of these states it is immutable.
+ * Allowed status transitions for the two-round protocol. MANIFEST_FILE_READY,
+ * FAILED, and CANCELLED are terminal.
  */
 const VALID_TRANSITIONS: Record<
    MaterializationStatus,
    MaterializationStatus[]
 > = {
-   PENDING: ["RUNNING", "CANCELLED"],
-   RUNNING: ["SUCCESS", "FAILED", "CANCELLED"],
-   SUCCESS: [],
+   PENDING: ["BUILD_PLAN_READY", "FAILED", "CANCELLED"],
+   BUILD_PLAN_READY: ["MANIFEST_ROWS_READY", "FAILED", "CANCELLED"],
+   MANIFEST_ROWS_READY: ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"],
+   MANIFEST_FILE_READY: [],
    FAILED: [],
    CANCELLED: [],
 };
 
 /**
- * Orchestrates package-level materialization builds: triggering builds,
- * cancellation, and the actual Malloy build that materializes persist
- * sources into database tables.
- *
- * A build targets an entire package — all models are compiled and all
- * persist sources across all models are processed in dependency order.
- * The manifest is optionally activated after a successful build so
- * subsequent queries resolve persist references to materialized tables.
+ * Orchestrates the two-round materialization protocol.
  *
- * Enforces at-most-one concurrent build per (environment, package) via a
- * DB-level unique index on `materializations.active_key` (see
- * `MaterializationRepository`), and supports cooperative cancellation
- * through `AbortController`.
+ * Round 1 (on create): compile the package, compute a build plan (per-source
+ * buildId, columns, lineage inputs, connection capability) and pause at
+ * BUILD_PLAN_READY without writing any tables. Round 2 (action=build): build
+ * only the control-plane-instructed sources into their assigned physical
+ * names and assemble the manifest, which is returned inline for the control
+ * plane to persist.
  *
- * **Multi-worker caveat:** the `materializations` table lives in each
- * worker's *local* DuckDB, so the active-materialization lock is only
- * enforced within a single Publisher process. In orchestrated deployments
- * (shared DuckLake manifest catalog), builds must be externally
- * single-writer until a shared lease is added — see the scope note on
- * `DuckLakeManifestStore`.
+ * At most one active materialization per (environment, package) is enforced
+ * by the DB-level unique index on `materializations.active_key` (see
+ * {@link MaterializationRepository}). Cancellation is cooperative via
+ * AbortController.
  */
 export class MaterializationService {
-   /**
-    * Tracks in-flight executions so they can be cancelled. This map only
-    * lives in-process memory — entries are lost on server restart, which is
-    * why `stopMaterialization` has an orphaned-execution fallback path.
-    */
+   /** In-flight runs, so they can be cancelled. In-process only. */
    private runningAbortControllers = new Map<string, AbortController>();
 
-   constructor(
-      private environmentStore: EnvironmentStore,
-      private manifestService: ManifestService,
-   ) {}
+   constructor(private environmentStore: EnvironmentStore) {}
 
    private get repository(): ResourceRepository {
       return this.environmentStore.storageManager.getRepository();
@@ -168,39 +191,37 @@ export class MaterializationService {
       current: MaterializationStatus,
       next: MaterializationStatus,
    ): void {
-      const allowed = VALID_TRANSITIONS[current];
-      if (!allowed.includes(next)) {
+      if (!VALID_TRANSITIONS[current].includes(next)) {
          throw new InvalidStateTransitionError(
             `Cannot transition from ${current} to ${next}`,
          );
       }
    }
 
-   private async transitionExecution(
-      executionId: string,
-      newStatus: MaterializationStatus,
-      extra?: {
-         startedAt?: Date;
-         completedAt?: Date;
-         error?: string | null;
-         metadata?: Record<string, unknown> | null;
-      },
+   private async transition(
+      id: string,
+      next: MaterializationStatus,
+      extra?: Omit<MaterializationUpdate, "status">,
    ): Promise<Materialization> {
-      const execution =
-         await this.repository.getMaterializationById(executionId);
-      if (!execution) {
+      const current = await this.repository.getMaterializationById(id);
+      if (!current) {
          throw new MaterializationNotFoundError(
-            `Execution ${executionId} not found`,
+            `Materialization ${id} not found`,
          );
       }
-      this.validateTransition(execution.status, newStatus);
-      return this.repository.updateMaterialization(executionId, {
-         status: newStatus,
+      this.validateTransition(current.status, next);
+      logger.info("Materialization transition", {
+         materializationId: id,
+         from: current.status,
+         to: next,
+      });
+      return this.repository.updateMaterialization(id, {
+         status: next,
          ...extra,
       });
    }
 
-   // ==================== BUILD QUERIES ====================
+   // ==================== QUERIES ====================
 
    async listMaterializations(
       environmentName: string,
@@ -218,45 +239,45 @@ export class MaterializationService {
    async getMaterialization(
       environmentName: string,
       packageName: string,
-      buildId: string,
+      id: string,
    ): Promise<Materialization> {
       const environmentId = await this.resolveEnvironmentId(environmentName);
-      const execution = await this.repository.getMaterializationById(buildId);
+      const m = await this.repository.getMaterializationById(id);
       if (
-         !execution ||
-         execution.environmentId !== environmentId ||
-         execution.packageName !== packageName
+         !m ||
+         m.environmentId !== environmentId ||
+         m.packageName !== packageName
       ) {
          throw new MaterializationNotFoundError(
-            `Materialization ${buildId} not found for package ${packageName}`,
+            `Materialization ${id} not found for package ${packageName}`,
          );
       }
-      return execution;
+      return m;
    }
 
-   // ==================== BUILD LIFECYCLE ====================
+   // ==================== ROUND 1: CREATE + PLAN ====================
 
    /**
-    * Creates a new build in PENDING state. Build options are stored in
-    * metadata so `startMaterialization` can read them back.
+    * Create a materialization and start Round 1 (compile + plan) in the
+    * background. Returns the PENDING record immediately.
     */
    async createMaterialization(
       environmentName: string,
       packageName: string,
-      options: { forceRefresh?: boolean; autoLoadManifest?: boolean } = {},
+      options: {
+         forceRefresh?: boolean;
+         sourceNames?: string[];
+         pauseBetweenPhases?: boolean;
+      } = {},
    ): Promise<Materialization> {
       const environmentId = await this.resolveEnvironmentId(environmentName);
 
-      // Verify the package exists.
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
          false,
       );
       await environment.getPackage(packageName, false);
 
-      // A non-atomic probe for a helpful error message. The DB-level unique
-      // index on active_key is the actual race-free guard — see the catch
-      // block below.
       const active = await this.repository.getActiveMaterialization(
          environmentId,
          packageName,
@@ -267,13 +288,17 @@ export class MaterializationService {
          );
       }
 
+      const pauseBetweenPhases = options.pauseBetweenPhases ?? false;
+      const forceRefresh = options.forceRefresh ?? false;
       const metadata = {
-         forceRefresh: options.forceRefresh ?? false,
-         autoLoadManifest: options.autoLoadManifest ?? false,
+         forceRefresh,
+         sourceNames: options.sourceNames ?? null,
+         pauseBetweenPhases,
       };
 
+      let created: Materialization;
       try {
-         return await this.repository.createMaterialization(
+         created = await this.repository.createMaterialization(
             environmentId,
             packageName,
             "PENDING",
@@ -281,8 +306,6 @@ export class MaterializationService {
          );
       } catch (err) {
          if (err instanceof DuplicateActiveMaterializationError) {
-            // Lost the race with a concurrent create. Re-read to report the
-            // winner's id for parity with the non-racy error above.
             const winner = await this.repository.getActiveMaterialization(
                environmentId,
                packageName,
@@ -295,343 +318,443 @@ export class MaterializationService {
          }
          throw err;
       }
+
+      // Default (auto-run): compile, self-assign names, build, and auto-load in
+      // one pass. Opt-in two-round: pause at BUILD_PLAN_READY for the control
+      // plane to drive Round 2.
+      this.runInBackground(created.id, (signal) =>
+         pauseBetweenPhases
+            ? this.runRound1(
+                 created.id,
+                 environmentName,
+                 packageName,
+                 options.sourceNames,
+                 signal,
+              )
+            : this.runAuto(
+                 created.id,
+                 environmentName,
+                 packageName,
+                 options.sourceNames,
+                 forceRefresh,
+                 signal,
+              ),
+      );
+
+      return created;
    }
 
-   /**
-    * Transitions a PENDING build to RUNNING and starts execution in the
-    * background. Returns the RUNNING execution immediately.
-    */
-   async startMaterialization(
+   private async runRound1(
+      id: string,
       environmentName: string,
       packageName: string,
-      buildId: string,
-   ): Promise<Materialization> {
-      const environmentId = await this.resolveEnvironmentId(environmentName);
-      const execution = await this.getMaterialization(
-         environmentName,
+      sourceNames: string[] | undefined,
+      signal?: AbortSignal,
+   ): Promise<void> {
+      logger.info("Materialization Round 1: compile + plan", {
+         materializationId: id,
          packageName,
-         buildId,
-      );
-
-      if (execution.status !== "PENDING") {
-         throw new InvalidStateTransitionError(
-            `Materialization ${buildId} is ${execution.status}, expected PENDING`,
-         );
-      }
+      });
+      const startedAt = Date.now();
 
-      // Check for a *different* active materialization on this package.
-      const active = await this.repository.getActiveMaterialization(
-         environmentId,
-         packageName,
-      );
-      if (active && active.id !== execution.id) {
-         throw new MaterializationConflictError(
-            `Package ${packageName} already has an active materialization (${active.id})`,
+      try {
+         const environment = await this.environmentStore.getEnvironment(
+            environmentName,
+            false,
          );
-      }
+         const pkg = await environment.getPackage(packageName, false);
 
-      const running = await this.transitionExecution(execution.id, "RUNNING", {
-         startedAt: new Date(),
-      });
+         const { graphs, sources, connectionDigests } =
+            await environment.withPackageLock(packageName, () =>
+               this.compilePackageBuildPlan(pkg, signal),
+            );
 
-      const metadata = (execution.metadata ?? {}) as Record<string, unknown>;
+         const buildPlan = this.deriveBuildPlan(
+            graphs,
+            sources,
+            connectionDigests,
+            sourceNames,
+         );
 
-      // Fire-and-forget: run the build in the background.
-      this.runMaterialization(
-         execution.id,
-         environmentName,
-         environmentId,
-         packageName,
-         metadata,
-      ).catch((err) => {
-         logger.error("Unhandled error in background build", {
-            executionId: execution.id,
-            error: err instanceof Error ? err.message : String(err),
+         await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
+         this.recordRound("round1", "success", startedAt);
+         logger.info("Materialization Round 1 complete", {
+            materializationId: id,
+            packageName,
+            sourceCount: Object.keys(buildPlan.sources).length,
+            durationMs: Date.now() - startedAt,
          });
-      });
-
-      return running;
+      } catch (err) {
+         this.recordRound("round1", outcomeFor(err, signal), startedAt);
+         throw err;
+      }
    }
 
-   private async runMaterialization(
-      executionId: string,
+   // ==================== AUTO-RUN (DEFAULT) ====================
+
+   /**
+    * Auto-run (default, pauseBetweenPhases=false). Compile + plan, then
+    * self-assign physical table names (the `#@ persist name=` value, else the
+    * source name) with realization=COPY, skip sources whose buildId is
+    * unchanged since the most recent successful manifest (unless forceRefresh),
+    * build the rest, assemble the manifest, and auto-load it into the package
+    * models so subsequent queries resolve to the materialized tables. No pause,
+    * no second round.
+    */
+   private async runAuto(
+      id: string,
       environmentName: string,
-      environmentId: string,
       packageName: string,
-      metadata: Record<string, unknown>,
+      sourceNames: string[] | undefined,
+      forceRefresh: boolean,
+      signal: AbortSignal,
    ): Promise<void> {
-      const abortController = new AbortController();
-      this.runningAbortControllers.set(executionId, abortController);
+      logger.info("Materialization auto-run: compile + build + load", {
+         materializationId: id,
+         packageName,
+      });
+      const startedAt = Date.now();
 
       try {
-         const buildMetadata = await this.executeBuild(
+         const environmentId = await this.resolveEnvironmentId(environmentName);
+         const environment = await this.environmentStore.getEnvironment(
             environmentName,
-            environmentId,
-            packageName,
-            !!metadata.forceRefresh,
-            abortController.signal,
+            false,
          );
+         const pkg = await environment.getPackage(packageName, false);
 
-         if (metadata.autoLoadManifest) {
-            const updatedManifest = await this.manifestService.getManifest(
-               environmentId,
-               packageName,
-            );
-            const environment = await this.environmentStore.getEnvironment(
-               environmentName,
-               false,
-            );
-            // Ensure the package is loaded, then reload models under the
-            // per-package mutex so the disk reads are serialized against
-            // installPackage / deletePackage.
-            await environment.getPackage(packageName, false);
-            await environment.reloadAllModelsForPackage(
-               packageName,
-               updatedManifest.entries,
-            );
-         }
+         const compiled = await environment.withPackageLock(packageName, () =>
+            this.compilePackageBuildPlan(pkg, signal),
+         );
 
-         await this.transitionExecution(executionId, "SUCCESS", {
-            completedAt: new Date(),
-            metadata: { ...metadata, ...buildMetadata },
+         const buildPlan = this.deriveBuildPlan(
+            compiled.graphs,
+            compiled.sources,
+            compiled.connectionDigests,
+            sourceNames,
+         );
+         await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
+
+         // Skip-if-unchanged: reuse tables from the most recent successful
+         // manifest for sources whose buildId is unchanged, unless forceRefresh.
+         const priorEntries = forceRefresh
+            ? {}
+            : await this.getMostRecentManifestEntries(
+                 environmentId,
+                 packageName,
+                 id,
+              );
+         const { instructions, carried } = this.deriveSelfInstructions(
+            compiled,
+            sourceNames,
+            priorEntries,
+         );
+
+         const entries = await this.executeInstructedBuild(
+            compiled,
+            instructions,
+            carried,
+            signal,
+         );
+
+         const durationMs = Date.now() - startedAt;
+         await this.commitManifest(id, entries, {
+            forceRefresh,
+            sourceNames: sourceNames ?? null,
+            pauseBetweenPhases: false,
+            sourcesBuilt: instructions.length,
+            sourcesReused: Object.keys(carried).length,
+            durationMs,
+         });
+
+         await this.autoLoadManifest(environment, packageName, entries);
+
+         this.recordRound("auto", "success", startedAt);
+         logger.info("Materialization auto-run complete", {
+            materializationId: id,
+            packageName,
+            sourcesBuilt: instructions.length,
+            sourcesReused: Object.keys(carried).length,
+            durationMs,
          });
       } catch (err) {
-         const errorMessage = err instanceof Error ? err.message : String(err);
+         this.recordRound("auto", outcomeFor(err, signal), startedAt);
+         throw err;
+      }
+   }
 
-         try {
-            if (abortController.signal.aborted) {
-               await this.transitionExecution(executionId, "CANCELLED", {
-                  completedAt: new Date(),
-                  error: "Build cancelled",
-               });
-            } else {
-               await this.transitionExecution(executionId, "FAILED", {
-                  completedAt: new Date(),
-                  error: errorMessage,
+   /**
+    * Derive the publisher's own build instructions for auto-run. Each persist
+    * source (respecting the optional sourceNames filter) gets a self-assigned
+    * physical table name and COPY realization, unless its buildId is unchanged
+    * since `priorEntries` — those are carried forward (reused) instead of
+    * rebuilt.
+    */
+   private deriveSelfInstructions(
+      compiled: {
+         graphs: MalloyBuildGraph[];
+         sources: Record<string, PersistSource>;
+         connectionDigests: Record<string, string>;
+      },
+      sourceNames: string[] | undefined,
+      priorEntries: Record<string, ManifestEntry>,
+   ): {
+      instructions: BuildInstruction[];
+      carried: Record<string, ManifestEntry>;
+   } {
+      const include = sourceNames ? new Set(sourceNames) : null;
+      const instructions: BuildInstruction[] = [];
+      const carried: Record<string, ManifestEntry> = {};
+      const seen = new Set<string>();
+
+      for (const graph of compiled.graphs) {
+         for (const level of graph.nodes) {
+            for (const node of level) {
+               const persistSource = compiled.sources[node.sourceID];
+               if (!persistSource) continue;
+               if (include && !include.has(persistSource.name)) continue;
+
+               const buildId = computeBuildId(
+                  persistSource,
+                  compiled.connectionDigests,
+               );
+               if (seen.has(buildId)) continue;
+               seen.add(buildId);
+
+               const prior = priorEntries[buildId];
+               if (prior && prior.physicalTableName) {
+                  carried[buildId] = prior;
+                  continue;
+               }
+
+               instructions.push({
+                  buildId,
+                  materializedTableId: `local-${buildId.substring(
+                     0,
+                     STAGING_BUILD_ID_LEN,
+                  )}`,
+                  physicalTableName: selfAssignTableName(persistSource),
+                  realization: "COPY",
                });
             }
-         } catch (transitionErr) {
-            logger.error("Failed to transition execution after build error", {
-               executionId,
-               originalError: errorMessage,
-               transitionError:
-                  transitionErr instanceof Error
-                     ? transitionErr.message
-                     : String(transitionErr),
-            });
          }
-      } finally {
-         this.runningAbortControllers.delete(executionId);
       }
+
+      return { instructions, carried };
    }
 
    /**
-    * Cancels a running build. Takes a specific buildId.
+    * Entries of the most recent successful (MANIFEST_FILE_READY) materialization
+    * for this package, used for skip-if-unchanged. Excludes the in-flight run.
     */
-   async stopMaterialization(
-      environmentName: string,
+   private async getMostRecentManifestEntries(
+      environmentId: string,
       packageName: string,
-      buildId: string,
-   ): Promise<Materialization> {
-      const execution = await this.getMaterialization(
-         environmentName,
+      excludeId: string,
+   ): Promise<Record<string, ManifestEntry>> {
+      const list = await this.repository.listMaterializations(
+         environmentId,
          packageName,
-         buildId,
       );
-
-      if (execution.status !== "RUNNING" && execution.status !== "PENDING") {
-         throw new InvalidStateTransitionError(
-            `Materialization ${buildId} is ${execution.status}, cannot stop`,
-         );
-      }
-
-      if (execution.status === "PENDING") {
-         return this.transitionExecution(execution.id, "CANCELLED", {
-            completedAt: new Date(),
-            error: "Build cancelled before starting",
-         });
-      }
-
-      const abortController = this.runningAbortControllers.get(execution.id);
-      if (abortController) {
-         abortController.abort();
-         return execution;
-      } else {
-         return this.transitionExecution(execution.id, "CANCELLED", {
-            completedAt: new Date(),
-            error: "Force cancelled: execution was orphaned",
-         });
+      for (const m of list) {
+         if (m.id === excludeId) continue;
+         if (m.status === "MANIFEST_FILE_READY" && m.manifest?.entries) {
+            return m.manifest.entries;
+         }
       }
+      return {};
    }
 
    /**
-    * Deletes a materialization record. Only terminal materializations
-    * (SUCCESS, FAILED, CANCELLED) can be deleted.
+    * Load a freshly produced manifest into the package's models so persist
+    * references resolve to the materialized tables. Best-effort: a load failure
+    * is logged, not fatal (the run already reached MANIFEST_FILE_READY).
     */
-   async deleteMaterialization(
-      environmentName: string,
+   private async autoLoadManifest(
+      environment: {
+         reloadAllModelsForPackage(
+            packageName: string,
+            manifest: BuildManifest["entries"],
+         ): Promise<void>;
+      },
       packageName: string,
-      materializationId: string,
+      entries: Record<string, ManifestEntry>,
    ): Promise<void> {
-      const execution = await this.getMaterialization(
-         environmentName,
-         packageName,
-         materializationId,
-      );
-
-      if (execution.status === "PENDING" || execution.status === "RUNNING") {
-         throw new InvalidStateTransitionError(
-            `Cannot delete materialization ${materializationId} while it is ${execution.status}`,
+      const manifestEntries: BuildManifest["entries"] = {};
+      for (const [buildId, entry] of Object.entries(entries)) {
+         if (entry.physicalTableName) {
+            manifestEntries[buildId] = { tableName: entry.physicalTableName };
+         }
+      }
+      try {
+         await environment.reloadAllModelsForPackage(
+            packageName,
+            manifestEntries,
          );
+         logger.info("Auto-run: loaded manifest into package models", {
+            packageName,
+            entryCount: Object.keys(manifestEntries).length,
+         });
+      } catch (err) {
+         logger.warn("Auto-run: failed to load manifest into package models", {
+            packageName,
+            error: err instanceof Error ? err.message : String(err),
+         });
       }
-
-      await this.repository.deleteMaterialization(execution.id);
    }
 
-   // ==================== PACKAGE TEARDOWN ====================
+   // ==================== ROUND 2: BUILD ====================
 
    /**
-    * Drop every materialized table and manifest row for a package.
-    *
-    * This is the only out-of-band teardown surface exposed by the
-    * publisher and is intended for a single caller: the controlplane,
-    * invoking it on the truly destructive path before the package/environment
-    * is torn down. The publisher's `DELETE` endpoints are *not* wired into
-    * teardown because the controlplane invokes them for non-destructive
-    * unload too (down-replicate, drain, archive) where the entity still
-    * exists on other replicas, and dropping shared tables there would
-    * corrupt surviving workers.
-    *
-    * Reconciliation of stale rows against the package's live source code
-    * happens inline at the end of every successful build (see
-    * {@link executeBuild} Step 5) — that is where "active" vs. "orphan"
-    * is authoritatively determined, using the manifest's own `touch()`
-    * bookkeeping. This endpoint does not re-derive that information; it
-    * drops the manifest in its entirety, which is the correct behavior
-    * for the pre-deletion use case and avoids pulling the package's
-    * source through the compiler just to reach a foregone conclusion.
-    *
-    * Refuses to run while a materialization is active for the package
-    * (same serialization the inline GC gets by piggy-backing on the
-    * build).
-    *
-    * `dryRun` returns what would be dropped without issuing any DROP or
-    * deleting manifest rows.
+    * Round 2. Validates the instructions against the stored build plan,
+    * transitions out of BUILD_PLAN_READY, and builds the instructed sources
+    * in the background. Returns the accepted record (202).
     */
-   async teardownPackage(
+   async buildMaterialization(
       environmentName: string,
       packageName: string,
-      options: { dryRun?: boolean } = {},
-   ): Promise<GcResult> {
-      const environmentId = await this.resolveEnvironmentId(environmentName);
+      id: string,
+      instructions: BuildInstruction[],
+   ): Promise<Materialization> {
+      const m = await this.getMaterialization(environmentName, packageName, id);
 
-      const active = await this.repository.getActiveMaterialization(
-         environmentId,
-         packageName,
-      );
-      if (active) {
-         throw new MaterializationConflictError(
-            `Package ${packageName} has an active materialization (${active.id}); cannot tear down`,
+      if (!m.pauseBetweenPhases) {
+         throw new InvalidStateTransitionError(
+            `Materialization ${id} is an auto-run materialization; action=build does not apply (set pauseBetweenPhases=true for the two-round flow)`,
+         );
+      }
+      if (m.status !== "BUILD_PLAN_READY") {
+         throw new InvalidStateTransitionError(
+            `Materialization ${id} is ${m.status}, expected BUILD_PLAN_READY`,
+         );
+      }
+      const plan = m.buildPlan;
+      if (!plan) {
+         throw new InvalidStateTransitionError(
+            `Materialization ${id} has no build plan`,
          );
       }
 
-      const environment = await this.environmentStore.getEnvironment(
-         environmentName,
-         false,
-      );
-      const pkg = await environment.getPackage(packageName, false);
+      this.validateInstructions(plan, instructions);
 
-      const entries = await this.manifestService.listEntries(
-         environmentId,
-         packageName,
+      this.runInBackground(id, (signal) =>
+         this.runRound2(id, environmentName, packageName, instructions, signal),
       );
 
-      const connections = await resolvePackageConnections(
-         pkg,
-         entries.map((e) => e.connectionName),
-      );
-
-      // `forceDeleteRowOnMissingConnection`: teardown is the one place
-      // where we'd rather lose the manifest row than leave it pointing at
-      // a vanished connection. We also deliberately omit `liveTables`:
-      // in teardown everything is stale, nothing is live.
-      return dropManifestEntries(entries, {
-         connections,
-         manifestService: this.manifestService,
-         environmentId,
-         dryRun: options.dryRun,
-         forceDeleteRowOnMissingConnection: true,
-      });
+      return m;
    }
 
-   // ==================== BUILD LOGIC ====================
+   private validateInstructions(
+      plan: BuildPlan,
+      instructions: BuildInstruction[],
+   ): void {
+      const plannedBuildIds = new Set<string>();
+      for (const source of Object.values(plan.sources)) {
+         plannedBuildIds.add(source.buildId);
+      }
 
-   /**
-    * Core build pipeline (5 steps):
-    *   1. LOAD  — Load existing manifest.
-    *   2. COMPILE & PLAN — Compile all models, collect dependency graphs.
-    *   3. BUILD — Walk graphs in dependency order, materialize each source.
-    *   4. GC — Drop stale physical tables + prune manifest rows.
-    *
-    * Build success is not gated on GC — failures are surfaced in
-    * `gcErrors` metadata so the controller/UI can show them.
-    */
-   private async executeBuild(
+      for (const instruction of instructions) {
+         if (!plannedBuildIds.has(instruction.buildId)) {
+            throw new BadRequestError(
+               `Instruction references unknown buildId '${instruction.buildId}'`,
+            );
+         }
+         // v0 is COPY-only; SNAPSHOT lands once clone semantics are defined.
+         if (instruction.realization === "SNAPSHOT") {
+            throw new BadRequestError(
+               "realization=SNAPSHOT is not supported in v0 (COPY only)",
+            );
+         }
+      }
+   }
+
+   private async runRound2(
+      id: string,
       environmentName: string,
-      environmentId: string,
       packageName: string,
-      forceRefresh: boolean,
+      instructions: BuildInstruction[],
       signal: AbortSignal,
-   ): Promise<Record<string, unknown>> {
-      logger.info("Starting materialization build", {
-         environmentName,
+   ): Promise<void> {
+      logger.info("Materialization Round 2: build", {
+         materializationId: id,
          packageName,
+         sourceCount: instructions.length,
       });
+      const startedAt = Date.now();
 
-      const environment = await this.environmentStore.getEnvironment(
-         environmentName,
-         false,
-      );
-      const pkg = await environment.getPackage(packageName, false);
-
-      // ── STEP 1: LOAD ───────────────────────────────────────────────
-      const manifest = new Manifest();
-      const existingManifest = await this.manifestService.getManifest(
-         environmentId,
-         packageName,
-      );
-      manifest.loadText(JSON.stringify(existingManifest));
-
-      const existingEntries = await this.manifestService.listEntries(
-         environmentId,
-         packageName,
-      );
-      const knownMaterializedTables = new Set(
-         existingEntries.map((e: ManifestEntry) =>
-            manifestTableKey(e.connectionName, e.tableName),
-         ),
-      );
+      try {
+         const environment = await this.environmentStore.getEnvironment(
+            environmentName,
+            false,
+         );
+         const pkg = await environment.getPackage(packageName, false);
 
-      // ── STEP 2: COMPILE & PLAN ─────────────────────────────────────
-      // `connections` is built lazily from the connection names the plan
-      // actually targets — no upfront ATTACH on every environment connection.
-      // Hold the per-package mutex for the duration of the compile so the
-      // `fs.stat` + `runtime.loadModel` calls inside `compilePackageBuildPlan`
-      // are serialized against `installPackage` / `deletePackage`.
-      const { graphs, sources, connectionDigests, connections } =
-         await environment.withPackageLock(packageName, () =>
+         const compiled = await environment.withPackageLock(packageName, () =>
             this.compilePackageBuildPlan(pkg, signal),
          );
 
-      if (graphs.length === 0) {
-         logger.info("No persist sources to build");
-         return { sourcesBuilt: 0, sourcesSkipped: 0 };
+         const entries = await this.executeInstructedBuild(
+            compiled,
+            instructions,
+            {},
+            signal,
+         );
+
+         const durationMs = Date.now() - startedAt;
+         await this.commitManifest(id, entries, {
+            sourcesBuilt: Object.keys(entries).length,
+            durationMs,
+         });
+
+         this.recordRound("round2", "success", startedAt);
+         logger.info("Materialization Round 2 complete", {
+            materializationId: id,
+            packageName,
+            sourcesBuilt: Object.keys(entries).length,
+            durationMs,
+         });
+      } catch (err) {
+         this.recordRound("round2", outcomeFor(err, signal), startedAt);
+         throw err;
       }
+   }
+
+   /**
+    * Shared build loop for both auto-run and Round 2. Seeds the manifest with
+    * carried-forward (reused) upstream entries so downstream references resolve,
+    * then builds each instructed source in dependency order. Returns the full
+    * entry map (carried + freshly built). The package was compiled by the
+    * caller; this runs outside the package lock.
+    */
+   private async executeInstructedBuild(
+      compiled: {
+         graphs: MalloyBuildGraph[];
+         sources: Record<string, PersistSource>;
+         connectionDigests: Record<string, string>;
+         connections: Map<string, MalloyConnection>;
+      },
+      instructions: BuildInstruction[],
+      seedEntries: Record<string, ManifestEntry>,
+      signal: AbortSignal,
+   ): Promise<Record<string, ManifestEntry>> {
+      const { graphs, sources, connectionDigests, connections } = compiled;
 
-      // ── STEP 3: BUILD ──────────────────────────────────────────────
-      let sourcesBuilt = 0;
-      let sourcesSkipped = 0;
-      const sourceResults: Record<string, unknown>[] = [];
+      const byBuildId = new Map<string, BuildInstruction>();
+      for (const instruction of instructions) {
+         byBuildId.set(instruction.buildId, instruction);
+      }
+
+      // Accumulates physical names as sources are built so downstream sources
+      // resolve their upstream references to the freshly-assigned tables. Seed
+      // it with carried-forward entries so reused upstreams resolve too.
+      const manifest = new Manifest();
+      const entries: Record<string, ManifestEntry> = {};
+      for (const [buildId, entry] of Object.entries(seedEntries)) {
+         if (entry.physicalTableName) {
+            manifest.update(buildId, { tableName: entry.physicalTableName });
+         }
+         entries[buildId] = entry;
+      }
 
       for (const graph of graphs) {
          const connection = connections.get(graph.connectionName);
@@ -640,66 +763,270 @@ export class MaterializationService {
                `Connection '${graph.connectionName}' not found`,
             );
          }
-
          for (const level of graph.nodes) {
             for (const node of level) {
                if (signal.aborted) throw new Error("Build cancelled");
-
                const persistSource = sources[node.sourceID];
-               if (!persistSource) {
-                  logger.warn(
-                     `Source ${node.sourceID} not found in build plan, skipping`,
-                  );
-                  continue;
-               }
+               if (!persistSource) continue;
+
+               const buildId = computeBuildId(persistSource, connectionDigests);
+               const instruction = byBuildId.get(buildId);
+               if (!instruction) continue;
 
-               const result = await this.buildOneSource(
+               const entry = await this.buildOneSource(
                   persistSource,
-                  manifest,
+                  instruction,
                   connection,
                   connectionDigests,
-                  forceRefresh,
-                  environmentId,
-                  packageName,
-                  knownMaterializedTables,
+                  manifest,
                );
-
-               sourceResults.push(result);
-               if (result.status === "built") sourcesBuilt++;
-               else sourcesSkipped++;
+               entries[buildId] = entry;
             }
          }
       }
 
-      // ── STEP 4: GC ─────────────────────────────────────────────────
-      const gcResult = await this.runPostBuildGc(
-         manifest,
-         environmentId,
-         packageName,
-         connections,
-      );
+      return entries;
+   }
+
+   /**
+    * Build a single instructed source into its assigned physical table.
+    * COPY uses a staging table + atomic rename for crash-safety; the staging
+    * name derives from the buildId. Records and returns the manifest entry.
+    */
+   private async buildOneSource(
+      persistSource: PersistSource,
+      instruction: BuildInstruction,
+      connection: MalloyConnection,
+      connectionDigests: Record<string, string>,
+      manifest: Manifest,
+   ): Promise<ManifestEntry> {
+      const buildId = instruction.buildId;
+      const physicalTableName = instruction.physicalTableName;
+      const buildSQL = persistSource.getSQL({
+         buildManifest: manifest.buildManifest,
+         connectionDigests,
+      });
+
+      const { bareName } = splitTablePath(physicalTableName);
+      const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
+
+      const startTime = performance.now();
+      await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
+      try {
+         await connection.runSQL(
+            `CREATE TABLE ${stagingTableName} AS (${buildSQL})`,
+         );
+         await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
+         await connection.runSQL(
+            `ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`,
+         );
+      } catch (err) {
+         try {
+            await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
+         } catch (cleanupErr) {
+            logger.warn(
+               "Round 2: failed to clean up staging table after a failed build; physical leak",
+               {
+                  stagingTableName,
+                  connectionName: persistSource.connectionName,
+                  cleanupError:
+                     cleanupErr instanceof Error
+                        ? cleanupErr.message
+                        : String(cleanupErr),
+               },
+            );
+         }
+         throw err;
+      }
 
-      logger.info("Materialization build complete", {
-         sourcesBuilt,
-         sourcesSkipped,
-         gcDropped: gcResult.dropped.length,
-         gcErrors: gcResult.errors.length,
+      // Make this table visible to downstream sources built later this round.
+      manifest.update(buildId, { tableName: physicalTableName });
+
+      const durationMs = Math.round(performance.now() - startTime);
+      recordSourceBuildDuration(durationMs);
+      // Shared by auto-run and Round 2, so the message is path-neutral.
+      logger.info(`Built materialized source ${persistSource.name}`, {
+         physicalTableName,
+         durationMs,
       });
 
       return {
-         sourcesBuilt,
-         sourcesSkipped,
-         sources: sourceResults,
-         gcDropped: gcResult.dropped,
-         gcErrors: gcResult.errors,
+         buildId,
+         sourceName: persistSource.name,
+         materializedTableId: instruction.materializedTableId,
+         physicalTableName,
+         connectionName: persistSource.connectionName,
+         realization: instruction.realization,
+         rowCount: null,
+      };
+   }
+
+   // ==================== CANCELLATION ====================
+
+   /** Cancel a non-terminal materialization. */
+   async stopMaterialization(
+      environmentName: string,
+      packageName: string,
+      id: string,
+   ): Promise<Materialization> {
+      const m = await this.getMaterialization(environmentName, packageName, id);
+
+      const cancellable: MaterializationStatus[] = [
+         "PENDING",
+         "BUILD_PLAN_READY",
+         "MANIFEST_ROWS_READY",
+      ];
+      if (!cancellable.includes(m.status)) {
+         throw new InvalidStateTransitionError(
+            `Materialization ${id} is ${m.status}, cannot stop`,
+         );
+      }
+
+      const abortController = this.runningAbortControllers.get(id);
+      if (abortController) {
+         abortController.abort();
+         return m;
+      }
+      return this.transition(id, "CANCELLED", {
+         completedAt: new Date(),
+         error: "Cancelled",
+      });
+   }
+
+   /**
+    * Delete a materialization record. Only terminal materializations
+    * (MANIFEST_FILE_READY, FAILED, CANCELLED) can be deleted; an active run must
+    * be stopped first.
+    *
+    * By default this removes the publisher's record only — the control plane
+    * owns table GC. When `dropTables` is set, the publisher additionally drops
+    * the physical tables recorded in this run's manifest as a best-effort
+    * cleanup (a drop failure is logged, not fatal, so the record still deletes).
+    */
+   async deleteMaterialization(
+      environmentName: string,
+      packageName: string,
+      id: string,
+      options: { dropTables?: boolean } = {},
+   ): Promise<void> {
+      const m = await this.getMaterialization(environmentName, packageName, id);
+
+      const terminal: MaterializationStatus[] = [
+         "MANIFEST_FILE_READY",
+         "FAILED",
+         "CANCELLED",
+      ];
+      if (!terminal.includes(m.status)) {
+         throw new InvalidStateTransitionError(
+            `Cannot delete materialization ${id} while it is ${m.status}`,
+         );
+      }
+
+      if (options.dropTables) {
+         await this.dropMaterializedTables(environmentName, packageName, m);
+      }
+
+      await this.repository.deleteMaterialization(id);
+   }
+
+   /**
+    * Best-effort drop of every physical table this run produced, read from the
+    * materialization's manifest. Resolves each entry's connection by name and
+    * issues `DROP TABLE IF EXISTS` for the physical table and its (possible)
+    * leftover staging table. Failures are logged and swallowed so a partial
+    * cleanup never blocks deletion of the record.
+    */
+   private async dropMaterializedTables(
+      environmentName: string,
+      packageName: string,
+      m: Materialization,
+   ): Promise<void> {
+      const entries = m.manifest?.entries;
+      if (!entries || Object.keys(entries).length === 0) {
+         return;
+      }
+
+      const environment = await this.environmentStore.getEnvironment(
+         environmentName,
+         false,
+      );
+      const pkg = await environment.getPackage(packageName, false);
+      const connectionCache = new Map<string, MalloyConnection>();
+
+      for (const entry of Object.values(entries)) {
+         const connectionName = entry.connectionName;
+         const physicalTableName = entry.physicalTableName;
+         if (!connectionName || !physicalTableName) {
+            logger.warn("Skipping manifest entry with no connection/table", {
+               materializationId: m.id,
+               buildId: entry.buildId,
+            });
+            continue;
+         }
+
+         try {
+            let connection = connectionCache.get(connectionName);
+            if (!connection) {
+               connection = await pkg.getMalloyConnection(connectionName);
+               connectionCache.set(connectionName, connection);
+            }
+            await connection.runSQL(
+               `DROP TABLE IF EXISTS ${physicalTableName}`,
+            );
+            // A crash between staging-create and rename can leave the staging
+            // table behind; clean it up too while we hold the connection.
+            await connection.runSQL(
+               `DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(
+                  entry.buildId,
+               )}`,
+            );
+            recordDropTables("success");
+            logger.info("Dropped materialized table on delete", {
+               materializationId: m.id,
+               physicalTableName,
+               connectionName,
+            });
+         } catch (err) {
+            recordDropTables("failure");
+            logger.warn("Failed to drop materialized table on delete", {
+               materializationId: m.id,
+               physicalTableName,
+               connectionName,
+               error: err instanceof Error ? err.message : String(err),
+            });
+         }
+      }
+   }
+
+   /**
+    * Finalize a successful build: advance MANIFEST_ROWS_READY -> FILE_READY and
+    * persist the assembled manifest. Shared by auto-run and Round 2; the caller
+    * supplies the per-path metadata. The build itself happens before this.
+    */
+   private async commitManifest(
+      id: string,
+      entries: Record<string, ManifestEntry>,
+      metadata: Record<string, unknown>,
+   ): Promise<void> {
+      await this.transition(id, "MANIFEST_ROWS_READY");
+      const manifest: BuildManifestResult = {
+         builtAt: new Date().toISOString(),
+         entries,
+         strict: false,
       };
+      await this.transition(id, "MANIFEST_FILE_READY", {
+         completedAt: new Date(),
+         manifest,
+         metadata,
+      });
    }
 
-   // ==================== BUILD HELPERS ====================
+   // ==================== BUILD PLAN COMPILATION ====================
 
    /**
     * Compile every model in the package and collect the dependency-ordered
-    * build graphs, persist sources, and pre-computed connection digests.
+    * build graphs, persist sources, connection digests, and resolved
+    * connections.
     */
    private async compilePackageBuildPlan(
       pkg: {
@@ -708,19 +1035,18 @@ export class MaterializationService {
          getMalloyConfig(): MalloyConfig;
          getMalloyConnection(name: string): Promise<MalloyConnection>;
       },
-      signal: AbortSignal,
+      signal?: AbortSignal,
    ): Promise<{
-      graphs: BuildGraph[];
+      graphs: MalloyBuildGraph[];
       sources: Record<string, PersistSource>;
       connectionDigests: Record<string, string>;
       connections: Map<string, MalloyConnection>;
    }> {
-      const modelPaths = pkg.getModelPaths();
-      const allGraphs: BuildGraph[] = [];
+      const allGraphs: MalloyBuildGraph[] = [];
       const allSources: Record<string, PersistSource> = {};
 
-      for (const modelPath of modelPaths) {
-         if (signal.aborted) throw new Error("Build cancelled");
+      for (const modelPath of pkg.getModelPaths()) {
+         if (signal?.aborted) throw new Error("Build cancelled");
 
          const { runtime, modelURL, importBaseURL } =
             await Model.getModelRuntime(
@@ -728,72 +1054,28 @@ export class MaterializationService {
                modelPath,
                pkg.getMalloyConfig(),
             );
+         const malloyModel = await runtime
+            .loadModel(modelURL, { importBaseURL })
+            .getModel();
 
-         const modelMaterializer = runtime.loadModel(modelURL, {
-            importBaseURL,
-         });
-         const malloyModel = await modelMaterializer.getModel();
-
-         // getBuildPlan() throws if the tag is missing, so check first to
-         // keep plain models in the same package buildable.
-         const modelTag = malloyModel.annotations.parseAsTag("!").tag;
-         if (!modelTag.has("experimental", "persistence")) {
-            logger.debug(
-               "Model has no ##! experimental.persistence tag, skipping",
-               { modelPath },
-            );
-            continue;
-         }
-
+         // getBuildPlan() returns empty graphs for models with no #@ persist
+         // sources, so non-persist models are simply skipped below.
          const buildPlan = malloyModel.getBuildPlan();
-
          for (const msg of buildPlan.tagParseLog) {
             logger.warn("Persist annotation issue", {
                modelPath,
                message: msg.message,
                severity: msg.severity,
-               code: msg.code,
             });
          }
+         if (buildPlan.graphs.length === 0) continue;
 
-         if (buildPlan.graphs.length > 0) {
-            allGraphs.push(...buildPlan.graphs);
-            for (const [sourceID, source] of Object.entries(
-               buildPlan.sources,
-            )) {
-               if (allSources[sourceID]) {
-                  logger.warn(
-                     `Duplicate sourceID "${sourceID}" from model ${modelPath}, overwriting previous definition`,
-                  );
-               }
-               allSources[sourceID] = source;
-            }
+         allGraphs.push(...buildPlan.graphs);
+         for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
+            allSources[sourceID] = source;
          }
       }
 
-      logger.info("Build plan", {
-         sourceCount: Object.keys(allSources).length,
-         graphCount: allGraphs.length,
-      });
-
-      // Fail fast if two persist sources target the same (connection, table).
-      const tableOwners = new Map<string, string>();
-      for (const [sourceID, source] of Object.entries(allSources)) {
-         const tableName =
-            source.annotations.parseAsTag("@").tag.text("name") || source.name;
-         const key = `${source.connectionName}::${tableName}`;
-         const existing = tableOwners.get(key);
-         if (existing) {
-            throw new BadRequestError(
-               `Persist target collision: sources '${existing}' and '${sourceID}' both resolve to table '${tableName}' on connection '${source.connectionName}'. Disambiguate with '#@ persist name=...'.`,
-            );
-         }
-         tableOwners.set(key, sourceID);
-      }
-
-      // Resolve only the connections this build plan actually targets;
-      // the package's MalloyConfig caches each lookup so the build phase
-      // sees the same Connection instance and avoids re-resolving.
       const connections = await resolvePackageConnections(
          pkg,
          allGraphs.map((g) => g.connectionName),
@@ -814,187 +1096,87 @@ export class MaterializationService {
       };
    }
 
-   /**
-    * Materialize a single persist source: skip if up-to-date, otherwise
-    * build via staging table (CREATE → DROP old → RENAME), then write
-    * the manifest entry. Stale entries are cleaned up by post-build GC.
-    */
-   private async buildOneSource(
-      persistSource: PersistSource,
-      manifest: Manifest,
-      connection: MalloyConnection,
+   /** Project the Malloy build plan into the trimmed v0 wire BuildPlan. */
+   private deriveBuildPlan(
+      graphs: MalloyBuildGraph[],
+      sources: Record<string, PersistSource>,
       connectionDigests: Record<string, string>,
-      forceRefresh: boolean,
-      environmentId: string,
-      packageName: string,
-      knownMaterializedTables: Set<string>,
-   ): Promise<Record<string, unknown>> {
-      const buildIdSQL = persistSource.getSQL();
-      const digest = connectionDigests[persistSource.connectionName];
-      const buildId = persistSource.makeBuildId(digest, buildIdSQL);
-
-      // Already built — mark active so it survives GC.
-      if (manifest.buildManifest.entries[buildId] && !forceRefresh) {
-         manifest.touch(buildId);
-         logger.info(`Source ${persistSource.name} up to date, skipping`, {
-            buildId,
-         });
-         return { name: persistSource.name, status: "skipped", buildId };
-      }
-
-      const buildSQL = persistSource.getSQL({
-         buildManifest: manifest.buildManifest,
-         connectionDigests,
-      });
-
-      const connectionName = persistSource.connectionName;
-      const tableName =
-         persistSource.annotations.parseAsTag("@").tag.text("name") ||
-         persistSource.name;
-      const { bareName } = splitTablePath(tableName);
-      const stagingTableName = `${tableName}${stagingSuffix(buildId)}`;
-
-      // Table names go into DDL verbatim. Malloy assumes a table name handed
-      // to it (here, via the build manifest) is already quoted for the
-      // dialect and substitutes it into generated SQL as-is; our DDL has to
-      // match that exact identifier or the CREATE and the read diverge. The
-      // model author owns quoting the `#@ persist name=...` value.
-
-      // Guard: refuse to overwrite a pre-existing table that was not
-      // created by a previous materialization build. Without this check a
-      // model author could accidentally target a table name that already
-      // holds real data (e.g. `#@ persist name=customers`), and the
-      // DROP TABLE below would silently destroy it.
-      const tableKey = manifestTableKey(connectionName, tableName);
-      if (!knownMaterializedTables.has(tableKey)) {
-         if (await tablePhysicallyExists(connection, tableName)) {
-            throw new BadRequestError(
-               `Refusing to materialize source '${persistSource.name}': ` +
-                  `target table '${tableName}' already exists on connection ` +
-                  `'${connectionName}' but was not created by a previous ` +
-                  `materialization build. Use '#@ persist name=...' to ` +
-                  `choose a different table name, or drop the existing ` +
-                  `table manually if it is no longer needed.`,
-            );
-         }
-      }
-
-      logger.info(`Building source ${persistSource.name}`, {
-         tableName,
-         connectionName,
-      });
-
-      const startTime = performance.now();
-
-      await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
-
-      // If any step after CREATE throws we must best-effort drop the
-      // staging table, else it orphans under a name that GC will never
-      // find (no manifest row is written for a failed build).
-      try {
-         await connection.runSQL(
-            `CREATE TABLE ${stagingTableName} AS (${buildSQL})`,
-         );
-         await connection.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
-         await connection.runSQL(
-            `ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`,
-         );
-      } catch (err) {
-         try {
-            await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
-         } catch (cleanupErr) {
-            logger.warn(
-               "Build: failed to clean up staging table after a failed rebuild; physical leak",
-               {
-                  stagingTableName,
-                  connectionName,
-                  cleanupError:
-                     cleanupErr instanceof Error
-                        ? cleanupErr.message
-                        : String(cleanupErr),
-               },
-            );
-         }
-         throw err;
+      sourceNames: string[] | undefined,
+   ): BuildPlan {
+      const include = sourceNames ? new Set(sourceNames) : null;
+
+      const wireGraphs: WireBuildGraph[] = graphs.map((graph) => ({
+         connectionName: graph.connectionName,
+         nodes: graph.nodes.map((level) =>
+            level.map((node) => ({
+               sourceID: node.sourceID,
+               dependsOn: flattenDependsOn(node),
+            })),
+         ),
+      }));
+
+      const wireSources: Record<string, WirePersistSourcePlan> = {};
+      for (const [sourceID, source] of Object.entries(sources)) {
+         if (include && !include.has(source.name)) continue;
+         wireSources[sourceID] = {
+            name: source.name,
+            sourceID: source.sourceID,
+            connectionName: source.connectionName,
+            dialect: source.dialectName,
+            buildId: computeBuildId(source, connectionDigests),
+            sql: source.getSQL(),
+            columns: deriveColumns(source),
+         };
       }
 
-      const duration = performance.now() - startTime;
-
-      knownMaterializedTables.add(tableKey);
-      manifest.update(buildId, { tableName });
-
-      await this.manifestService.writeEntry(
-         environmentId,
-         packageName,
-         buildId,
-         tableName,
-         persistSource.name,
-         connectionName,
-      );
-
-      logger.info(`Built source ${persistSource.name}`, {
-         tableName,
-         durationMs: Math.round(duration),
-      });
-
-      return {
-         name: persistSource.name,
-         status: "built",
-         buildId,
-         tableName,
-         durationMs: Math.round(duration),
-      };
+      return { graphs: wireGraphs, sources: wireSources };
    }
 
-   /**
-    * Post-build GC: drop physical tables + manifest rows for entries whose
-    * BuildID is no longer produced by an active persist source.
-    *
-    * `liveTables` prevents a fresh build from having its table dropped when
-    * a stale row still references the same `(connection, tableName)` pair.
-    */
-   private async runPostBuildGc(
-      manifest: Manifest,
-      environmentId: string,
-      packageName: string,
-      connections: Map<string, MalloyConnection>,
-   ): Promise<GcResult> {
-      const activeManifest = manifest.activeEntries;
-      const allDbEntries = await this.manifestService.listEntries(
-         environmentId,
-         packageName,
-      );
-
-      const liveTables = new Set<string>();
-      for (const entry of allDbEntries) {
-         if (activeManifest.entries[entry.buildId]) {
-            liveTables.add(liveTableKey(entry.connectionName, entry.tableName));
-         }
-      }
-
-      const staleEntries = allDbEntries.filter(
-         (entry) => !activeManifest.entries[entry.buildId],
-      );
+   // ==================== HELPERS ====================
 
-      const gcResult = await dropManifestEntries(staleEntries, {
-         connections,
-         manifestService: this.manifestService,
-         environmentId,
-         liveTables,
-      });
+   private recordRound(
+      round: MaterializationRound,
+      outcome: "success" | "failed" | "cancelled",
+      startedAtMs: number,
+   ): void {
+      recordMaterializationRound(round, outcome, Date.now() - startedAtMs);
+   }
 
-      if (gcResult.errors.length > 0) {
-         logger.warn("Materialization GC surfaced errors", {
-            errorCount: gcResult.errors.length,
-            droppedCount: gcResult.dropped.length,
+   private runInBackground(
+      id: string,
+      run: (signal: AbortSignal) => Promise<void>,
+   ): void {
+      const abortController = new AbortController();
+      this.runningAbortControllers.set(id, abortController);
+
+      run(abortController.signal)
+         .catch(async (err) => {
+            const message = err instanceof Error ? err.message : String(err);
+            const next = abortController.signal.aborted
+               ? "CANCELLED"
+               : "FAILED";
+            try {
+               await this.repository.updateMaterialization(id, {
+                  status: next,
+                  completedAt: new Date(),
+                  error: abortController.signal.aborted ? "Cancelled" : message,
+               });
+            } catch (transitionErr) {
+               logger.error("Failed to record materialization failure", {
+                  materializationId: id,
+                  originalError: message,
+                  transitionError:
+                     transitionErr instanceof Error
+                        ? transitionErr.message
+                        : String(transitionErr),
+               });
+            }
+         })
+         .finally(() => {
+            this.runningAbortControllers.delete(id);
          });
-      }
-
-      return gcResult;
    }
 
-   // ==================== HELPERS ====================
-
    private resolveEnvironmentId(environmentName: string): Promise<string> {
       return resolveEnvironmentId(this.repository, environmentName);
    }