@malloy-publisher/server 0.0.205 → 0.0.207

package/src/service/materialization_service.spec.ts

@@ -1,41 +1,78 @@
-import type { Connection } from "@malloydata/malloy";
+import { Manifest } from "@malloydata/malloy";
 import { beforeEach, describe, expect, it } from "bun:test";
 import * as sinon from "sinon";
 import {
+   BadRequestError,
    EnvironmentNotFoundError,
    InvalidStateTransitionError,
    MaterializationConflictError,
    MaterializationNotFoundError,
 } from "../errors";
 import {
-   ManifestEntry,
+   BuildInstruction,
+   BuildPlan,
    Materialization,
-   MaterializationStatus,
    ResourceRepository,
 } from "../storage/DatabaseInterface";
 import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
 import { EnvironmentStore } from "./environment_store";
-import { ManifestService } from "./manifest_service";
 import {
-   manifestTableKey,
    MaterializationService,
-   tablePhysicallyExists,
+   stagingSuffix,
 } from "./materialization_service";
 
-function makeExecution(
+function makeMaterialization(
    overrides: Partial<Materialization> = {},
 ): Materialization {
    return {
-      id: "exec-1",
-      environmentId: "proj-1",
+      id: "mat-1",
+      environmentId: "env-1",
       packageName: "pkg",
+      pauseBetweenPhases: false,
       status: "PENDING",
+      buildPlan: null,
+      manifest: null,
       startedAt: null,
       completedAt: null,
       error: null,
       metadata: null,
-      createdAt: new Date("2025-01-01"),
-      updatedAt: new Date("2025-01-01"),
+      createdAt: new Date("2026-01-01"),
+      updatedAt: new Date("2026-01-01"),
+      ...overrides,
+   };
+}
+
+function makeBuildPlan(overrides: Partial<BuildPlan> = {}): BuildPlan {
+   return {
+      graphs: [
+         {
+            connectionName: "duckdb",
+            nodes: [[{ sourceID: "orders@m.malloy", dependsOn: [] }]],
+         },
+      ],
+      sources: {
+         "orders@m.malloy": {
+            name: "orders",
+            sourceID: "orders@m.malloy",
+            connectionName: "duckdb",
+            dialect: "duckdb",
+            buildId: "build-orders",
+            sql: "SELECT 1",
+            columns: [],
+         },
+      },
+      ...overrides,
+   };
+}
+
+function makeInstruction(
+   overrides: Partial<BuildInstruction> = {},
+): BuildInstruction {
+   return {
+      buildId: "build-orders",
+      materializedTableId: "mt-1",
+      physicalTableName: '"orders_v1"',
+      realization: "COPY",
       ...overrides,
    };
 }
@@ -45,69 +82,40 @@ type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
 function createMocks() {
    const sandbox = sinon.createSandbox();
 
-   const repository: MockRepo = {
-      listEnvironments: sandbox.stub(),
-      getEnvironmentById: sandbox.stub(),
+   const repository = {
       getEnvironmentByName: sandbox.stub(),
-      createEnvironment: sandbox.stub(),
-      updateEnvironment: sandbox.stub(),
-      deleteEnvironment: sandbox.stub(),
-      listPackages: sandbox.stub(),
-      getPackageById: sandbox.stub(),
-      getPackageByName: sandbox.stub(),
-      createPackage: sandbox.stub(),
-      updatePackage: sandbox.stub(),
-      deletePackage: sandbox.stub(),
-      listConnections: sandbox.stub(),
-      getConnectionById: sandbox.stub(),
-      getConnectionByName: sandbox.stub(),
-      createConnection: sandbox.stub(),
-      updateConnection: sandbox.stub(),
-      deleteConnection: sandbox.stub(),
       listMaterializations: sandbox.stub(),
       getMaterializationById: sandbox.stub(),
       getActiveMaterialization: sandbox.stub(),
       createMaterialization: sandbox.stub(),
       updateMaterialization: sandbox.stub(),
       deleteMaterialization: sandbox.stub(),
-      listManifestEntries: sandbox.stub(),
-      upsertManifestEntry: sandbox.stub(),
-      deleteManifestEntry: sandbox.stub(),
    } as unknown as MockRepo;
 
-   const storageManager = {
-      getRepository: () => repository,
-      getManifestStore: sandbox.stub(),
-   };
+   const storageManager = { getRepository: () => repository };
 
    const environmentStore = {
       storageManager,
       getEnvironment: sandbox.stub(),
    } as unknown as EnvironmentStore;
 
-   const manifestService = {
-      getManifest: sandbox.stub().resolves({ entries: {}, strict: false }),
-      writeEntry: sandbox.stub().resolves(),
-      deleteEntry: sandbox.stub().resolves(),
-      reloadManifest: sandbox.stub().resolves({ entries: {}, strict: false }),
-      listEntries: sandbox.stub().resolves([]),
-   } as unknown as sinon.SinonStubbedInstance<ManifestService>;
-
-   const service = new MaterializationService(
-      environmentStore,
-      manifestService as unknown as ManifestService,
-   );
-
-   // Default: resolveEnvironmentId succeeds
+   // Default: environment resolves and yields a package.
    repository.getEnvironmentByName.resolves({
-      id: "proj-1",
-      name: "my-environment",
+      id: "env-1",
+      name: "my-env",
       path: "/test",
       createdAt: new Date(),
       updatedAt: new Date(),
    });
+   (environmentStore.getEnvironment as sinon.SinonStub).resolves({
+      getPackage: sinon.stub().resolves({}),
+      withPackageLock: async (_name: string, fn: () => Promise<unknown>) =>
+         fn(),
+   });
 
-   return { sandbox, repository, environmentStore, manifestService, service };
+   const service = new MaterializationService(environmentStore);
+
+   return { sandbox, repository, environmentStore, service };
 }
 
 describe("MaterializationService", () => {
@@ -117,572 +125,648 @@ describe("MaterializationService", () => {
       ctx = createMocks();
    });
 
-   // ==================== resolveEnvironmentId ====================
-
-   describe("resolveEnvironmentId (via listMaterializations)", () => {
-      it("should throw EnvironmentNotFoundError when environment is not in DB", async () => {
+   describe("resolveEnvironmentId", () => {
+      it("throws EnvironmentNotFoundError when the environment is missing", async () => {
          ctx.repository.getEnvironmentByName.resolves(null);
-
          await expect(
             ctx.service.listMaterializations("unknown", "pkg"),
          ).rejects.toThrow(EnvironmentNotFoundError);
       });
    });
 
-   // ==================== BUILD QUERIES ====================
-
-   describe("listMaterializations", () => {
-      it("should list builds for a package", async () => {
-         const builds = [makeExecution(), makeExecution({ id: "exec-2" })];
-         ctx.repository.listMaterializations.resolves(builds);
-
-         const result = await ctx.service.listMaterializations(
-            "my-environment",
-            "pkg",
-         );
-
-         expect(result).toEqual(builds);
+   describe("queries", () => {
+      it("lists materializations for a package", async () => {
+         const rows = [
+            makeMaterialization(),
+            makeMaterialization({ id: "m2" }),
+         ];
+         ctx.repository.listMaterializations.resolves(rows);
+         expect(
+            await ctx.service.listMaterializations("my-env", "pkg"),
+         ).toEqual(rows);
       });
-   });
-
-   describe("getMaterialization", () => {
-      it("should return a specific build", async () => {
-         const exec = makeExecution();
-         ctx.repository.getMaterializationById.resolves(exec);
 
-         const result = await ctx.service.getMaterialization(
-            "my-environment",
-            "pkg",
-            "exec-1",
-         );
-
-         expect(result).toEqual(exec);
+      it("returns a specific materialization", async () => {
+         const m = makeMaterialization();
+         ctx.repository.getMaterializationById.resolves(m);
+         expect(
+            await ctx.service.getMaterialization("my-env", "pkg", "mat-1"),
+         ).toEqual(m);
       });
 
-      it("should throw when build not found", async () => {
+      it("throws when the materialization is missing", async () => {
          ctx.repository.getMaterializationById.resolves(null);
-
          await expect(
-            ctx.service.getMaterialization("my-environment", "pkg", "missing"),
+            ctx.service.getMaterialization("my-env", "pkg", "nope"),
          ).rejects.toThrow(MaterializationNotFoundError);
       });
 
-      it("should throw when build belongs to a different package", async () => {
+      it("throws when the materialization belongs to another package", async () => {
          ctx.repository.getMaterializationById.resolves(
-            makeExecution({ packageName: "other-pkg" }),
+            makeMaterialization({ packageName: "other" }),
          );
-
          await expect(
-            ctx.service.getMaterialization("my-environment", "pkg", "exec-1"),
+            ctx.service.getMaterialization("my-env", "pkg", "mat-1"),
          ).rejects.toThrow(MaterializationNotFoundError);
       });
    });
 
-   // ==================== STATE MACHINE ====================
-
-   describe("state transitions", () => {
-      const validTransitions: [MaterializationStatus, MaterializationStatus][] =
-         [
-            ["PENDING", "RUNNING"],
-            ["PENDING", "CANCELLED"],
-            ["RUNNING", "SUCCESS"],
-            ["RUNNING", "FAILED"],
-            ["RUNNING", "CANCELLED"],
-         ];
-
-      const invalidTransitions: [
-         MaterializationStatus,
-         MaterializationStatus,
-      ][] = [
-         ["PENDING", "SUCCESS"],
-         ["PENDING", "FAILED"],
-         ["RUNNING", "PENDING"],
-         ["SUCCESS", "RUNNING"],
-         ["SUCCESS", "FAILED"],
-         ["FAILED", "RUNNING"],
-         ["FAILED", "SUCCESS"],
-         ["CANCELLED", "RUNNING"],
-         ["CANCELLED", "SUCCESS"],
-      ];
-
-      for (const [from, to] of validTransitions) {
-         it(`should allow ${from} -> ${to}`, async () => {
-            const exec = makeExecution({ status: from });
-            ctx.repository.getMaterializationById.resolves(exec);
-            ctx.repository.updateMaterialization.resolves(
-               makeExecution({ status: to }),
-            );
-
-            // Trigger via stopMaterialization for RUNNING->CANCELLED (orphaned path)
-            if (from === "RUNNING" && to === "CANCELLED") {
-               const result = await ctx.service.stopMaterialization(
-                  "my-environment",
-                  "pkg",
-                  exec.id,
-               );
-               expect(result).not.toBeNull();
-            }
-         });
-      }
-
-      for (const [from, to] of invalidTransitions) {
-         it(`should reject ${from} -> ${to}`, async () => {
-            const exec = makeExecution({ status: from });
-            ctx.repository.getMaterializationById.resolves(exec);
-
-            await expect(
-               // eslint-disable-next-line @typescript-eslint/no-explicit-any
-               (ctx.service as any).transitionExecution(exec.id, to),
-            ).rejects.toThrow(InvalidStateTransitionError);
-         });
-      }
-   });
-
-   // ==================== CREATE / START / STOP ====================
-
-   describe("createMaterialization", () => {
-      it("should create a PENDING build", async () => {
-         (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
-            getPackage: sinon.stub().resolves({}),
-         });
+   describe("createMaterialization (Round 1)", () => {
+      it("creates a PENDING record and persists options", async () => {
          ctx.repository.getActiveMaterialization.resolves(null);
-         const pending = makeExecution({
-            status: "PENDING",
-            metadata: { forceRefresh: false, autoLoadManifest: true },
-         });
+         const pending = makeMaterialization({ status: "PENDING" });
          ctx.repository.createMaterialization.resolves(pending);
 
          const result = await ctx.service.createMaterialization(
-            "my-environment",
+            "my-env",
             "pkg",
             {
-               autoLoadManifest: true,
+               forceRefresh: true,
+               sourceNames: ["orders"],
+               pauseBetweenPhases: true,
             },
          );
 
          expect(result.status).toBe("PENDING");
-         expect(
-            (result.metadata as Record<string, unknown>)?.autoLoadManifest,
-         ).toBe(true);
-         // Options persist on the INSERT itself; no follow-up update.
-         expect(ctx.repository.createMaterialization.calledOnce).toBe(true);
          expect(ctx.repository.createMaterialization.firstCall.args).toEqual([
-            "proj-1",
+            "env-1",
             "pkg",
             "PENDING",
-            { forceRefresh: false, autoLoadManifest: true },
+            {
+               forceRefresh: true,
+               sourceNames: ["orders"],
+               pauseBetweenPhases: true,
+            },
          ]);
-         expect(ctx.repository.updateMaterialization.called).toBe(false);
       });
 
-      it("should reject creation when an active materialization exists", async () => {
-         (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
-            getPackage: sinon.stub().resolves({}),
-         });
-         ctx.repository.getActiveMaterialization.resolves(
-            makeExecution({ id: "existing", status: "RUNNING" }),
+      it("defaults pauseBetweenPhases to false (auto-run) in metadata", async () => {
+         ctx.repository.getActiveMaterialization.resolves(null);
+         ctx.repository.createMaterialization.resolves(
+            makeMaterialization({ status: "PENDING" }),
+         );
+
+         await ctx.service.createMaterialization("my-env", "pkg", {});
+
+         expect(ctx.repository.createMaterialization.firstCall.args[3]).toEqual(
+            {
+               forceRefresh: false,
+               sourceNames: null,
+               pauseBetweenPhases: false,
+            },
          );
+      });
 
+      it("rejects when an active materialization already exists", async () => {
+         ctx.repository.getActiveMaterialization.resolves(
+            makeMaterialization({ id: "existing", status: "BUILD_PLAN_READY" }),
+         );
          await expect(
-            ctx.service.createMaterialization("my-environment", "pkg"),
+            ctx.service.createMaterialization("my-env", "pkg"),
          ).rejects.toThrow(MaterializationConflictError);
       });
 
-      it("should translate DuplicateActiveMaterializationError from a lost race", async () => {
-         (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
-            getPackage: sinon.stub().resolves({}),
-         });
-         // The pre-check finds nothing (race is still possible), but the
-         // atomic insert loses to a concurrent create and the repository
-         // raises DuplicateActiveMaterializationError.
+      it("translates a lost create race into a conflict", async () => {
          ctx.repository.getActiveMaterialization
             .onFirstCall()
             .resolves(null)
             .onSecondCall()
-            .resolves(makeExecution({ id: "winner", status: "PENDING" }));
+            .resolves(makeMaterialization({ id: "winner" }));
          ctx.repository.createMaterialization.rejects(
-            new DuplicateActiveMaterializationError("proj-1", "pkg"),
+            new DuplicateActiveMaterializationError("env-1", "pkg"),
          );
-
          await expect(
-            ctx.service.createMaterialization("my-environment", "pkg"),
+            ctx.service.createMaterialization("my-env", "pkg"),
          ).rejects.toThrow(/winner/);
       });
    });
 
-   describe("startMaterialization", () => {
-      it("should transition PENDING to RUNNING", async () => {
-         const pending = makeExecution({
-            status: "PENDING",
-            metadata: { forceRefresh: false, autoLoadManifest: false },
-         });
-         const running = makeExecution({ status: "RUNNING" });
-         ctx.repository.getMaterializationById.resolves(pending);
-         ctx.repository.getActiveMaterialization.resolves(null);
-         ctx.repository.updateMaterialization.resolves(running);
-
-         const result = await ctx.service.startMaterialization(
-            "my-environment",
-            "pkg",
-            "exec-1",
+   describe("buildMaterialization (Round 2)", () => {
+      it("rejects action=build on an auto-run materialization", async () => {
+         ctx.repository.getMaterializationById.resolves(
+            makeMaterialization({
+               status: "BUILD_PLAN_READY",
+               pauseBetweenPhases: false,
+               buildPlan: makeBuildPlan(),
+            }),
          );
-
-         expect(result.status).toBe("RUNNING");
-      });
-
-      it("should reject non-PENDING builds", async () => {
-         const running = makeExecution({ status: "RUNNING" });
-         ctx.repository.getMaterializationById.resolves(running);
-
          await expect(
-            ctx.service.startMaterialization("my-environment", "pkg", "exec-1"),
+            ctx.service.buildMaterialization("my-env", "pkg", "mat-1", [
+               makeInstruction(),
+            ]),
          ).rejects.toThrow(InvalidStateTransitionError);
       });
 
-      it("should throw MaterializationConflictError when another is already running", async () => {
-         const pending = makeExecution({
-            status: "PENDING",
-            metadata: { forceRefresh: false, autoLoadManifest: false },
-         });
-         ctx.repository.getMaterializationById.resolves(pending);
-         ctx.repository.getActiveMaterialization.resolves(
-            makeExecution({ id: "other", status: "RUNNING" }),
-         );
-
-         await expect(
-            ctx.service.startMaterialization("my-environment", "pkg", "exec-1"),
-         ).rejects.toThrow(MaterializationConflictError);
-      });
-   });
-
-   describe("stopMaterialization", () => {
-      it("should cancel a PENDING build", async () => {
-         const pending = makeExecution({ status: "PENDING" });
-         ctx.repository.getMaterializationById.resolves(pending);
-         ctx.repository.updateMaterialization.resolves(
-            makeExecution({ status: "CANCELLED" }),
-         );
-
-         const result = await ctx.service.stopMaterialization(
-            "my-environment",
-            "pkg",
-            "exec-1",
-         );
-
-         expect(result.status).toBe("CANCELLED");
-      });
-
-      it("should force-cancel an orphaned RUNNING build", async () => {
-         const running = makeExecution({ id: "orphan", status: "RUNNING" });
-         ctx.repository.getMaterializationById.resolves(running);
-         ctx.repository.updateMaterialization.resolves(
-            makeExecution({ id: "orphan", status: "CANCELLED" }),
-         );
-
-         const result = await ctx.service.stopMaterialization(
-            "my-environment",
-            "pkg",
-            "orphan",
+      it("rejects when not at BUILD_PLAN_READY", async () => {
+         ctx.repository.getMaterializationById.resolves(
+            makeMaterialization({
+               status: "PENDING",
+               pauseBetweenPhases: true,
+            }),
          );
-
-         expect(result.status).toBe("CANCELLED");
-      });
-
-      it("should reject stopping a terminal build", async () => {
-         const succeeded = makeExecution({ status: "SUCCESS" });
-         ctx.repository.getMaterializationById.resolves(succeeded);
-
          await expect(
-            ctx.service.stopMaterialization("my-environment", "pkg", "exec-1"),
+            ctx.service.buildMaterialization("my-env", "pkg", "mat-1", [
+               makeInstruction(),
+            ]),
          ).rejects.toThrow(InvalidStateTransitionError);
       });
-   });
-
-   // ==================== DELETE ====================
 
-   describe("deleteMaterialization", () => {
-      it("should delete a SUCCESS materialization", async () => {
-         const succeeded = makeExecution({ status: "SUCCESS" });
-         ctx.repository.getMaterializationById.resolves(succeeded);
-         ctx.repository.deleteMaterialization.resolves();
-
-         await ctx.service.deleteMaterialization(
-            "my-environment",
-            "pkg",
-            "exec-1",
-         );
-
-         expect(ctx.repository.deleteMaterialization.calledOnce).toBe(true);
-         expect(ctx.repository.deleteMaterialization.firstCall.args[0]).toBe(
-            "exec-1",
+      it("rejects instructions that reference an unknown buildId", async () => {
+         ctx.repository.getMaterializationById.resolves(
+            makeMaterialization({
+               status: "BUILD_PLAN_READY",
+               pauseBetweenPhases: true,
+               buildPlan: makeBuildPlan(),
+            }),
          );
+         await expect(
+            ctx.service.buildMaterialization("my-env", "pkg", "mat-1", [
+               makeInstruction({ buildId: "ghost" }),
+            ]),
+         ).rejects.toThrow(BadRequestError);
       });
 
-      it("should delete a FAILED materialization", async () => {
-         const failed = makeExecution({ status: "FAILED" });
-         ctx.repository.getMaterializationById.resolves(failed);
-         ctx.repository.deleteMaterialization.resolves();
-
-         await ctx.service.deleteMaterialization(
-            "my-environment",
-            "pkg",
-            "exec-1",
+      it("rejects SNAPSHOT when the connection does not support it", async () => {
+         ctx.repository.getMaterializationById.resolves(
+            makeMaterialization({
+               status: "BUILD_PLAN_READY",
+               pauseBetweenPhases: true,
+               buildPlan: makeBuildPlan(),
+            }),
          );
-
-         expect(ctx.repository.deleteMaterialization.calledOnce).toBe(true);
+         await expect(
+            ctx.service.buildMaterialization("my-env", "pkg", "mat-1", [
+               makeInstruction({ realization: "SNAPSHOT" }),
+            ]),
+         ).rejects.toThrow(BadRequestError);
       });
 
-      it("should delete a CANCELLED materialization", async () => {
-         const cancelled = makeExecution({ status: "CANCELLED" });
-         ctx.repository.getMaterializationById.resolves(cancelled);
-         ctx.repository.deleteMaterialization.resolves();
-
-         await ctx.service.deleteMaterialization(
-            "my-environment",
+      it("accepts a valid build and returns the record (202)", async () => {
+         const m = makeMaterialization({
+            status: "BUILD_PLAN_READY",
+            pauseBetweenPhases: true,
+            buildPlan: makeBuildPlan(),
+         });
+         ctx.repository.getMaterializationById.resolves(m);
+         const result = await ctx.service.buildMaterialization(
+            "my-env",
             "pkg",
-            "exec-1",
+            "mat-1",
+            [makeInstruction()],
          );
-
-         expect(ctx.repository.deleteMaterialization.calledOnce).toBe(true);
+         expect(result.id).toBe("mat-1");
       });
+   });
 
-      it("should reject deleting a PENDING materialization", async () => {
-         const pending = makeExecution({ status: "PENDING" });
-         ctx.repository.getMaterializationById.resolves(pending);
-
-         await expect(
-            ctx.service.deleteMaterialization(
-               "my-environment",
-               "pkg",
-               "exec-1",
-            ),
-         ).rejects.toThrow(InvalidStateTransitionError);
-      });
-
-      it("should reject deleting a RUNNING materialization", async () => {
-         const running = makeExecution({ status: "RUNNING" });
-         ctx.repository.getMaterializationById.resolves(running);
-
-         await expect(
-            ctx.service.deleteMaterialization(
-               "my-environment",
+   describe("stopMaterialization", () => {
+      const cancellable = [
+         "PENDING",
+         "BUILD_PLAN_READY",
+         "MANIFEST_ROWS_READY",
+      ] as const;
+
+      for (const status of cancellable) {
+         it(`cancels a ${status} materialization`, async () => {
+            ctx.repository.getMaterializationById.resolves(
+               makeMaterialization({ status }),
+            );
+            ctx.repository.updateMaterialization.resolves(
+               makeMaterialization({ status: "CANCELLED" }),
+            );
+            const result = await ctx.service.stopMaterialization(
+               "my-env",
                "pkg",
-               "exec-1",
-            ),
-         ).rejects.toThrow(InvalidStateTransitionError);
-      });
-
-      it("should throw when materialization not found", async () => {
-         ctx.repository.getMaterializationById.resolves(null);
+               "mat-1",
+            );
+            expect(result.status).toBe("CANCELLED");
+         });
+      }
 
-         await expect(
-            ctx.service.deleteMaterialization(
-               "my-environment",
-               "pkg",
-               "missing",
-            ),
-         ).rejects.toThrow(MaterializationNotFoundError);
-      });
+      const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"] as const;
+      for (const status of terminal) {
+         it(`rejects stopping a ${status} materialization`, async () => {
+            ctx.repository.getMaterializationById.resolves(
+               makeMaterialization({ status }),
+            );
+            await expect(
+               ctx.service.stopMaterialization("my-env", "pkg", "mat-1"),
+            ).rejects.toThrow(InvalidStateTransitionError);
+         });
+      }
    });
 
-   // ==================== TEARDOWN ====================
-
-   describe("teardownPackage", () => {
-      it("refuses to run while a materialization is active", async () => {
-         ctx.repository.getActiveMaterialization.resolves(
-            makeExecution({ status: "RUNNING" }),
-         );
+   describe("deleteMaterialization", () => {
+      const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"] as const;
+      for (const status of terminal) {
+         it(`deletes a ${status} materialization`, async () => {
+            ctx.repository.getMaterializationById.resolves(
+               makeMaterialization({ status }),
+            );
+            await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
+            expect(
+               ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
+            ).toBe(true);
+         });
+      }
 
-         await expect(
-            ctx.service.teardownPackage("my-environment", "pkg"),
-         ).rejects.toThrow(MaterializationConflictError);
-      });
+      const active = [
+         "PENDING",
+         "BUILD_PLAN_READY",
+         "MANIFEST_ROWS_READY",
+      ] as const;
+      for (const status of active) {
+         it(`rejects deleting a ${status} materialization`, async () => {
+            ctx.repository.getMaterializationById.resolves(
+               makeMaterialization({ status }),
+            );
+            await expect(
+               ctx.service.deleteMaterialization("my-env", "pkg", "mat-1"),
+            ).rejects.toThrow(InvalidStateTransitionError);
+            expect(ctx.repository.deleteMaterialization.called).toBe(false);
+         });
+      }
 
-      it("drops every manifest entry for the package and deletes its row", async () => {
+      it("does not drop tables by default (record-only delete)", async () => {
          const runSQL = sinon.stub().resolves();
-         const connection = {
-            dialectName: "duckdb",
-            runSQL,
-         } as unknown as Connection;
-         const pkg = {
-            getMalloyConnection: async (name: string): Promise<Connection> => {
-               if (name === "conn") return connection;
-               throw new Error(`unknown connection: ${name}`);
-            },
-         };
+         const getMalloyConnection = sinon.stub().resolves({ runSQL });
          (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
-            getPackage: sinon.stub().resolves(pkg),
+            getPackage: sinon.stub().resolves({ getMalloyConnection }),
+            withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
+               fn(),
          });
-         ctx.repository.getActiveMaterialization.resolves(null);
-         const entries: ManifestEntry[] = [
-            {
-               id: "entry-1",
-               environmentId: "proj-1",
-               packageName: "pkg",
-               buildId: "abcdef1234567890abcdef1234567890",
-               tableName: "table_a",
-               sourceName: "src",
-               connectionName: "conn",
-               createdAt: new Date(),
-               updatedAt: new Date(),
-            },
-            {
-               id: "entry-2",
-               environmentId: "proj-1",
-               packageName: "pkg",
-               buildId: "1234567890abcdef1234567890abcdef",
-               tableName: "table_b",
-               sourceName: "src",
-               connectionName: "conn",
-               createdAt: new Date(),
-               updatedAt: new Date(),
-            },
-         ];
-         (ctx.manifestService.listEntries as sinon.SinonStub).resolves(entries);
-
-         const result = await ctx.service.teardownPackage(
-            "my-environment",
-            "pkg",
+         ctx.repository.getMaterializationById.resolves(
+            makeMaterialization({
+               status: "MANIFEST_FILE_READY",
+               manifest: {
+                  builtAt: new Date().toISOString(),
+                  strict: false,
+                  entries: {
+                     b1: {
+                        buildId: "b1",
+                        physicalTableName: "orders_mz",
+                        connectionName: "duckdb",
+                     },
+                  },
+               },
+            }),
          );
 
-         expect(result.dropped).toHaveLength(2);
-         expect(result.errors).toHaveLength(0);
-         // Each entry triggers target DROP + staging DROP = 2 calls per entry.
-         expect(runSQL.callCount).toBe(4);
+         await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1");
+
+         expect(getMalloyConnection.called).toBe(false);
+         expect(runSQL.called).toBe(false);
          expect(
-            (ctx.manifestService.deleteEntry as sinon.SinonStub).callCount,
-         ).toBe(2);
+            ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
+         ).toBe(true);
       });
 
-      it("deletes rows whose connection is no longer registered (teardown force-delete)", async () => {
+      it("drops manifest tables (and staging) when dropTables is set", async () => {
          const runSQL = sinon.stub().resolves();
-         const livingConn = {
-            dialectName: "duckdb",
-            runSQL,
-         } as unknown as Connection;
-         // Only "live_conn" is registered; the manifest row below points at
-         // a vanished "ghost_conn", which used to be impossible to tear down.
-         // `teardownPackage` must force-delete the row anyway so teardown
-         // can complete.
-         const pkg = {
-            getMalloyConnection: async (name: string): Promise<Connection> => {
-               if (name === "live_conn") return livingConn;
-               throw new Error(`unknown connection: ${name}`);
-            },
-         };
+         const getMalloyConnection = sinon.stub().resolves({ runSQL });
          (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
-            getPackage: sinon.stub().resolves(pkg),
+            getPackage: sinon.stub().resolves({ getMalloyConnection }),
+            withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
+               fn(),
          });
-         ctx.repository.getActiveMaterialization.resolves(null);
-         const entries: ManifestEntry[] = [
-            {
-               id: "entry-ghost",
-               environmentId: "proj-1",
-               packageName: "pkg",
-               buildId: "abcdef1234567890abcdef1234567890",
-               tableName: "table_ghost",
-               sourceName: "src",
-               connectionName: "ghost_conn",
-               createdAt: new Date(),
-               updatedAt: new Date(),
-            },
-         ];
-         (ctx.manifestService.listEntries as sinon.SinonStub).resolves(entries);
-
-         const result = await ctx.service.teardownPackage(
-            "my-environment",
-            "pkg",
+         ctx.repository.getMaterializationById.resolves(
+            makeMaterialization({
+               status: "MANIFEST_FILE_READY",
+               manifest: {
+                  builtAt: new Date().toISOString(),
+                  strict: false,
+                  entries: {
+                     b1: {
+                        buildId: "b1",
+                        physicalTableName: "orders_mz",
+                        connectionName: "duckdb",
+                     },
+                  },
+               },
+            }),
          );
 
-         expect(result.dropped).toHaveLength(1);
-         expect(result.dropped[0].targetDropSkipped).toBe(true);
-         expect(result.errors).toHaveLength(0);
-         // No DROP on the vanished connection, but the manifest row is gone.
-         expect(runSQL.called).toBe(false);
+         await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
+            dropTables: true,
+         });
+
+         expect(getMalloyConnection.calledOnceWith("duckdb")).toBe(true);
+         const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
+         expect(dropped).toContain("DROP TABLE IF EXISTS orders_mz");
+         expect(dropped.some((s) => s.includes("orders_mz_"))).toBe(true);
          expect(
-            (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
+            ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
          ).toBe(true);
       });
 
-      it("dryRun reports would-drop entries without running DROP or deleting rows", async () => {
-         const runSQL = sinon.stub().resolves();
-         const connection = {
-            dialectName: "duckdb",
-            runSQL,
-         } as unknown as Connection;
-         const pkg = {
-            getMalloyConnection: async (name: string): Promise<Connection> => {
-               if (name === "conn") return connection;
-               throw new Error(`unknown connection: ${name}`);
-            },
-         };
+      it("still deletes the record when a table drop fails", async () => {
+         const runSQL = sinon.stub().rejects(new Error("boom"));
+         const getMalloyConnection = sinon.stub().resolves({ runSQL });
          (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
-            getPackage: sinon.stub().resolves(pkg),
+            getPackage: sinon.stub().resolves({ getMalloyConnection }),
+            withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
+               fn(),
          });
-         ctx.repository.getActiveMaterialization.resolves(null);
-         const entry: ManifestEntry = {
-            id: "entry-1",
-            environmentId: "proj-1",
-            packageName: "pkg",
-            buildId: "abcdef1234567890abcdef1234567890",
-            tableName: "orphan",
-            sourceName: "src",
-            connectionName: "conn",
-            createdAt: new Date(),
-            updatedAt: new Date(),
-         };
-         (ctx.manifestService.listEntries as sinon.SinonStub).resolves([entry]);
-
-         const result = await ctx.service.teardownPackage(
-            "my-environment",
-            "pkg",
-            {
-               dryRun: true,
-            },
+         ctx.repository.getMaterializationById.resolves(
+            makeMaterialization({
+               status: "FAILED",
+               manifest: {
+                  builtAt: new Date().toISOString(),
+                  strict: false,
+                  entries: {
+                     b1: {
+                        buildId: "b1",
+                        physicalTableName: "orders_mz",
+                        connectionName: "duckdb",
+                     },
+                  },
+               },
+            }),
          );
 
-         expect(result.dropped).toHaveLength(1);
-         expect(result.dropped[0].tableName).toBe("orphan");
-         expect(runSQL.called).toBe(false);
+         await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
+            dropTables: true,
+         });
+
          expect(
-            (ctx.manifestService.deleteEntry as sinon.SinonStub).called,
-         ).toBe(false);
+            ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
+         ).toBe(true);
       });
    });
 });
 
-// ==================== HELPER UNIT TESTS ====================
+describe("stagingSuffix", () => {
+   it("derives a short, stable suffix from the buildId", () => {
+      expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
+   });
+});
 
-describe("manifestTableKey", () => {
-   it("returns connectionName::tableName", () => {
-      expect(manifestTableKey("duckdb", "orders")).toBe("duckdb::orders");
+// Characterization (Pass 0): lock in the auto-run instruction-derivation and
+// the single-source build SQL sequence before the simplify pass moves them.
+// deriveSelfInstructions / buildOneSource are private; we exercise them via a
+// typed cast rather than the heavyweight runtime path.
+
+// A minimal stand-in for a Malloy PersistSource exposing only what the build
+// internals touch.
+function fakeSource(opts: {
+   name: string;
+   buildId: string;
+   sql?: string;
+   connectionName?: string;
+}): unknown {
+   return {
+      name: opts.name,
+      sourceID: opts.name,
+      connectionName: opts.connectionName ?? "duckdb",
+      makeBuildId: () => opts.buildId,
+      getSQL: () => opts.sql ?? "SELECT 1",
+      annotations: {
+         parseAsTag: () => ({ tag: { text: () => undefined } }),
+      },
+   };
+}
+
+describe("deriveSelfInstructions (characterization)", () => {
+   let ctx: ReturnType<typeof createMocks>;
+   beforeEach(() => {
+      ctx = createMocks();
+   });
+
+   function compiledWith(sources: Record<string, unknown>, levels: string[][]) {
+      return {
+         graphs: [
+            {
+               connectionName: "duckdb",
+               nodes: levels.map((level) =>
+                  level.map((sourceID) => ({ sourceID, dependsOn: [] })),
+               ),
+            },
+         ],
+         sources,
+         connectionDigests: { duckdb: "dig" },
+      };
+   }
+
+   it("carries forward unchanged buildIds and builds the rest (deduping repeats)", () => {
+      const compiled = compiledWith(
+         {
+            s1: fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" }),
+            s2: fakeSource({ name: "s2", buildId: "b2bbbbbbbbbbbbbb" }),
+         },
+         [["s1", "s2"], ["s2"]],
+      );
+      const priorEntries = {
+         b1aaaaaaaaaaaaaa: {
+            buildId: "b1aaaaaaaaaaaaaa",
+            physicalTableName: "s1_prev",
+            connectionName: "duckdb",
+         },
+      };
+
+      const { instructions, carried } = (
+         ctx.service as unknown as {
+            deriveSelfInstructions: (
+               c: unknown,
+               n: string[] | undefined,
+               p: unknown,
+            ) => { instructions: BuildInstruction[]; carried: unknown };
+         }
+      ).deriveSelfInstructions(compiled, undefined, priorEntries);
+
+      expect(instructions).toHaveLength(1);
+      expect(instructions[0].buildId).toBe("b2bbbbbbbbbbbbbb");
+      expect(instructions[0].physicalTableName).toBe("s2");
+      expect(instructions[0].realization).toBe("COPY");
+      expect(instructions[0].materializedTableId).toBe("local-b2bbbbbbbbbb");
+      expect(
+         (carried as Record<string, unknown>)["b1aaaaaaaaaaaaaa"],
+      ).toBeDefined();
    });
 
-   it("handles schema-qualified table names", () => {
-      expect(manifestTableKey("pg", "analytics.summary")).toBe(
-         "pg::analytics.summary",
+   it("honors the sourceNames filter (excluded sources are neither built nor carried)", () => {
+      const compiled = compiledWith(
+         {
+            s1: fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" }),
+            s2: fakeSource({ name: "s2", buildId: "b2bbbbbbbbbbbbbb" }),
+         },
+         [["s1", "s2"]],
       );
+      const { instructions, carried } = (
+         ctx.service as unknown as {
+            deriveSelfInstructions: (
+               c: unknown,
+               n: string[] | undefined,
+               p: unknown,
+            ) => { instructions: BuildInstruction[]; carried: unknown };
+         }
+      ).deriveSelfInstructions(compiled, ["s2"], {});
+
+      expect(instructions).toHaveLength(1);
+      expect(instructions[0].buildId).toBe("b2bbbbbbbbbbbbbb");
+      expect(Object.keys(carried as Record<string, unknown>)).toHaveLength(0);
    });
 });
 
-describe("tablePhysicallyExists", () => {
-   it("returns true when SELECT succeeds", async () => {
-      const conn = {
-         runSQL: sinon.stub().resolves({ rows: [] }),
-      } as unknown as Connection;
+describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
+   let ctx: ReturnType<typeof createMocks>;
+   beforeEach(() => {
+      ctx = createMocks();
+   });
+
+   it("returns entries from the most recent successful run, excluding the in-flight one", async () => {
+      const entries = {
+         b1: {
+            buildId: "b1",
+            physicalTableName: "orders_v1",
+            connectionName: "duckdb",
+         },
+      };
+      ctx.repository.listMaterializations.resolves([
+         makeMaterialization({
+            id: "in-flight",
+            status: "MANIFEST_ROWS_READY",
+         }),
+         makeMaterialization({
+            id: "old-success",
+            status: "MANIFEST_FILE_READY",
+            manifest: { builtAt: "t", strict: false, entries },
+         }),
+      ]);
+
+      const result = await (
+         ctx.service as unknown as {
+            getMostRecentManifestEntries: (
+               e: string,
+               p: string,
+               x: string,
+            ) => Promise<unknown>;
+         }
+      ).getMostRecentManifestEntries("env-1", "pkg", "in-flight");
+
+      expect(result).toEqual(entries);
+   });
+
+   it("returns {} when the only successful run is the excluded one", async () => {
+      ctx.repository.listMaterializations.resolves([
+         makeMaterialization({
+            id: "self",
+            status: "MANIFEST_FILE_READY",
+            manifest: {
+               builtAt: "t",
+               strict: false,
+               entries: { b1: { buildId: "b1", physicalTableName: "t1" } },
+            },
+         }),
+      ]);
+
+      const result = await (
+         ctx.service as unknown as {
+            getMostRecentManifestEntries: (
+               e: string,
+               p: string,
+               x: string,
+            ) => Promise<unknown>;
+         }
+      ).getMostRecentManifestEntries("env-1", "pkg", "self");
+
+      expect(result).toEqual({});
+   });
+});
+
+describe("stopMaterialization (in-flight)", () => {
+   let ctx: ReturnType<typeof createMocks>;
+   beforeEach(() => {
+      ctx = createMocks();
+   });
 
-      expect(await tablePhysicallyExists(conn, '"my_table"')).toBe(true);
-      expect((conn.runSQL as sinon.SinonStub).calledOnce).toBe(true);
-      expect((conn.runSQL as sinon.SinonStub).firstCall.args[0]).toBe(
-         'SELECT 1 FROM "my_table" WHERE 1=0',
+   it("aborts a running build cooperatively without forcing a transition", async () => {
+      ctx.repository.getMaterializationById.resolves(
+         makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
       );
+      const controller = new AbortController();
+      (
+         ctx.service as unknown as {
+            runningAbortControllers: Map<string, AbortController>;
+         }
+      ).runningAbortControllers.set("mat-1", controller);
+
+      const result = await ctx.service.stopMaterialization(
+         "my-env",
+         "pkg",
+         "mat-1",
+      );
+
+      expect(controller.signal.aborted).toBe(true);
+      // The background run records the terminal state; stop must not also write.
+      expect(ctx.repository.updateMaterialization.called).toBe(false);
+      expect(result.status).toBe("MANIFEST_ROWS_READY");
    });
+});
 
-   it("returns false when SELECT throws (table does not exist)", async () => {
-      const conn = {
-         runSQL: sinon.stub().rejects(new Error("table not found")),
-      } as unknown as Connection;
+describe("buildOneSource (characterization)", () => {
+   let ctx: ReturnType<typeof createMocks>;
+   beforeEach(() => {
+      ctx = createMocks();
+   });
 
-      expect(await tablePhysicallyExists(conn, '"missing"')).toBe(false);
+   function callBuildOneSource(
+      connection: { runSQL: sinon.SinonStub },
+      physicalTableName: string,
+   ): Promise<{ buildId: string; physicalTableName: string }> {
+      const source = fakeSource({
+         name: "orders",
+         buildId: "abcdef1234567890",
+         sql: "SELECT * FROM t",
+      });
+      const instruction: BuildInstruction = {
+         buildId: "abcdef1234567890",
+         materializedTableId: "mt-1",
+         physicalTableName,
+         realization: "COPY",
+      };
+      return (
+         ctx.service as unknown as {
+            buildOneSource: (
+               s: unknown,
+               i: BuildInstruction,
+               c: unknown,
+               d: Record<string, string>,
+               m: Manifest,
+            ) => Promise<{ buildId: string; physicalTableName: string }>;
+         }
+      ).buildOneSource(
+         source,
+         instruction,
+         connection,
+         { duckdb: "dig" },
+         new Manifest(),
+      );
+   }
+
+   it("stages, swaps, and renames in a crash-safe order", async () => {
+      const runSQL = sinon.stub().resolves();
+      const entry = await callBuildOneSource({ runSQL }, "orders_v1");
+
+      const sql = runSQL.getCalls().map((c) => c.args[0] as string);
+      expect(sql).toEqual([
+         "DROP TABLE IF EXISTS orders_v1_abcdef123456",
+         "CREATE TABLE orders_v1_abcdef123456 AS (SELECT * FROM t)",
+         "DROP TABLE IF EXISTS orders_v1",
+         "ALTER TABLE orders_v1_abcdef123456 RENAME TO orders_v1",
+      ]);
+      expect(entry.physicalTableName).toBe("orders_v1");
+      expect(entry.buildId).toBe("abcdef1234567890");
+   });
+
+   it("drops the staging table and rethrows when the build SQL fails", async () => {
+      const runSQL = sinon.stub();
+      runSQL.onCall(0).resolves(); // initial staging drop
+      runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
+      runSQL.onCall(2).resolves(); // cleanup staging drop
+      await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
+         "create boom",
+      );
+      expect(runSQL.lastCall.args[0]).toBe(
+         "DROP TABLE IF EXISTS orders_v1_abcdef123456",
+      );
    });
 });