@malloy-publisher/server 0.0.188 → 0.0.382-dev

package/src/service/materialization_service.spec.ts

@@ -0,0 +1,648 @@
+import type { Connection } from "@malloydata/malloy";
+import { beforeEach, describe, expect, it } from "bun:test";
+import * as sinon from "sinon";
+import {
+   InvalidStateTransitionError,
+   MaterializationConflictError,
+   MaterializationNotFoundError,
+   ProjectNotFoundError,
+} from "../errors";
+import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
+import {
+   ManifestEntry,
+   Materialization,
+   MaterializationStatus,
+   ResourceRepository,
+} from "../storage/DatabaseInterface";
+import { ManifestService } from "./manifest_service";
+import {
+   manifestTableKey,
+   MaterializationService,
+   tablePhysicallyExists,
+} from "./materialization_service";
+import { ProjectStore } from "./project_store";
+
+function makeExecution(
+   overrides: Partial<Materialization> = {},
+): Materialization {
+   return {
+      id: "exec-1",
+      projectId: "proj-1",
+      packageName: "pkg",
+      status: "PENDING",
+      startedAt: null,
+      completedAt: null,
+      error: null,
+      metadata: null,
+      createdAt: new Date("2025-01-01"),
+      updatedAt: new Date("2025-01-01"),
+      ...overrides,
+   };
+}
+
+type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
+
+function createMocks() {
+   const sandbox = sinon.createSandbox();
+
+   const repository: MockRepo = {
+      listProjects: sandbox.stub(),
+      getProjectById: sandbox.stub(),
+      getProjectByName: sandbox.stub(),
+      createProject: sandbox.stub(),
+      updateProject: sandbox.stub(),
+      deleteProject: sandbox.stub(),
+      listPackages: sandbox.stub(),
+      getPackageById: sandbox.stub(),
+      getPackageByName: sandbox.stub(),
+      createPackage: sandbox.stub(),
+      updatePackage: sandbox.stub(),
+      deletePackage: sandbox.stub(),
+      listConnections: sandbox.stub(),
+      getConnectionById: sandbox.stub(),
+      getConnectionByName: sandbox.stub(),
+      createConnection: sandbox.stub(),
+      updateConnection: sandbox.stub(),
+      deleteConnection: sandbox.stub(),
+      listMaterializations: sandbox.stub(),
+      getMaterializationById: sandbox.stub(),
+      getActiveMaterialization: sandbox.stub(),
+      createMaterialization: sandbox.stub(),
+      updateMaterialization: sandbox.stub(),
+      deleteMaterialization: sandbox.stub(),
+      listManifestEntries: sandbox.stub(),
+      upsertManifestEntry: sandbox.stub(),
+      deleteManifestEntry: sandbox.stub(),
+   } as unknown as MockRepo;
+
+   const storageManager = {
+      getRepository: () => repository,
+      getManifestStore: sandbox.stub(),
+   };
+
+   const projectStore = {
+      storageManager,
+      getProject: sandbox.stub(),
+   } as unknown as ProjectStore;
+
+   const manifestService = {
+      getManifest: sandbox.stub().resolves({ entries: {}, strict: false }),
+      writeEntry: sandbox.stub().resolves(),
+      deleteEntry: sandbox.stub().resolves(),
+      reloadManifest: sandbox.stub().resolves({ entries: {}, strict: false }),
+      listEntries: sandbox.stub().resolves([]),
+   } as unknown as sinon.SinonStubbedInstance<ManifestService>;
+
+   const service = new MaterializationService(
+      projectStore,
+      manifestService as unknown as ManifestService,
+   );
+
+   // Default: resolveProjectId succeeds
+   repository.getProjectByName.resolves({
+      id: "proj-1",
+      name: "my-project",
+      path: "/test",
+      createdAt: new Date(),
+      updatedAt: new Date(),
+   });
+
+   return { sandbox, repository, projectStore, manifestService, service };
+}
+
+describe("MaterializationService", () => {
+   let ctx: ReturnType<typeof createMocks>;
+
+   beforeEach(() => {
+      ctx = createMocks();
+   });
+
+   // ==================== resolveProjectId ====================
+
+   describe("resolveProjectId (via listMaterializations)", () => {
+      it("should throw ProjectNotFoundError when project is not in DB", async () => {
+         ctx.repository.getProjectByName.resolves(null);
+
+         await expect(
+            ctx.service.listMaterializations("unknown", "pkg"),
+         ).rejects.toThrow(ProjectNotFoundError);
+      });
+   });
+
+   // ==================== BUILD QUERIES ====================
+
+   describe("listMaterializations", () => {
+      it("should list builds for a package", async () => {
+         const builds = [makeExecution(), makeExecution({ id: "exec-2" })];
+         ctx.repository.listMaterializations.resolves(builds);
+
+         const result = await ctx.service.listMaterializations(
+            "my-project",
+            "pkg",
+         );
+
+         expect(result).toEqual(builds);
+      });
+   });
+
+   describe("getMaterialization", () => {
+      it("should return a specific build", async () => {
+         const exec = makeExecution();
+         ctx.repository.getMaterializationById.resolves(exec);
+
+         const result = await ctx.service.getMaterialization(
+            "my-project",
+            "pkg",
+            "exec-1",
+         );
+
+         expect(result).toEqual(exec);
+      });
+
+      it("should throw when build not found", async () => {
+         ctx.repository.getMaterializationById.resolves(null);
+
+         await expect(
+            ctx.service.getMaterialization("my-project", "pkg", "missing"),
+         ).rejects.toThrow(MaterializationNotFoundError);
+      });
+
+      it("should throw when build belongs to a different package", async () => {
+         ctx.repository.getMaterializationById.resolves(
+            makeExecution({ packageName: "other-pkg" }),
+         );
+
+         await expect(
+            ctx.service.getMaterialization("my-project", "pkg", "exec-1"),
+         ).rejects.toThrow(MaterializationNotFoundError);
+      });
+   });
+
+   // ==================== STATE MACHINE ====================
+
+   describe("state transitions", () => {
+      const validTransitions: [MaterializationStatus, MaterializationStatus][] =
+         [
+            ["PENDING", "RUNNING"],
+            ["PENDING", "CANCELLED"],
+            ["RUNNING", "SUCCESS"],
+            ["RUNNING", "FAILED"],
+            ["RUNNING", "CANCELLED"],
+         ];
+
+      const invalidTransitions: [
+         MaterializationStatus,
+         MaterializationStatus,
+      ][] = [
+         ["PENDING", "SUCCESS"],
+         ["PENDING", "FAILED"],
+         ["RUNNING", "PENDING"],
+         ["SUCCESS", "RUNNING"],
+         ["SUCCESS", "FAILED"],
+         ["FAILED", "RUNNING"],
+         ["FAILED", "SUCCESS"],
+         ["CANCELLED", "RUNNING"],
+         ["CANCELLED", "SUCCESS"],
+      ];
+
+      for (const [from, to] of validTransitions) {
+         it(`should allow ${from} -> ${to}`, async () => {
+            const exec = makeExecution({ status: from });
+            ctx.repository.getMaterializationById.resolves(exec);
+            ctx.repository.updateMaterialization.resolves(
+               makeExecution({ status: to }),
+            );
+
+            // Trigger via stopMaterialization for RUNNING->CANCELLED (orphaned path)
+            if (from === "RUNNING" && to === "CANCELLED") {
+               const result = await ctx.service.stopMaterialization(
+                  "my-project",
+                  "pkg",
+                  exec.id,
+               );
+               expect(result).not.toBeNull();
+            }
+         });
+      }
+
+      for (const [from, to] of invalidTransitions) {
+         it(`should reject ${from} -> ${to}`, async () => {
+            const exec = makeExecution({ status: from });
+            ctx.repository.getMaterializationById.resolves(exec);
+
+            await expect(
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               (ctx.service as any).transitionExecution(exec.id, to),
+            ).rejects.toThrow(InvalidStateTransitionError);
+         });
+      }
+   });
+
+   // ==================== CREATE / START / STOP ====================
+
+   describe("createMaterialization", () => {
+      it("should create a PENDING build", async () => {
+         (ctx.projectStore.getProject as sinon.SinonStub).resolves({
+            getPackage: sinon.stub().resolves({}),
+         });
+         ctx.repository.getActiveMaterialization.resolves(null);
+         const pending = makeExecution({
+            status: "PENDING",
+            metadata: { forceRefresh: false, autoLoadManifest: true },
+         });
+         ctx.repository.createMaterialization.resolves(pending);
+
+         const result = await ctx.service.createMaterialization(
+            "my-project",
+            "pkg",
+            {
+               autoLoadManifest: true,
+            },
+         );
+
+         expect(result.status).toBe("PENDING");
+         expect(
+            (result.metadata as Record<string, unknown>)?.autoLoadManifest,
+         ).toBe(true);
+         // Options persist on the INSERT itself; no follow-up update.
+         expect(ctx.repository.createMaterialization.calledOnce).toBe(true);
+         expect(ctx.repository.createMaterialization.firstCall.args).toEqual([
+            "proj-1",
+            "pkg",
+            "PENDING",
+            { forceRefresh: false, autoLoadManifest: true },
+         ]);
+         expect(ctx.repository.updateMaterialization.called).toBe(false);
+      });
+
+      it("should reject creation when an active materialization exists", async () => {
+         (ctx.projectStore.getProject as sinon.SinonStub).resolves({
+            getPackage: sinon.stub().resolves({}),
+         });
+         ctx.repository.getActiveMaterialization.resolves(
+            makeExecution({ id: "existing", status: "RUNNING" }),
+         );
+
+         await expect(
+            ctx.service.createMaterialization("my-project", "pkg"),
+         ).rejects.toThrow(MaterializationConflictError);
+      });
+
+      it("should translate DuplicateActiveMaterializationError from a lost race", async () => {
+         (ctx.projectStore.getProject as sinon.SinonStub).resolves({
+            getPackage: sinon.stub().resolves({}),
+         });
+         // The pre-check finds nothing (race is still possible), but the
+         // atomic insert loses to a concurrent create and the repository
+         // raises DuplicateActiveMaterializationError.
+         ctx.repository.getActiveMaterialization
+            .onFirstCall()
+            .resolves(null)
+            .onSecondCall()
+            .resolves(makeExecution({ id: "winner", status: "PENDING" }));
+         ctx.repository.createMaterialization.rejects(
+            new DuplicateActiveMaterializationError("proj-1", "pkg"),
+         );
+
+         await expect(
+            ctx.service.createMaterialization("my-project", "pkg"),
+         ).rejects.toThrow(/winner/);
+      });
+   });
+
+   describe("startMaterialization", () => {
+      it("should transition PENDING to RUNNING", async () => {
+         const pending = makeExecution({
+            status: "PENDING",
+            metadata: { forceRefresh: false, autoLoadManifest: false },
+         });
+         const running = makeExecution({ status: "RUNNING" });
+         ctx.repository.getMaterializationById.resolves(pending);
+         ctx.repository.getActiveMaterialization.resolves(null);
+         ctx.repository.updateMaterialization.resolves(running);
+
+         const result = await ctx.service.startMaterialization(
+            "my-project",
+            "pkg",
+            "exec-1",
+         );
+
+         expect(result.status).toBe("RUNNING");
+      });
+
+      it("should reject non-PENDING builds", async () => {
+         const running = makeExecution({ status: "RUNNING" });
+         ctx.repository.getMaterializationById.resolves(running);
+
+         await expect(
+            ctx.service.startMaterialization("my-project", "pkg", "exec-1"),
+         ).rejects.toThrow(InvalidStateTransitionError);
+      });
+
+      it("should throw MaterializationConflictError when another is already running", async () => {
+         const pending = makeExecution({
+            status: "PENDING",
+            metadata: { forceRefresh: false, autoLoadManifest: false },
+         });
+         ctx.repository.getMaterializationById.resolves(pending);
+         ctx.repository.getActiveMaterialization.resolves(
+            makeExecution({ id: "other", status: "RUNNING" }),
+         );
+
+         await expect(
+            ctx.service.startMaterialization("my-project", "pkg", "exec-1"),
+         ).rejects.toThrow(MaterializationConflictError);
+      });
+   });
+
+   describe("stopMaterialization", () => {
+      it("should cancel a PENDING build", async () => {
+         const pending = makeExecution({ status: "PENDING" });
+         ctx.repository.getMaterializationById.resolves(pending);
+         ctx.repository.updateMaterialization.resolves(
+            makeExecution({ status: "CANCELLED" }),
+         );
+
+         const result = await ctx.service.stopMaterialization(
+            "my-project",
+            "pkg",
+            "exec-1",
+         );
+
+         expect(result.status).toBe("CANCELLED");
+      });
+
+      it("should force-cancel an orphaned RUNNING build", async () => {
+         const running = makeExecution({ id: "orphan", status: "RUNNING" });
+         ctx.repository.getMaterializationById.resolves(running);
+         ctx.repository.updateMaterialization.resolves(
+            makeExecution({ id: "orphan", status: "CANCELLED" }),
+         );
+
+         const result = await ctx.service.stopMaterialization(
+            "my-project",
+            "pkg",
+            "orphan",
+         );
+
+         expect(result.status).toBe("CANCELLED");
+      });
+
+      it("should reject stopping a terminal build", async () => {
+         const succeeded = makeExecution({ status: "SUCCESS" });
+         ctx.repository.getMaterializationById.resolves(succeeded);
+
+         await expect(
+            ctx.service.stopMaterialization("my-project", "pkg", "exec-1"),
+         ).rejects.toThrow(InvalidStateTransitionError);
+      });
+   });
+
+   // ==================== DELETE ====================
+
+   describe("deleteMaterialization", () => {
+      it("should delete a SUCCESS materialization", async () => {
+         const succeeded = makeExecution({ status: "SUCCESS" });
+         ctx.repository.getMaterializationById.resolves(succeeded);
+         ctx.repository.deleteMaterialization.resolves();
+
+         await ctx.service.deleteMaterialization("my-project", "pkg", "exec-1");
+
+         expect(ctx.repository.deleteMaterialization.calledOnce).toBe(true);
+         expect(ctx.repository.deleteMaterialization.firstCall.args[0]).toBe(
+            "exec-1",
+         );
+      });
+
+      it("should delete a FAILED materialization", async () => {
+         const failed = makeExecution({ status: "FAILED" });
+         ctx.repository.getMaterializationById.resolves(failed);
+         ctx.repository.deleteMaterialization.resolves();
+
+         await ctx.service.deleteMaterialization("my-project", "pkg", "exec-1");
+
+         expect(ctx.repository.deleteMaterialization.calledOnce).toBe(true);
+      });
+
+      it("should delete a CANCELLED materialization", async () => {
+         const cancelled = makeExecution({ status: "CANCELLED" });
+         ctx.repository.getMaterializationById.resolves(cancelled);
+         ctx.repository.deleteMaterialization.resolves();
+
+         await ctx.service.deleteMaterialization("my-project", "pkg", "exec-1");
+
+         expect(ctx.repository.deleteMaterialization.calledOnce).toBe(true);
+      });
+
+      it("should reject deleting a PENDING materialization", async () => {
+         const pending = makeExecution({ status: "PENDING" });
+         ctx.repository.getMaterializationById.resolves(pending);
+
+         await expect(
+            ctx.service.deleteMaterialization("my-project", "pkg", "exec-1"),
+         ).rejects.toThrow(InvalidStateTransitionError);
+      });
+
+      it("should reject deleting a RUNNING materialization", async () => {
+         const running = makeExecution({ status: "RUNNING" });
+         ctx.repository.getMaterializationById.resolves(running);
+
+         await expect(
+            ctx.service.deleteMaterialization("my-project", "pkg", "exec-1"),
+         ).rejects.toThrow(InvalidStateTransitionError);
+      });
+
+      it("should throw when materialization not found", async () => {
+         ctx.repository.getMaterializationById.resolves(null);
+
+         await expect(
+            ctx.service.deleteMaterialization("my-project", "pkg", "missing"),
+         ).rejects.toThrow(MaterializationNotFoundError);
+      });
+   });
+
+   // ==================== TEARDOWN ====================
+
+   describe("teardownPackage", () => {
+      it("refuses to run while a materialization is active", async () => {
+         ctx.repository.getActiveMaterialization.resolves(
+            makeExecution({ status: "RUNNING" }),
+         );
+
+         await expect(
+            ctx.service.teardownPackage("my-project", "pkg"),
+         ).rejects.toThrow(MaterializationConflictError);
+      });
+
+      it("drops every manifest entry for the package and deletes its row", async () => {
+         const runSQL = sinon.stub().resolves();
+         const connection = {
+            dialectName: "duckdb",
+            runSQL,
+         } as unknown as Connection;
+         const connections = new Map<string, Connection>([
+            ["conn", connection],
+         ]);
+         const pkg = { getConnections: () => connections };
+         (ctx.projectStore.getProject as sinon.SinonStub).resolves({
+            getPackage: sinon.stub().resolves(pkg),
+         });
+         ctx.repository.getActiveMaterialization.resolves(null);
+         const entries: ManifestEntry[] = [
+            {
+               id: "entry-1",
+               projectId: "proj-1",
+               packageName: "pkg",
+               buildId: "abcdef1234567890abcdef1234567890",
+               tableName: "table_a",
+               sourceName: "src",
+               connectionName: "conn",
+               createdAt: new Date(),
+               updatedAt: new Date(),
+            },
+            {
+               id: "entry-2",
+               projectId: "proj-1",
+               packageName: "pkg",
+               buildId: "1234567890abcdef1234567890abcdef",
+               tableName: "table_b",
+               sourceName: "src",
+               connectionName: "conn",
+               createdAt: new Date(),
+               updatedAt: new Date(),
+            },
+         ];
+         (ctx.manifestService.listEntries as sinon.SinonStub).resolves(entries);
+
+         const result = await ctx.service.teardownPackage("my-project", "pkg");
+
+         expect(result.dropped).toHaveLength(2);
+         expect(result.errors).toHaveLength(0);
+         // Each entry triggers target DROP + staging DROP = 2 calls per entry.
+         expect(runSQL.callCount).toBe(4);
+         expect(
+            (ctx.manifestService.deleteEntry as sinon.SinonStub).callCount,
+         ).toBe(2);
+      });
+
+      it("deletes rows whose connection is no longer registered (teardown force-delete)", async () => {
+         const runSQL = sinon.stub().resolves();
+         const livingConn = {
+            dialectName: "duckdb",
+            runSQL,
+         } as unknown as Connection;
+         // Only "live_conn" is registered; the manifest row below points at
+         // a vanished "ghost_conn", which used to be impossible to tear down.
+         // `teardownPackage` must force-delete the row anyway so teardown
+         // can complete.
+         const connections = new Map<string, Connection>([
+            ["live_conn", livingConn],
+         ]);
+         const pkg = { getConnections: () => connections };
+         (ctx.projectStore.getProject as sinon.SinonStub).resolves({
+            getPackage: sinon.stub().resolves(pkg),
+         });
+         ctx.repository.getActiveMaterialization.resolves(null);
+         const entries: ManifestEntry[] = [
+            {
+               id: "entry-ghost",
+               projectId: "proj-1",
+               packageName: "pkg",
+               buildId: "abcdef1234567890abcdef1234567890",
+               tableName: "table_ghost",
+               sourceName: "src",
+               connectionName: "ghost_conn",
+               createdAt: new Date(),
+               updatedAt: new Date(),
+            },
+         ];
+         (ctx.manifestService.listEntries as sinon.SinonStub).resolves(entries);
+
+         const result = await ctx.service.teardownPackage("my-project", "pkg");
+
+         expect(result.dropped).toHaveLength(1);
+         expect(result.dropped[0].targetDropSkipped).toBe(true);
+         expect(result.errors).toHaveLength(0);
+         // No DROP on the vanished connection, but the manifest row is gone.
+         expect(runSQL.called).toBe(false);
+         expect(
+            (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
+         ).toBe(true);
+      });
+
+      it("dryRun reports would-drop entries without running DROP or deleting rows", async () => {
+         const runSQL = sinon.stub().resolves();
+         const connection = {
+            dialectName: "duckdb",
+            runSQL,
+         } as unknown as Connection;
+         const connections = new Map<string, Connection>([
+            ["conn", connection],
+         ]);
+         const pkg = { getConnections: () => connections };
+         (ctx.projectStore.getProject as sinon.SinonStub).resolves({
+            getPackage: sinon.stub().resolves(pkg),
+         });
+         ctx.repository.getActiveMaterialization.resolves(null);
+         const entry: ManifestEntry = {
+            id: "entry-1",
+            projectId: "proj-1",
+            packageName: "pkg",
+            buildId: "abcdef1234567890abcdef1234567890",
+            tableName: "orphan",
+            sourceName: "src",
+            connectionName: "conn",
+            createdAt: new Date(),
+            updatedAt: new Date(),
+         };
+         (ctx.manifestService.listEntries as sinon.SinonStub).resolves([entry]);
+
+         const result = await ctx.service.teardownPackage("my-project", "pkg", {
+            dryRun: true,
+         });
+
+         expect(result.dropped).toHaveLength(1);
+         expect(result.dropped[0].tableName).toBe("orphan");
+         expect(runSQL.called).toBe(false);
+         expect(
+            (ctx.manifestService.deleteEntry as sinon.SinonStub).called,
+         ).toBe(false);
+      });
+   });
+});
+
+// ==================== HELPER UNIT TESTS ====================
+
+describe("manifestTableKey", () => {
+   it("returns connectionName::tableName", () => {
+      expect(manifestTableKey("duckdb", "orders")).toBe("duckdb::orders");
+   });
+
+   it("handles schema-qualified table names", () => {
+      expect(manifestTableKey("pg", "analytics.summary")).toBe(
+         "pg::analytics.summary",
+      );
+   });
+});
+
+describe("tablePhysicallyExists", () => {
+   it("returns true when SELECT succeeds", async () => {
+      const conn = {
+         runSQL: sinon.stub().resolves({ rows: [] }),
+      } as unknown as Connection;
+
+      expect(await tablePhysicallyExists(conn, '"my_table"')).toBe(true);
+      expect((conn.runSQL as sinon.SinonStub).calledOnce).toBe(true);
+      expect((conn.runSQL as sinon.SinonStub).firstCall.args[0]).toBe(
+         'SELECT 1 FROM "my_table" WHERE 1=0',
+      );
+   });
+
+   it("returns false when SELECT throws (table does not exist)", async () => {
+      const conn = {
+         runSQL: sinon.stub().rejects(new Error("table not found")),
+      } as unknown as Connection;
+
+      expect(await tablePhysicallyExists(conn, '"missing"')).toBe(false);
+   });
+});