@malloy-publisher/server 0.0.188 → 0.0.382-dev

package/src/service/materialized_table_gc.spec.ts

@@ -0,0 +1,383 @@
+import type { Connection } from "@malloydata/malloy";
+import { beforeEach, describe, expect, it } from "bun:test";
+import * as sinon from "sinon";
+import { ManifestEntry } from "../storage/DatabaseInterface";
+import { ManifestService } from "./manifest_service";
+import { stagingSuffix } from "./materialization_service";
+import { dropManifestEntries, liveTableKey } from "./materialized_table_gc";
+
+function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
+   return {
+      id: "entry-1",
+      projectId: "proj-1",
+      packageName: "pkg",
+      buildId: "abcdef1234567890abcdef1234567890",
+      tableName: "my_table",
+      sourceName: "my_source",
+      connectionName: "conn",
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      ...overrides,
+   };
+}
+
+interface MockConnection {
+   dialectName: string;
+   runSQL: sinon.SinonStub;
+}
+
+function makeConnection(dialectName = "duckdb"): MockConnection {
+   return {
+      dialectName,
+      runSQL: sinon.stub().resolves(),
+   };
+}
+
+interface TestCtx {
+   connections: Map<string, Connection>;
+   manifestService: sinon.SinonStubbedInstance<ManifestService>;
+   conn: MockConnection;
+}
+
+function makeCtx(dialectName = "duckdb"): TestCtx {
+   const conn = makeConnection(dialectName);
+   const connections = new Map<string, Connection>();
+   connections.set("conn", conn as unknown as Connection);
+   const manifestService = {
+      deleteEntry: sinon.stub().resolves(),
+      listEntries: sinon.stub().resolves([]),
+      getManifest: sinon.stub().resolves({ entries: {}, strict: false }),
+      writeEntry: sinon.stub().resolves(),
+      reloadManifest: sinon.stub().resolves({ entries: {}, strict: false }),
+   } as unknown as sinon.SinonStubbedInstance<ManifestService>;
+   return { connections, manifestService, conn };
+}
+
+describe("dropManifestEntries", () => {
+   let ctx: TestCtx;
+
+   beforeEach(() => {
+      ctx = makeCtx();
+   });
+
+   it("deletes the manifest row first, then drops the target and staging companion", async () => {
+      const entry = makeEntry();
+      const deleteEntry = ctx.manifestService.deleteEntry as sinon.SinonStub;
+
+      const result = await dropManifestEntries([entry], {
+         connections: ctx.connections,
+         manifestService: ctx.manifestService,
+         projectId: "proj-1",
+      });
+
+      expect(result.dropped).toHaveLength(1);
+      expect(result.errors).toHaveLength(0);
+      expect(result.dropped[0].tableName).toBe("my_table");
+      expect(result.dropped[0].stagingTableName).toBe(
+         `my_table${stagingSuffix(entry.buildId)}`,
+      );
+
+      expect(deleteEntry.calledOnceWith("proj-1", entry.id)).toBe(true);
+
+      expect(ctx.conn.runSQL.callCount).toBe(2);
+      const targetSql = ctx.conn.runSQL.getCall(0).args[0];
+      const stagingSql = ctx.conn.runSQL.getCall(1).args[0];
+      expect(targetSql).toMatch(/^DROP TABLE IF EXISTS /);
+      expect(targetSql).toContain("my_table");
+      expect(stagingSql).toMatch(/^DROP TABLE IF EXISTS /);
+      expect(stagingSql).toContain(`my_table${stagingSuffix(entry.buildId)}`);
+
+      // Ordering: manifest row must be deleted before any physical DROP, so
+      // we never leave a manifest row pointing at a missing table.
+      expect(
+         deleteEntry.getCall(0).calledBefore(ctx.conn.runSQL.getCall(0)),
+      ).toBe(true);
+   });
+
+   it("records an error and skips physical DROPs when manifest delete fails", async () => {
+      const entry = makeEntry();
+      const deleteEntry = ctx.manifestService.deleteEntry as sinon.SinonStub;
+      deleteEntry.rejects(new Error("manifest store down"));
+
+      const result = await dropManifestEntries([entry], {
+         connections: ctx.connections,
+         manifestService: ctx.manifestService,
+         projectId: "proj-1",
+      });
+
+      expect(result.dropped).toHaveLength(0);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].error).toContain("manifest store down");
+      // No physical DROP must fire — we still need the table so the source
+      // remains queryable while GC retries on a later pass.
+      expect(ctx.conn.runSQL.called).toBe(false);
+   });
+
+   it("still counts as dropped when the target DROP fails after manifest delete (orphan warned, not retried)", async () => {
+      const entry = makeEntry();
+      ctx.conn.runSQL
+         .onFirstCall()
+         .rejects(new Error("permission denied on relation my_table"));
+
+      const result = await dropManifestEntries([entry], {
+         connections: ctx.connections,
+         manifestService: ctx.manifestService,
+         projectId: "proj-1",
+      });
+
+      // Manifest row is gone → entry is authoritatively dropped. The
+      // physical target becomes an orphan (logged as a warning); we do NOT
+      // push to errors because there is nothing the caller can retry —
+      // the manifest row no longer exists to drive another GC pass.
+      expect(result.dropped).toHaveLength(1);
+      expect(result.errors).toHaveLength(0);
+      expect(
+         (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
+      ).toBe(true);
+      // Staging DROP still attempted for completeness.
+      expect(ctx.conn.runSQL.callCount).toBe(2);
+   });
+
+   it("still counts as dropped when only the staging DROP fails (best-effort)", async () => {
+      const entry = makeEntry();
+      ctx.conn.runSQL.onFirstCall().resolves();
+      ctx.conn.runSQL.onSecondCall().rejects(new Error("staging locked"));
+
+      const result = await dropManifestEntries([entry], {
+         connections: ctx.connections,
+         manifestService: ctx.manifestService,
+         projectId: "proj-1",
+      });
+
+      expect(result.dropped).toHaveLength(1);
+      expect(result.errors).toHaveLength(0);
+      expect(
+         (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
+      ).toBe(true);
+   });
+
+   it("skips DROP and records an error when the connection is missing", async () => {
+      const entry = makeEntry({ connectionName: "ghost" });
+
+      const result = await dropManifestEntries([entry], {
+         connections: ctx.connections,
+         manifestService: ctx.manifestService,
+         projectId: "proj-1",
+      });
+
+      expect(result.dropped).toHaveLength(0);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].error).toMatch(/Connection 'ghost'/);
+      expect(ctx.conn.runSQL.called).toBe(false);
+      expect((ctx.manifestService.deleteEntry as sinon.SinonStub).called).toBe(
+         false,
+      );
+   });
+
+   it("skips DROP and records an error when the dialect is unknown", async () => {
+      const ctx2 = makeCtx("martian-sql");
+      const entry = makeEntry();
+
+      const result = await dropManifestEntries([entry], {
+         connections: ctx2.connections,
+         manifestService: ctx2.manifestService,
+         projectId: "proj-1",
+      });
+
+      expect(result.dropped).toHaveLength(0);
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].error).toMatch(/martian-sql/);
+      expect(ctx2.conn.runSQL.called).toBe(false);
+   });
+
+   it("dryRun lists what would drop without issuing SQL or deleting rows", async () => {
+      const entries = [
+         makeEntry(),
+         makeEntry({ id: "entry-2", tableName: "schema.other" }),
+      ];
+
+      const result = await dropManifestEntries(entries, {
+         connections: ctx.connections,
+         manifestService: ctx.manifestService,
+         projectId: "proj-1",
+         dryRun: true,
+      });
+
+      expect(result.dropped).toHaveLength(2);
+      expect(result.errors).toHaveLength(0);
+      expect(ctx.conn.runSQL.called).toBe(false);
+      expect((ctx.manifestService.deleteEntry as sinon.SinonStub).called).toBe(
+         false,
+      );
+   });
+
+   it("isolates per-entry failures — other entries still drop", async () => {
+      const good = makeEntry({ id: "good", tableName: "good_table" });
+      const bad = makeEntry({
+         id: "bad",
+         tableName: "bad_table",
+         connectionName: "ghost",
+      });
+
+      const result = await dropManifestEntries([bad, good], {
+         connections: ctx.connections,
+         manifestService: ctx.manifestService,
+         projectId: "proj-1",
+      });
+
+      expect(result.dropped).toHaveLength(1);
+      expect(result.dropped[0].tableName).toBe("good_table");
+      expect(result.errors).toHaveLength(1);
+      expect(result.errors[0].tableName).toBe("bad_table");
+   });
+
+   // When a source is rebuilt with new SQL it gets a new BuildID but keeps
+   // the same tableName. Without `liveTables`, GC would drop the stale
+   // (old BuildID) manifest row and then issue `DROP TABLE IF EXISTS
+   // <tableName>` — obliterating the fresh physical target produced by
+   // the rebuild. `liveTables` short-circuits the physical DROP when
+   // another active entry still claims the same (connection, tableName).
+   describe("liveTables (rebuild safety)", () => {
+      it("skips the target DROP when another active entry claims the same (connection, tableName)", async () => {
+         const staleEntry = makeEntry({
+            id: "stale",
+            buildId: "old_buildid_000000000000000000000",
+            tableName: "my_table",
+            connectionName: "conn",
+         });
+         const live = new Set<string>([liveTableKey("conn", "my_table")]);
+
+         const result = await dropManifestEntries([staleEntry], {
+            connections: ctx.connections,
+            manifestService: ctx.manifestService,
+            projectId: "proj-1",
+            liveTables: live,
+         });
+
+         expect(result.dropped).toHaveLength(1);
+         expect(result.dropped[0].targetDropSkipped).toBe(true);
+         expect(result.errors).toHaveLength(0);
+
+         expect(
+            (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
+         ).toBe(true);
+
+         // The target DROP must NOT fire; only the staging best-effort drop
+         // should go through.
+         expect(ctx.conn.runSQL.callCount).toBe(1);
+         const only = ctx.conn.runSQL.getCall(0).args[0];
+         expect(only).toMatch(/^DROP TABLE IF EXISTS /);
+         expect(only).toContain(`my_table${stagingSuffix(staleEntry.buildId)}`);
+         expect(only).not.toMatch(/my_table\b(?!_)/);
+      });
+
+      it("still drops the target when liveTables claims a different connection", async () => {
+         const staleEntry = makeEntry({ tableName: "my_table" });
+         // Different connection with the same tableName is not the same
+         // physical table; the DROP must proceed.
+         const live = new Set<string>([liveTableKey("other_conn", "my_table")]);
+
+         const result = await dropManifestEntries([staleEntry], {
+            connections: ctx.connections,
+            manifestService: ctx.manifestService,
+            projectId: "proj-1",
+            liveTables: live,
+         });
+
+         expect(result.dropped).toHaveLength(1);
+         expect(result.dropped[0].targetDropSkipped).toBeUndefined();
+         expect(ctx.conn.runSQL.callCount).toBe(2);
+      });
+
+      it("dryRun reports would-skip via targetDropSkipped", async () => {
+         const staleEntry = makeEntry();
+         const live = new Set<string>([liveTableKey("conn", "my_table")]);
+
+         const result = await dropManifestEntries([staleEntry], {
+            connections: ctx.connections,
+            manifestService: ctx.manifestService,
+            projectId: "proj-1",
+            dryRun: true,
+            liveTables: live,
+         });
+
+         expect(result.dropped).toHaveLength(1);
+         expect(result.dropped[0].targetDropSkipped).toBe(true);
+         expect(ctx.conn.runSQL.called).toBe(false);
+         expect(
+            (ctx.manifestService.deleteEntry as sinon.SinonStub).called,
+         ).toBe(false);
+      });
+   });
+
+   describe("forceDeleteRowOnMissingConnection (teardown)", () => {
+      it("deletes the manifest row without any DROP when the connection is gone", async () => {
+         const entry = makeEntry({ connectionName: "ghost" });
+
+         const result = await dropManifestEntries([entry], {
+            connections: ctx.connections,
+            manifestService: ctx.manifestService,
+            projectId: "proj-1",
+            forceDeleteRowOnMissingConnection: true,
+         });
+
+         expect(result.dropped).toHaveLength(1);
+         expect(result.dropped[0].targetDropSkipped).toBe(true);
+         expect(result.errors).toHaveLength(0);
+         expect(
+            (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
+         ).toBe(true);
+         expect(ctx.conn.runSQL.called).toBe(false);
+      });
+
+      it("records an error if the manifest-row delete itself fails", async () => {
+         const entry = makeEntry({ connectionName: "ghost" });
+         (ctx.manifestService.deleteEntry as sinon.SinonStub).rejects(
+            new Error("manifest store down"),
+         );
+
+         const result = await dropManifestEntries([entry], {
+            connections: ctx.connections,
+            manifestService: ctx.manifestService,
+            projectId: "proj-1",
+            forceDeleteRowOnMissingConnection: true,
+         });
+
+         expect(result.dropped).toHaveLength(0);
+         expect(result.errors).toHaveLength(1);
+         expect(result.errors[0].error).toContain("manifest store down");
+      });
+
+      it("falls back to the error path in dryRun (keeps dryRun side-effect-free)", async () => {
+         const entry = makeEntry({ connectionName: "ghost" });
+
+         const result = await dropManifestEntries([entry], {
+            connections: ctx.connections,
+            manifestService: ctx.manifestService,
+            projectId: "proj-1",
+            forceDeleteRowOnMissingConnection: true,
+            dryRun: true,
+         });
+
+         // In dryRun the row is NOT deleted; surface the same error signal
+         // so operators see the connection gap they'd hit on a live run.
+         expect(result.dropped).toHaveLength(0);
+         expect(result.errors).toHaveLength(1);
+         expect(result.errors[0].error).toMatch(/ghost/);
+         expect(
+            (ctx.manifestService.deleteEntry as sinon.SinonStub).called,
+         ).toBe(false);
+      });
+   });
+});
+
+describe("stagingSuffix", () => {
+   it("uses a fixed-length prefix of the BuildID", () => {
+      const suffix = stagingSuffix(
+         "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+      );
+      // 12 hex chars + the "_" prefix = 13 chars total.
+      expect(suffix).toBe("_0123456789ab");
+      expect(suffix.length).toBe(13);
+   });
+});

package/src/service/materialized_table_gc.ts

@@ -0,0 +1,279 @@
+import type { Connection } from "@malloydata/malloy";
+import {
+   DatabricksDialect,
+   DuckDBDialect,
+   MySQLDialect,
+   PostgresDialect,
+   SnowflakeDialect,
+   StandardSQLDialect,
+   TrinoDialect,
+} from "@malloydata/malloy";
+import { logger } from "../logger";
+import { ManifestEntry } from "../storage/DatabaseInterface";
+import { ManifestService } from "./manifest_service";
+import { stagingSuffix } from "./materialization_service";
+import { type Quoter, quoteTablePath } from "./quoting";
+
+/**
+ * Registry of built-in dialects keyed by `Connection.dialectName`. Malloy's
+ * internal `getDialect` helper isn't part of the package's public exports,
+ * so we assemble our own registry from the exported dialect classes.
+ *
+ * Note: `presto` (extends `TrinoDialect`) is not re-exported publicly and
+ * is niche enough to omit; if/when it ships as a publisher connection type,
+ * add it here.
+ */
+const DIALECTS: Readonly<Record<string, Quoter>> = Object.freeze({
+   duckdb: new DuckDBDialect(),
+   standardsql: new StandardSQLDialect(),
+   trino: new TrinoDialect(),
+   postgres: new PostgresDialect(),
+   snowflake: new SnowflakeDialect(),
+   mysql: new MySQLDialect(),
+   databricks: new DatabricksDialect(),
+});
+
+/** Build a stable key for a `(connectionName, tableName)` tuple. */
+export function liveTableKey(
+   connectionName: string,
+   tableName: string,
+): string {
+   return `${connectionName}::${tableName}`;
+}
+
+/** One manifest entry that was successfully GC'd. */
+export interface GcDropped {
+   buildId: string;
+   tableName: string;
+   connectionName: string;
+   stagingTableName: string;
+   /**
+    * True when the physical target was left in place because another
+    * active manifest entry still claims this `(connection, tableName)`
+    * pair. The stale manifest row was still deleted — only the DROP was
+    * suppressed.
+    */
+   targetDropSkipped?: boolean;
+}
+
+/** One manifest entry that could not be GC'd. The row is left in place. */
+export interface GcError {
+   buildId: string;
+   tableName: string;
+   connectionName: string;
+   error: string;
+}
+
+export interface GcResult {
+   dropped: GcDropped[];
+   errors: GcError[];
+}
+
+export interface GcContext {
+   connections: Map<string, Connection>;
+   manifestService: ManifestService;
+   projectId: string;
+   dryRun?: boolean;
+   /**
+    * Set of `liveTableKey(connectionName, tableName)` tuples that some
+    * *active* manifest entry still claims. When a stale entry's target
+    * is in this set the physical DROP is skipped (the fresh row needs
+    * the table); the stale manifest row is still deleted and the
+    * staging companion is still best-effort dropped.
+    */
+   liveTables?: ReadonlySet<string>;
+   /**
+    * When true, delete the manifest row even if the row's
+    * `connectionName` is no longer registered with the package. Used by
+    * `teardownPackage` where retaining rows that point at a vanished
+    * connection just makes them un-GC-able.
+    */
+   forceDeleteRowOnMissingConnection?: boolean;
+}
+
+/**
+ * Process a single manifest entry for GC. Returns either a `dropped` or
+ * `error` result (never both).
+ *
+ * Ordering invariant: the manifest row is deleted **before** any physical
+ * DROP so a failed DROP leaves an orphaned table (recoverable) rather
+ * than an orphaned manifest row (causes skipped builds + query failures).
+ */
+async function processOneEntry(
+   entry: ManifestEntry,
+   ctx: GcContext,
+   liveTables: ReadonlySet<string>,
+): Promise<{ dropped?: GcDropped; error?: GcError }> {
+   const stagingTableName = `${entry.tableName}${stagingSuffix(entry.buildId)}`;
+   const targetIsLive = liveTables.has(
+      liveTableKey(entry.connectionName, entry.tableName),
+   );
+
+   const connection = ctx.connections.get(entry.connectionName);
+
+   // ── Missing connection ────────────────────────────────────────
+   if (!connection) {
+      if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
+         try {
+            await ctx.manifestService.deleteEntry(ctx.projectId, entry.id);
+            logger.warn(
+               "GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned",
+               {
+                  manifestEntryId: entry.id,
+                  tableName: entry.tableName,
+                  connectionName: entry.connectionName,
+               },
+            );
+            return {
+               dropped: {
+                  buildId: entry.buildId,
+                  tableName: entry.tableName,
+                  connectionName: entry.connectionName,
+                  stagingTableName,
+                  targetDropSkipped: true,
+               },
+            };
+         } catch (err) {
+            return {
+               error: {
+                  buildId: entry.buildId,
+                  tableName: entry.tableName,
+                  connectionName: entry.connectionName,
+                  error: err instanceof Error ? err.message : String(err),
+               },
+            };
+         }
+      }
+      return {
+         error: {
+            buildId: entry.buildId,
+            tableName: entry.tableName,
+            connectionName: entry.connectionName,
+            error: `Connection '${entry.connectionName}' is not available`,
+         },
+      };
+   }
+
+   // ── Unknown dialect ───────────────────────────────────────────
+   const dialect = DIALECTS[connection.dialectName];
+   if (!dialect) {
+      return {
+         error: {
+            buildId: entry.buildId,
+            tableName: entry.tableName,
+            connectionName: entry.connectionName,
+            error: `No dialect registered for '${connection.dialectName}'`,
+         },
+      };
+   }
+
+   // ── Dry run ───────────────────────────────────────────────────
+   if (ctx.dryRun) {
+      return {
+         dropped: {
+            buildId: entry.buildId,
+            tableName: entry.tableName,
+            connectionName: entry.connectionName,
+            stagingTableName,
+            targetDropSkipped: targetIsLive || undefined,
+         },
+      };
+   }
+
+   // ── Live run: delete manifest row first ───────────────────────
+   const quoted = (p: string) => quoteTablePath(p, dialect);
+
+   try {
+      await ctx.manifestService.deleteEntry(ctx.projectId, entry.id);
+   } catch (err) {
+      const error = err instanceof Error ? err.message : String(err);
+      logger.warn("GC: failed to delete manifest row; skipping physical drop", {
+         manifestEntryId: entry.id,
+         tableName: entry.tableName,
+         error,
+      });
+      return {
+         error: {
+            buildId: entry.buildId,
+            tableName: entry.tableName,
+            connectionName: entry.connectionName,
+            error,
+         },
+      };
+   }
+
+   // ── Drop target table (unless still live) ─────────────────────
+   if (targetIsLive) {
+      logger.info(
+         "GC: skipping target DROP; another active manifest entry claims this (connection, tableName)",
+         {
+            tableName: entry.tableName,
+            connectionName: entry.connectionName,
+            retiredBuildId: entry.buildId,
+         },
+      );
+   } else {
+      try {
+         await connection.runSQL(
+            `DROP TABLE IF EXISTS ${quoted(entry.tableName)}`,
+         );
+      } catch (err) {
+         logger.warn(
+            "GC: deleted manifest row but failed to drop materialized table (orphaned)",
+            {
+               tableName: entry.tableName,
+               connectionName: entry.connectionName,
+               error: err instanceof Error ? err.message : String(err),
+            },
+         );
+      }
+   }
+
+   // ── Best-effort drop staging companion ────────────────────────
+   try {
+      await connection.runSQL(
+         `DROP TABLE IF EXISTS ${quoted(stagingTableName)}`,
+      );
+   } catch (err) {
+      logger.warn("GC: failed to drop staging table (best-effort)", {
+         stagingTableName,
+         connectionName: entry.connectionName,
+         error: err instanceof Error ? err.message : String(err),
+      });
+   }
+
+   return {
+      dropped: {
+         buildId: entry.buildId,
+         tableName: entry.tableName,
+         connectionName: entry.connectionName,
+         stagingTableName,
+         targetDropSkipped: targetIsLive || undefined,
+      },
+   };
+}
+
+/**
+ * Idempotent manifest-row-delete + DROP TABLE for each entry.
+ *
+ * The manifest row is deleted **before** the physical tables so a failed
+ * DROP leaves an orphaned table (recoverable by the next build that
+ * targets the same tableName) rather than an orphaned manifest row
+ * (causes skipped builds + query failures).
+ */
+export async function dropManifestEntries(
+   entries: ManifestEntry[],
+   ctx: GcContext,
+): Promise<GcResult> {
+   const dropped: GcDropped[] = [];
+   const errors: GcError[] = [];
+   const liveTables = ctx.liveTables ?? new Set<string>();
+
+   for (const entry of entries) {
+      const result = await processOneEntry(entry, ctx, liveTables);
+      if (result.dropped) dropped.push(result.dropped);
+      if (result.error) errors.push(result.error);
+   }
+
+   return { dropped, errors };
+}