@malloy-publisher/server 0.0.205 → 0.0.207

package/src/service/materialized_table_gc.spec.ts

@@ -1,384 +0,0 @@
-import type { Connection } from "@malloydata/malloy";
-import { beforeEach, describe, expect, it } from "bun:test";
-import * as sinon from "sinon";
-import { ManifestEntry } from "../storage/DatabaseInterface";
-import { ManifestService } from "./manifest_service";
-import { stagingSuffix } from "./materialization_service";
-import { dropManifestEntries, liveTableKey } from "./materialized_table_gc";
-
-function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
-   return {
-      id: "entry-1",
-      environmentId: "proj-1",
-      packageName: "pkg",
-      buildId: "abcdef1234567890abcdef1234567890",
-      tableName: "my_table",
-      sourceName: "my_source",
-      connectionName: "conn",
-      createdAt: new Date(),
-      updatedAt: new Date(),
-      ...overrides,
-   };
-}
-
-interface MockConnection {
-   dialectName: string;
-   runSQL: sinon.SinonStub;
-}
-
-function makeConnection(dialectName = "duckdb"): MockConnection {
-   return {
-      dialectName,
-      runSQL: sinon.stub().resolves(),
-   };
-}
-
-interface TestCtx {
-   connections: Map<string, Connection>;
-   manifestService: sinon.SinonStubbedInstance<ManifestService>;
-   conn: MockConnection;
-}
-
-function makeCtx(dialectName = "duckdb"): TestCtx {
-   const conn = makeConnection(dialectName);
-   const connections = new Map<string, Connection>();
-   connections.set("conn", conn as unknown as Connection);
-   const manifestService = {
-      deleteEntry: sinon.stub().resolves(),
-      listEntries: sinon.stub().resolves([]),
-      getManifest: sinon.stub().resolves({ entries: {}, strict: false }),
-      writeEntry: sinon.stub().resolves(),
-      reloadManifest: sinon.stub().resolves({ entries: {}, strict: false }),
-   } as unknown as sinon.SinonStubbedInstance<ManifestService>;
-   return { connections, manifestService, conn };
-}
-
-describe("dropManifestEntries", () => {
-   let ctx: TestCtx;
-
-   beforeEach(() => {
-      ctx = makeCtx();
-   });
-
-   it("deletes the manifest row first, then drops the target and staging companion", async () => {
-      const entry = makeEntry();
-      const deleteEntry = ctx.manifestService.deleteEntry as sinon.SinonStub;
-
-      const result = await dropManifestEntries([entry], {
-         connections: ctx.connections,
-         manifestService: ctx.manifestService,
-         environmentId: "proj-1",
-      });
-
-      expect(result.dropped).toHaveLength(1);
-      expect(result.errors).toHaveLength(0);
-      expect(result.dropped[0].tableName).toBe("my_table");
-      expect(result.dropped[0].stagingTableName).toBe(
-         `my_table${stagingSuffix(entry.buildId)}`,
-      );
-
-      expect(deleteEntry.calledOnceWith("proj-1", entry.id)).toBe(true);
-
-      expect(ctx.conn.runSQL.callCount).toBe(2);
-      const targetSql = ctx.conn.runSQL.getCall(0).args[0];
-      const stagingSql = ctx.conn.runSQL.getCall(1).args[0];
-      expect(targetSql).toMatch(/^DROP TABLE IF EXISTS /);
-      expect(targetSql).toContain("my_table");
-      expect(stagingSql).toMatch(/^DROP TABLE IF EXISTS /);
-      expect(stagingSql).toContain(`my_table${stagingSuffix(entry.buildId)}`);
-
-      // Ordering: manifest row must be deleted before any physical DROP, so
-      // we never leave a manifest row pointing at a missing table.
-      expect(
-         deleteEntry.getCall(0).calledBefore(ctx.conn.runSQL.getCall(0)),
-      ).toBe(true);
-   });
-
-   it("records an error and skips physical DROPs when manifest delete fails", async () => {
-      const entry = makeEntry();
-      const deleteEntry = ctx.manifestService.deleteEntry as sinon.SinonStub;
-      deleteEntry.rejects(new Error("manifest store down"));
-
-      const result = await dropManifestEntries([entry], {
-         connections: ctx.connections,
-         manifestService: ctx.manifestService,
-         environmentId: "proj-1",
-      });
-
-      expect(result.dropped).toHaveLength(0);
-      expect(result.errors).toHaveLength(1);
-      expect(result.errors[0].error).toContain("manifest store down");
-      // No physical DROP must fire — we still need the table so the source
-      // remains queryable while GC retries on a later pass.
-      expect(ctx.conn.runSQL.called).toBe(false);
-   });
-
-   it("still counts as dropped when the target DROP fails after manifest delete (orphan warned, not retried)", async () => {
-      const entry = makeEntry();
-      ctx.conn.runSQL
-         .onFirstCall()
-         .rejects(new Error("permission denied on relation my_table"));
-
-      const result = await dropManifestEntries([entry], {
-         connections: ctx.connections,
-         manifestService: ctx.manifestService,
-         environmentId: "proj-1",
-      });
-
-      // Manifest row is gone → entry is authoritatively dropped. The
-      // physical target becomes an orphan (logged as a warning); we do NOT
-      // push to errors because there is nothing the caller can retry —
-      // the manifest row no longer exists to drive another GC pass.
-      expect(result.dropped).toHaveLength(1);
-      expect(result.errors).toHaveLength(0);
-      expect(
-         (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
-      ).toBe(true);
-      // Staging DROP still attempted for completeness.
-      expect(ctx.conn.runSQL.callCount).toBe(2);
-   });
-
-   it("still counts as dropped when only the staging DROP fails (best-effort)", async () => {
-      const entry = makeEntry();
-      ctx.conn.runSQL.onFirstCall().resolves();
-      ctx.conn.runSQL.onSecondCall().rejects(new Error("staging locked"));
-
-      const result = await dropManifestEntries([entry], {
-         connections: ctx.connections,
-         manifestService: ctx.manifestService,
-         environmentId: "proj-1",
-      });
-
-      expect(result.dropped).toHaveLength(1);
-      expect(result.errors).toHaveLength(0);
-      expect(
-         (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
-      ).toBe(true);
-   });
-
-   it("skips DROP and records an error when the connection is missing", async () => {
-      const entry = makeEntry({ connectionName: "ghost" });
-
-      const result = await dropManifestEntries([entry], {
-         connections: ctx.connections,
-         manifestService: ctx.manifestService,
-         environmentId: "proj-1",
-      });
-
-      expect(result.dropped).toHaveLength(0);
-      expect(result.errors).toHaveLength(1);
-      expect(result.errors[0].error).toMatch(/Connection 'ghost'/);
-      expect(ctx.conn.runSQL.called).toBe(false);
-      expect((ctx.manifestService.deleteEntry as sinon.SinonStub).called).toBe(
-         false,
-      );
-   });
-
-   it("drops regardless of connection dialect — DDL uses table names verbatim", async () => {
-      const ctx2 = makeCtx("martian-sql");
-      const entry = makeEntry();
-
-      const result = await dropManifestEntries([entry], {
-         connections: ctx2.connections,
-         manifestService: ctx2.manifestService,
-         environmentId: "proj-1",
-      });
-
-      // GC no longer needs a per-dialect quoter: table names are used
-      // verbatim, so an unrecognized dialect is no longer a special case.
-      expect(result.dropped).toHaveLength(1);
-      expect(result.errors).toHaveLength(0);
-      expect(ctx2.conn.runSQL.callCount).toBe(2);
-   });
-
-   it("dryRun lists what would drop without issuing SQL or deleting rows", async () => {
-      const entries = [
-         makeEntry(),
-         makeEntry({ id: "entry-2", tableName: "schema.other" }),
-      ];
-
-      const result = await dropManifestEntries(entries, {
-         connections: ctx.connections,
-         manifestService: ctx.manifestService,
-         environmentId: "proj-1",
-         dryRun: true,
-      });
-
-      expect(result.dropped).toHaveLength(2);
-      expect(result.errors).toHaveLength(0);
-      expect(ctx.conn.runSQL.called).toBe(false);
-      expect((ctx.manifestService.deleteEntry as sinon.SinonStub).called).toBe(
-         false,
-      );
-   });
-
-   it("isolates per-entry failures — other entries still drop", async () => {
-      const good = makeEntry({ id: "good", tableName: "good_table" });
-      const bad = makeEntry({
-         id: "bad",
-         tableName: "bad_table",
-         connectionName: "ghost",
-      });
-
-      const result = await dropManifestEntries([bad, good], {
-         connections: ctx.connections,
-         manifestService: ctx.manifestService,
-         environmentId: "proj-1",
-      });
-
-      expect(result.dropped).toHaveLength(1);
-      expect(result.dropped[0].tableName).toBe("good_table");
-      expect(result.errors).toHaveLength(1);
-      expect(result.errors[0].tableName).toBe("bad_table");
-   });
-
-   // When a source is rebuilt with new SQL it gets a new BuildID but keeps
-   // the same tableName. Without `liveTables`, GC would drop the stale
-   // (old BuildID) manifest row and then issue `DROP TABLE IF EXISTS
-   // <tableName>` — obliterating the fresh physical target produced by
-   // the rebuild. `liveTables` short-circuits the physical DROP when
-   // another active entry still claims the same (connection, tableName).
-   describe("liveTables (rebuild safety)", () => {
-      it("skips the target DROP when another active entry claims the same (connection, tableName)", async () => {
-         const staleEntry = makeEntry({
-            id: "stale",
-            buildId: "old_buildid_000000000000000000000",
-            tableName: "my_table",
-            connectionName: "conn",
-         });
-         const live = new Set<string>([liveTableKey("conn", "my_table")]);
-
-         const result = await dropManifestEntries([staleEntry], {
-            connections: ctx.connections,
-            manifestService: ctx.manifestService,
-            environmentId: "proj-1",
-            liveTables: live,
-         });
-
-         expect(result.dropped).toHaveLength(1);
-         expect(result.dropped[0].targetDropSkipped).toBe(true);
-         expect(result.errors).toHaveLength(0);
-
-         expect(
-            (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
-         ).toBe(true);
-
-         // The target DROP must NOT fire; only the staging best-effort drop
-         // should go through.
-         expect(ctx.conn.runSQL.callCount).toBe(1);
-         const only = ctx.conn.runSQL.getCall(0).args[0];
-         expect(only).toMatch(/^DROP TABLE IF EXISTS /);
-         expect(only).toContain(`my_table${stagingSuffix(staleEntry.buildId)}`);
-         expect(only).not.toMatch(/my_table\b(?!_)/);
-      });
-
-      it("still drops the target when liveTables claims a different connection", async () => {
-         const staleEntry = makeEntry({ tableName: "my_table" });
-         // Different connection with the same tableName is not the same
-         // physical table; the DROP must proceed.
-         const live = new Set<string>([liveTableKey("other_conn", "my_table")]);
-
-         const result = await dropManifestEntries([staleEntry], {
-            connections: ctx.connections,
-            manifestService: ctx.manifestService,
-            environmentId: "proj-1",
-            liveTables: live,
-         });
-
-         expect(result.dropped).toHaveLength(1);
-         expect(result.dropped[0].targetDropSkipped).toBeUndefined();
-         expect(ctx.conn.runSQL.callCount).toBe(2);
-      });
-
-      it("dryRun reports would-skip via targetDropSkipped", async () => {
-         const staleEntry = makeEntry();
-         const live = new Set<string>([liveTableKey("conn", "my_table")]);
-
-         const result = await dropManifestEntries([staleEntry], {
-            connections: ctx.connections,
-            manifestService: ctx.manifestService,
-            environmentId: "proj-1",
-            dryRun: true,
-            liveTables: live,
-         });
-
-         expect(result.dropped).toHaveLength(1);
-         expect(result.dropped[0].targetDropSkipped).toBe(true);
-         expect(ctx.conn.runSQL.called).toBe(false);
-         expect(
-            (ctx.manifestService.deleteEntry as sinon.SinonStub).called,
-         ).toBe(false);
-      });
-   });
-
-   describe("forceDeleteRowOnMissingConnection (teardown)", () => {
-      it("deletes the manifest row without any DROP when the connection is gone", async () => {
-         const entry = makeEntry({ connectionName: "ghost" });
-
-         const result = await dropManifestEntries([entry], {
-            connections: ctx.connections,
-            manifestService: ctx.manifestService,
-            environmentId: "proj-1",
-            forceDeleteRowOnMissingConnection: true,
-         });
-
-         expect(result.dropped).toHaveLength(1);
-         expect(result.dropped[0].targetDropSkipped).toBe(true);
-         expect(result.errors).toHaveLength(0);
-         expect(
-            (ctx.manifestService.deleteEntry as sinon.SinonStub).calledOnce,
-         ).toBe(true);
-         expect(ctx.conn.runSQL.called).toBe(false);
-      });
-
-      it("records an error if the manifest-row delete itself fails", async () => {
-         const entry = makeEntry({ connectionName: "ghost" });
-         (ctx.manifestService.deleteEntry as sinon.SinonStub).rejects(
-            new Error("manifest store down"),
-         );
-
-         const result = await dropManifestEntries([entry], {
-            connections: ctx.connections,
-            manifestService: ctx.manifestService,
-            environmentId: "proj-1",
-            forceDeleteRowOnMissingConnection: true,
-         });
-
-         expect(result.dropped).toHaveLength(0);
-         expect(result.errors).toHaveLength(1);
-         expect(result.errors[0].error).toContain("manifest store down");
-      });
-
-      it("falls back to the error path in dryRun (keeps dryRun side-effect-free)", async () => {
-         const entry = makeEntry({ connectionName: "ghost" });
-
-         const result = await dropManifestEntries([entry], {
-            connections: ctx.connections,
-            manifestService: ctx.manifestService,
-            environmentId: "proj-1",
-            forceDeleteRowOnMissingConnection: true,
-            dryRun: true,
-         });
-
-         // In dryRun the row is NOT deleted; surface the same error signal
-         // so operators see the connection gap they'd hit on a live run.
-         expect(result.dropped).toHaveLength(0);
-         expect(result.errors).toHaveLength(1);
-         expect(result.errors[0].error).toMatch(/ghost/);
-         expect(
-            (ctx.manifestService.deleteEntry as sinon.SinonStub).called,
-         ).toBe(false);
-      });
-   });
-});
-
-describe("stagingSuffix", () => {
-   it("uses a fixed-length prefix of the BuildID", () => {
-      const suffix = stagingSuffix(
-         "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
-      );
-      // 12 hex chars + the "_" prefix = 13 chars total.
-      expect(suffix).toBe("_0123456789ab");
-      expect(suffix.length).toBe(13);
-   });
-});

package/src/service/materialized_table_gc.ts

@@ -1,231 +0,0 @@
-import type { Connection } from "@malloydata/malloy";
-import { logger } from "../logger";
-import { ManifestEntry } from "../storage/DatabaseInterface";
-import { ManifestService } from "./manifest_service";
-import { stagingSuffix } from "./materialization_service";
-
-/** Build a stable key for a `(connectionName, tableName)` tuple. */
-export function liveTableKey(
-   connectionName: string,
-   tableName: string,
-): string {
-   return `${connectionName}::${tableName}`;
-}
-
-/** One manifest entry that was successfully GC'd. */
-export interface GcDropped {
-   buildId: string;
-   tableName: string;
-   connectionName: string;
-   stagingTableName: string;
-   /**
-    * True when the physical target was left in place because another
-    * active manifest entry still claims this `(connection, tableName)`
-    * pair. The stale manifest row was still deleted — only the DROP was
-    * suppressed.
-    */
-   targetDropSkipped?: boolean;
-}
-
-/** One manifest entry that could not be GC'd. The row is left in place. */
-export interface GcError {
-   buildId: string;
-   tableName: string;
-   connectionName: string;
-   error: string;
-}
-
-export interface GcResult {
-   dropped: GcDropped[];
-   errors: GcError[];
-}
-
-export interface GcContext {
-   connections: Map<string, Connection>;
-   manifestService: ManifestService;
-   environmentId: string;
-   dryRun?: boolean;
-   /**
-    * Set of `liveTableKey(connectionName, tableName)` tuples that some
-    * *active* manifest entry still claims. When a stale entry's target
-    * is in this set the physical DROP is skipped (the fresh row needs
-    * the table); the stale manifest row is still deleted and the
-    * staging companion is still best-effort dropped.
-    */
-   liveTables?: ReadonlySet<string>;
-   /**
-    * When true, delete the manifest row even if the row's
-    * `connectionName` is no longer registered with the package. Used by
-    * `teardownPackage` where retaining rows that point at a vanished
-    * connection just makes them un-GC-able.
-    */
-   forceDeleteRowOnMissingConnection?: boolean;
-}
-
-/**
- * Process a single manifest entry for GC. Returns either a `dropped` or
- * `error` result (never both).
- *
- * Ordering invariant: the manifest row is deleted **before** any physical
- * DROP so a failed DROP leaves an orphaned table (recoverable) rather
- * than an orphaned manifest row (causes skipped builds + query failures).
- */
-async function processOneEntry(
-   entry: ManifestEntry,
-   ctx: GcContext,
-   liveTables: ReadonlySet<string>,
-): Promise<{ dropped?: GcDropped; error?: GcError }> {
-   const stagingTableName = `${entry.tableName}${stagingSuffix(entry.buildId)}`;
-   const targetIsLive = liveTables.has(
-      liveTableKey(entry.connectionName, entry.tableName),
-   );
-
-   const connection = ctx.connections.get(entry.connectionName);
-
-   // ── Missing connection ────────────────────────────────────────
-   if (!connection) {
-      if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
-         try {
-            await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
-            logger.warn(
-               "GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned",
-               {
-                  manifestEntryId: entry.id,
-                  tableName: entry.tableName,
-                  connectionName: entry.connectionName,
-               },
-            );
-            return {
-               dropped: {
-                  buildId: entry.buildId,
-                  tableName: entry.tableName,
-                  connectionName: entry.connectionName,
-                  stagingTableName,
-                  targetDropSkipped: true,
-               },
-            };
-         } catch (err) {
-            return {
-               error: {
-                  buildId: entry.buildId,
-                  tableName: entry.tableName,
-                  connectionName: entry.connectionName,
-                  error: err instanceof Error ? err.message : String(err),
-               },
-            };
-         }
-      }
-      return {
-         error: {
-            buildId: entry.buildId,
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            error: `Connection '${entry.connectionName}' is not available`,
-         },
-      };
-   }
-
-   // ── Dry run ───────────────────────────────────────────────────
-   if (ctx.dryRun) {
-      return {
-         dropped: {
-            buildId: entry.buildId,
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            stagingTableName,
-            targetDropSkipped: targetIsLive || undefined,
-         },
-      };
-   }
-
-   // ── Live run: delete manifest row first ───────────────────────
-   try {
-      await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
-   } catch (err) {
-      const error = err instanceof Error ? err.message : String(err);
-      logger.warn("GC: failed to delete manifest row; skipping physical drop", {
-         manifestEntryId: entry.id,
-         tableName: entry.tableName,
-         error,
-      });
-      return {
-         error: {
-            buildId: entry.buildId,
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            error,
-         },
-      };
-   }
-
-   // ── Drop target table (unless still live) ─────────────────────
-   if (targetIsLive) {
-      logger.info(
-         "GC: skipping target DROP; another active manifest entry claims this (connection, tableName)",
-         {
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            retiredBuildId: entry.buildId,
-         },
-      );
-   } else {
-      try {
-         await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
-      } catch (err) {
-         logger.warn(
-            "GC: deleted manifest row but failed to drop materialized table (orphaned)",
-            {
-               tableName: entry.tableName,
-               connectionName: entry.connectionName,
-               error: err instanceof Error ? err.message : String(err),
-            },
-         );
-      }
-   }
-
-   // ── Best-effort drop staging companion ────────────────────────
-   try {
-      await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
-   } catch (err) {
-      logger.warn("GC: failed to drop staging table (best-effort)", {
-         stagingTableName,
-         connectionName: entry.connectionName,
-         error: err instanceof Error ? err.message : String(err),
-      });
-   }
-
-   return {
-      dropped: {
-         buildId: entry.buildId,
-         tableName: entry.tableName,
-         connectionName: entry.connectionName,
-         stagingTableName,
-         targetDropSkipped: targetIsLive || undefined,
-      },
-   };
-}
-
-/**
- * Idempotent manifest-row-delete + DROP TABLE for each entry.
- *
- * The manifest row is deleted **before** the physical tables so a failed
- * DROP leaves an orphaned table (recoverable by the next build that
- * targets the same tableName) rather than an orphaned manifest row
- * (causes skipped builds + query failures).
- */
-export async function dropManifestEntries(
-   entries: ManifestEntry[],
-   ctx: GcContext,
-): Promise<GcResult> {
-   const dropped: GcDropped[] = [];
-   const errors: GcError[] = [];
-   const liveTables = ctx.liveTables ?? new Set<string>();
-
-   for (const entry of entries) {
-      const result = await processOneEntry(entry, ctx, liveTables);
-      if (result.dropped) dropped.push(result.dropped);
-      if (result.error) errors.push(result.error);
-   }
-
-   return { dropped, errors };
-}

package/src/storage/duckdb/DuckDBManifestStore.ts

@@ -1,70 +0,0 @@
-import {
-   BuildManifest,
-   ManifestEntry,
-   ManifestStore,
-   ResourceRepository,
-} from "../DatabaseInterface";
-
-/**
- * DuckDB-backed ManifestStore that delegates to the local ResourceRepository.
- *
- * In standalone mode this is the active store. Orchestrated deployments swap
- * in a DuckLake-backed implementation that reads/writes manifests through the
- * shared catalog instead of the local DuckDB build_manifests table.
- */
-export class DuckDBManifestStore implements ManifestStore {
-   constructor(private repository: ResourceRepository) {}
-
-   /**
-    * Assembles a {@link BuildManifest} by folding all manifest rows for the
-    * given package into a build-ID-keyed map. `strict: false` lets the Malloy
-    * runtime fall back to executing the underlying query when a persist
-    * reference has no manifest entry (e.g. before the first materialization).
-    */
-   async getManifest(
-      environmentId: string,
-      packageName: string,
-   ): Promise<BuildManifest> {
-      const entries = await this.repository.listManifestEntries(
-         environmentId,
-         packageName,
-      );
-      const manifest: BuildManifest = {
-         entries: {},
-         strict: false,
-      };
-      for (const entry of entries) {
-         manifest.entries[entry.buildId] = { tableName: entry.tableName };
-      }
-      return manifest;
-   }
-
-   async writeEntry(
-      environmentId: string,
-      packageName: string,
-      buildId: string,
-      tableName: string,
-      sourceName: string,
-      connectionName: string,
-   ): Promise<void> {
-      await this.repository.upsertManifestEntry({
-         environmentId,
-         packageName,
-         buildId,
-         tableName,
-         sourceName,
-         connectionName,
-      });
-   }
-
-   async deleteEntry(id: string): Promise<void> {
-      await this.repository.deleteManifestEntry(id);
-   }
-
-   async listEntries(
-      environmentId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      return this.repository.listManifestEntries(environmentId, packageName);
-   }
-}