@malloy-publisher/server 0.0.204 → 0.0.206

package/src/service/materialized_table_gc.ts

@@ -1,231 +0,0 @@
-import type { Connection } from "@malloydata/malloy";
-import { logger } from "../logger";
-import { ManifestEntry } from "../storage/DatabaseInterface";
-import { ManifestService } from "./manifest_service";
-import { stagingSuffix } from "./materialization_service";
-
-/** Build a stable key for a `(connectionName, tableName)` tuple. */
-export function liveTableKey(
-   connectionName: string,
-   tableName: string,
-): string {
-   return `${connectionName}::${tableName}`;
-}
-
-/** One manifest entry that was successfully GC'd. */
-export interface GcDropped {
-   buildId: string;
-   tableName: string;
-   connectionName: string;
-   stagingTableName: string;
-   /**
-    * True when the physical target was left in place because another
-    * active manifest entry still claims this `(connection, tableName)`
-    * pair. The stale manifest row was still deleted — only the DROP was
-    * suppressed.
-    */
-   targetDropSkipped?: boolean;
-}
-
-/** One manifest entry that could not be GC'd. The row is left in place. */
-export interface GcError {
-   buildId: string;
-   tableName: string;
-   connectionName: string;
-   error: string;
-}
-
-export interface GcResult {
-   dropped: GcDropped[];
-   errors: GcError[];
-}
-
-export interface GcContext {
-   connections: Map<string, Connection>;
-   manifestService: ManifestService;
-   environmentId: string;
-   dryRun?: boolean;
-   /**
-    * Set of `liveTableKey(connectionName, tableName)` tuples that some
-    * *active* manifest entry still claims. When a stale entry's target
-    * is in this set the physical DROP is skipped (the fresh row needs
-    * the table); the stale manifest row is still deleted and the
-    * staging companion is still best-effort dropped.
-    */
-   liveTables?: ReadonlySet<string>;
-   /**
-    * When true, delete the manifest row even if the row's
-    * `connectionName` is no longer registered with the package. Used by
-    * `teardownPackage` where retaining rows that point at a vanished
-    * connection just makes them un-GC-able.
-    */
-   forceDeleteRowOnMissingConnection?: boolean;
-}
-
-/**
- * Process a single manifest entry for GC. Returns either a `dropped` or
- * `error` result (never both).
- *
- * Ordering invariant: the manifest row is deleted **before** any physical
- * DROP so a failed DROP leaves an orphaned table (recoverable) rather
- * than an orphaned manifest row (causes skipped builds + query failures).
- */
-async function processOneEntry(
-   entry: ManifestEntry,
-   ctx: GcContext,
-   liveTables: ReadonlySet<string>,
-): Promise<{ dropped?: GcDropped; error?: GcError }> {
-   const stagingTableName = `${entry.tableName}${stagingSuffix(entry.buildId)}`;
-   const targetIsLive = liveTables.has(
-      liveTableKey(entry.connectionName, entry.tableName),
-   );
-
-   const connection = ctx.connections.get(entry.connectionName);
-
-   // ── Missing connection ────────────────────────────────────────
-   if (!connection) {
-      if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
-         try {
-            await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
-            logger.warn(
-               "GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned",
-               {
-                  manifestEntryId: entry.id,
-                  tableName: entry.tableName,
-                  connectionName: entry.connectionName,
-               },
-            );
-            return {
-               dropped: {
-                  buildId: entry.buildId,
-                  tableName: entry.tableName,
-                  connectionName: entry.connectionName,
-                  stagingTableName,
-                  targetDropSkipped: true,
-               },
-            };
-         } catch (err) {
-            return {
-               error: {
-                  buildId: entry.buildId,
-                  tableName: entry.tableName,
-                  connectionName: entry.connectionName,
-                  error: err instanceof Error ? err.message : String(err),
-               },
-            };
-         }
-      }
-      return {
-         error: {
-            buildId: entry.buildId,
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            error: `Connection '${entry.connectionName}' is not available`,
-         },
-      };
-   }
-
-   // ── Dry run ───────────────────────────────────────────────────
-   if (ctx.dryRun) {
-      return {
-         dropped: {
-            buildId: entry.buildId,
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            stagingTableName,
-            targetDropSkipped: targetIsLive || undefined,
-         },
-      };
-   }
-
-   // ── Live run: delete manifest row first ───────────────────────
-   try {
-      await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
-   } catch (err) {
-      const error = err instanceof Error ? err.message : String(err);
-      logger.warn("GC: failed to delete manifest row; skipping physical drop", {
-         manifestEntryId: entry.id,
-         tableName: entry.tableName,
-         error,
-      });
-      return {
-         error: {
-            buildId: entry.buildId,
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            error,
-         },
-      };
-   }
-
-   // ── Drop target table (unless still live) ─────────────────────
-   if (targetIsLive) {
-      logger.info(
-         "GC: skipping target DROP; another active manifest entry claims this (connection, tableName)",
-         {
-            tableName: entry.tableName,
-            connectionName: entry.connectionName,
-            retiredBuildId: entry.buildId,
-         },
-      );
-   } else {
-      try {
-         await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
-      } catch (err) {
-         logger.warn(
-            "GC: deleted manifest row but failed to drop materialized table (orphaned)",
-            {
-               tableName: entry.tableName,
-               connectionName: entry.connectionName,
-               error: err instanceof Error ? err.message : String(err),
-            },
-         );
-      }
-   }
-
-   // ── Best-effort drop staging companion ────────────────────────
-   try {
-      await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
-   } catch (err) {
-      logger.warn("GC: failed to drop staging table (best-effort)", {
-         stagingTableName,
-         connectionName: entry.connectionName,
-         error: err instanceof Error ? err.message : String(err),
-      });
-   }
-
-   return {
-      dropped: {
-         buildId: entry.buildId,
-         tableName: entry.tableName,
-         connectionName: entry.connectionName,
-         stagingTableName,
-         targetDropSkipped: targetIsLive || undefined,
-      },
-   };
-}
-
-/**
- * Idempotent manifest-row-delete + DROP TABLE for each entry.
- *
- * The manifest row is deleted **before** the physical tables so a failed
- * DROP leaves an orphaned table (recoverable by the next build that
- * targets the same tableName) rather than an orphaned manifest row
- * (causes skipped builds + query failures).
- */
-export async function dropManifestEntries(
-   entries: ManifestEntry[],
-   ctx: GcContext,
-): Promise<GcResult> {
-   const dropped: GcDropped[] = [];
-   const errors: GcError[] = [];
-   const liveTables = ctx.liveTables ?? new Set<string>();
-
-   for (const entry of entries) {
-      const result = await processOneEntry(entry, ctx, liveTables);
-      if (result.dropped) dropped.push(result.dropped);
-      if (result.error) errors.push(result.error);
-   }
-
-   return { dropped, errors };
-}

package/src/storage/duckdb/DuckDBManifestStore.ts

@@ -1,70 +0,0 @@
-import {
-   BuildManifest,
-   ManifestEntry,
-   ManifestStore,
-   ResourceRepository,
-} from "../DatabaseInterface";
-
-/**
- * DuckDB-backed ManifestStore that delegates to the local ResourceRepository.
- *
- * In standalone mode this is the active store. Orchestrated deployments swap
- * in a DuckLake-backed implementation that reads/writes manifests through the
- * shared catalog instead of the local DuckDB build_manifests table.
- */
-export class DuckDBManifestStore implements ManifestStore {
-   constructor(private repository: ResourceRepository) {}
-
-   /**
-    * Assembles a {@link BuildManifest} by folding all manifest rows for the
-    * given package into a build-ID-keyed map. `strict: false` lets the Malloy
-    * runtime fall back to executing the underlying query when a persist
-    * reference has no manifest entry (e.g. before the first materialization).
-    */
-   async getManifest(
-      environmentId: string,
-      packageName: string,
-   ): Promise<BuildManifest> {
-      const entries = await this.repository.listManifestEntries(
-         environmentId,
-         packageName,
-      );
-      const manifest: BuildManifest = {
-         entries: {},
-         strict: false,
-      };
-      for (const entry of entries) {
-         manifest.entries[entry.buildId] = { tableName: entry.tableName };
-      }
-      return manifest;
-   }
-
-   async writeEntry(
-      environmentId: string,
-      packageName: string,
-      buildId: string,
-      tableName: string,
-      sourceName: string,
-      connectionName: string,
-   ): Promise<void> {
-      await this.repository.upsertManifestEntry({
-         environmentId,
-         packageName,
-         buildId,
-         tableName,
-         sourceName,
-         connectionName,
-      });
-   }
-
-   async deleteEntry(id: string): Promise<void> {
-      await this.repository.deleteManifestEntry(id);
-   }
-
-   async listEntries(
-      environmentId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      return this.repository.listManifestEntries(environmentId, packageName);
-   }
-}

package/src/storage/duckdb/ManifestRepository.ts

@@ -1,120 +0,0 @@
-import { ManifestEntry } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-/**
- * Repository for build manifest entries stored in the `build_manifests` table.
- *
- * A build manifest records the materialized table produced by a specific build
- * of a package, linking it back to the Malloy source and connection that
- * generated it. Manifests are keyed by (environment, package, build) and enable
- * downstream consumers to discover which physical tables correspond to a given
- * package version.
- */
-export class ManifestRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
-   }
-
-   /** Returns all manifest entries for a package, most recent first. */
-   async listEntries(
-      environmentId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         "SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? ORDER BY created_at DESC",
-         [environmentId, packageName],
-      );
-      return rows.map(this.mapToEntry);
-   }
-
-   /** Looks up the manifest entry for a specific build, or null if none exists. */
-   async getEntryByBuildId(
-      environmentId: string,
-      packageName: string,
-      buildId: string,
-   ): Promise<ManifestEntry | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? AND build_id = ?",
-         [environmentId, packageName, buildId],
-      );
-      return row ? this.mapToEntry(row) : null;
-   }
-
-   /**
-    * Inserts a new manifest entry, or updates the existing one if a row with
-    * the same (environment, package, build) triple already exists. Uses INSERT ON
-    * CONFLICT to avoid the TOCTOU race of SELECT-then-INSERT/UPDATE.
-    */
-   async upsertEntry(
-      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
-   ): Promise<ManifestEntry> {
-      const id = this.generateId();
-      const now = new Date();
-      const iso = now.toISOString();
-
-      const rows = await this.db.all<Record<string, unknown>>(
-         `INSERT INTO build_manifests (id, environment_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-         ON CONFLICT (environment_id, package_name, build_id)
-         DO UPDATE SET table_name = EXCLUDED.table_name,
-                       source_name = EXCLUDED.source_name,
-                       connection_name = EXCLUDED.connection_name,
-                       updated_at = EXCLUDED.updated_at
-         RETURNING *`,
-         [
-            id,
-            entry.environmentId,
-            entry.packageName,
-            entry.buildId,
-            entry.tableName,
-            entry.sourceName,
-            entry.connectionName,
-            iso,
-            iso,
-         ],
-      );
-
-      return this.mapToEntry(rows[0]);
-   }
-
-   /** Deletes a single manifest entry by ID. */
-   async deleteEntry(id: string): Promise<void> {
-      await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
-   }
-
-   /** Removes all manifest entries belonging to an environment (used on environment deletion). */
-   async deleteEntriesByEnvironmentId(environmentId: string): Promise<void> {
-      await this.db.run(
-         "DELETE FROM build_manifests WHERE environment_id = ?",
-         [environmentId],
-      );
-   }
-
-   /** Removes all manifest entries for a specific package. */
-   async deleteEntriesByPackage(
-      environmentId: string,
-      packageName: string,
-   ): Promise<void> {
-      await this.db.run(
-         "DELETE FROM build_manifests WHERE environment_id = ? AND package_name = ?",
-         [environmentId, packageName],
-      );
-   }
-
-   /** Maps a raw DB row (snake_case columns) to a {@link ManifestEntry}. */
-   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
-      return {
-         id: row.id as string,
-         environmentId: row.environment_id as string,
-         packageName: row.package_name as string,
-         buildId: row.build_id as string,
-         tableName: row.table_name as string,
-         sourceName: row.source_name as string,
-         connectionName: row.connection_name as string,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}

package/src/storage/duckdb/manifest_store.spec.ts

@@ -1,133 +0,0 @@
-import { beforeEach, describe, expect, it } from "bun:test";
-import * as sinon from "sinon";
-import { ManifestEntry, ResourceRepository } from "../DatabaseInterface";
-import { DuckDBManifestStore } from "./DuckDBManifestStore";
-
-function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
-   return {
-      id: "entry-1",
-      environmentId: "proj-1",
-      packageName: "pkg",
-      buildId: "build-abc",
-      tableName: "my_table",
-      sourceName: "my_source",
-      connectionName: "duckdb",
-      createdAt: new Date("2026-04-03"),
-      updatedAt: new Date("2026-04-03"),
-      ...overrides,
-   };
-}
-
-function createMocks() {
-   const sandbox = sinon.createSandbox();
-
-   const repository = {
-      listManifestEntries: sandbox.stub(),
-      upsertManifestEntry: sandbox.stub(),
-      deleteManifestEntry: sandbox.stub(),
-   };
-
-   const store = new DuckDBManifestStore(
-      repository as unknown as ResourceRepository,
-   );
-
-   return { sandbox, repository, store };
-}
-
-describe("DuckDBManifestStore", () => {
-   let ctx: ReturnType<typeof createMocks>;
-
-   beforeEach(() => {
-      ctx = createMocks();
-   });
-
-   describe("getManifest", () => {
-      it("should assemble a BuildManifest from repository entries", async () => {
-         ctx.repository.listManifestEntries.resolves([
-            makeEntry({ buildId: "b1", tableName: "tbl_a" }),
-            makeEntry({ buildId: "b2", tableName: "tbl_b" }),
-         ]);
-
-         const manifest = await ctx.store.getManifest("proj-1", "pkg");
-
-         expect(manifest.strict).toBe(false);
-         expect(manifest.entries).toEqual({
-            b1: { tableName: "tbl_a" },
-            b2: { tableName: "tbl_b" },
-         });
-      });
-
-      it("should return empty entries when no rows exist", async () => {
-         ctx.repository.listManifestEntries.resolves([]);
-
-         const manifest = await ctx.store.getManifest("proj-1", "pkg");
-
-         expect(manifest.strict).toBe(false);
-         expect(manifest.entries).toEqual({});
-      });
-   });
-
-   describe("writeEntry", () => {
-      it("should upsert an entry with all fields", async () => {
-         ctx.repository.upsertManifestEntry.resolves(makeEntry());
-
-         await ctx.store.writeEntry(
-            "proj-1",
-            "pkg",
-            "build-abc",
-            "tbl",
-            "src",
-            "conn",
-         );
-
-         expect(ctx.repository.upsertManifestEntry.calledOnce).toBe(true);
-         const arg = ctx.repository.upsertManifestEntry.firstCall.args[0];
-         expect(arg).toEqual({
-            environmentId: "proj-1",
-            packageName: "pkg",
-            buildId: "build-abc",
-            tableName: "tbl",
-            sourceName: "src",
-            connectionName: "conn",
-         });
-      });
-   });
-
-   describe("deleteEntry", () => {
-      it("should delegate to repository", async () => {
-         ctx.repository.deleteManifestEntry.resolves();
-
-         await ctx.store.deleteEntry("entry-1");
-
-         expect(ctx.repository.deleteManifestEntry.calledOnce).toBe(true);
-         expect(ctx.repository.deleteManifestEntry.firstCall.args[0]).toBe(
-            "entry-1",
-         );
-      });
-   });
-
-   describe("listEntries", () => {
-      it("should return entries from the repository", async () => {
-         const entries = [
-            makeEntry(),
-            makeEntry({ id: "entry-2", buildId: "build-def" }),
-         ];
-         ctx.repository.listManifestEntries.resolves(entries);
-
-         const result = await ctx.store.listEntries("proj-1", "pkg");
-
-         expect(result).toEqual(entries);
-         expect(
-            ctx.repository.listManifestEntries.calledWith("proj-1", "pkg"),
-         ).toBe(true);
-      });
-
-      it("should return empty array when no entries exist", async () => {
-         ctx.repository.listManifestEntries.resolves([]);
-
-         const result = await ctx.store.listEntries("proj-1", "pkg");
-
-         expect(result).toEqual([]);
-      });
-   });
-});