@malloy-publisher/server 0.0.187 → 0.0.189-dev

package/src/storage/duckdb/DuckDBRepository.ts

@@ -1,23 +1,32 @@
 import {
-   ResourceRepository,
-   Project,
-   Package,
    Connection,
+   ManifestEntry,
+   Materialization,
+   MaterializationStatus,
+   Package,
+   Project,
+   ResourceRepository,
 } from "../DatabaseInterface";
+import { ConnectionRepository } from "./ConnectionRepository";
 import { DuckDBConnection } from "./DuckDBConnection";
-import { ProjectRepository } from "./ProjectRepository";
+import { ManifestRepository } from "./ManifestRepository";
+import { MaterializationRepository } from "./MaterializationRepository";
 import { PackageRepository } from "./PackageRepository";
-import { ConnectionRepository } from "./ConnectionRepository";
+import { ProjectRepository } from "./ProjectRepository";
 
 export class DuckDBRepository implements ResourceRepository {
    private projectRepo: ProjectRepository;
    private packageRepo: PackageRepository;
    private connectionRepo: ConnectionRepository;
+   private materializationRepo: MaterializationRepository;
+   private manifestRepo: ManifestRepository;
 
    constructor(public db: DuckDBConnection) {
       this.projectRepo = new ProjectRepository(db);
       this.packageRepo = new PackageRepository(db);
       this.connectionRepo = new ConnectionRepository(db);
+      this.materializationRepo = new MaterializationRepository(db);
+      this.manifestRepo = new ManifestRepository(db);
    }
 
    // ==================== PROJECTS ====================
@@ -48,11 +57,10 @@ export class DuckDBRepository implements ResourceRepository {
    }
 
    async deleteProject(id: string): Promise<void> {
-      // Delete related connections and packages first
+      await this.manifestRepo.deleteEntriesByProjectId(id);
+      await this.materializationRepo.deleteByProjectId(id);
       await this.connectionRepo.deleteConnectionsByProjectId(id);
       await this.packageRepo.deletePackagesByProjectId(id);
-
-      // Then delete the project
       await this.projectRepo.deleteProject(id);
    }
 
@@ -87,7 +95,18 @@ export class DuckDBRepository implements ResourceRepository {
    }
 
    async deletePackage(id: string): Promise<void> {
-      return this.packageRepo.deletePackage(id);
+      const pkg = await this.packageRepo.getPackageById(id);
+      if (pkg) {
+         await this.manifestRepo.deleteEntriesByPackage(
+            pkg.projectId,
+            pkg.name,
+         );
+         await this.materializationRepo.deleteByPackage(
+            pkg.projectId,
+            pkg.name,
+         );
+      }
+      await this.packageRepo.deletePackage(id);
    }
 
    async deletePackagesByProjectId(id: string): Promise<void> {
@@ -131,4 +150,75 @@ export class DuckDBRepository implements ResourceRepository {
    async deleteConnectionsByProjectId(id: string): Promise<void> {
       return this.connectionRepo.deleteConnectionsByProjectId(id);
    }
+
+   // ==================== MATERIALIZATIONS ====================
+
+   async listMaterializations(
+      projectId: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]> {
+      return this.materializationRepo.list(projectId, packageName, options);
+   }
+
+   async getMaterializationById(id: string): Promise<Materialization | null> {
+      return this.materializationRepo.getById(id);
+   }
+
+   async getActiveMaterialization(
+      projectId: string,
+      packageName: string,
+   ): Promise<Materialization | null> {
+      return this.materializationRepo.getActive(projectId, packageName);
+   }
+
+   async createMaterialization(
+      projectId: string,
+      packageName: string,
+      status: MaterializationStatus = "PENDING",
+      metadata: Record<string, unknown> | null = null,
+   ): Promise<Materialization> {
+      return this.materializationRepo.create(
+         projectId,
+         packageName,
+         status,
+         metadata,
+      );
+   }
+
+   async updateMaterialization(
+      id: string,
+      updates: {
+         status?: MaterializationStatus;
+         startedAt?: Date;
+         completedAt?: Date;
+         error?: string | null;
+         metadata?: Record<string, unknown> | null;
+      },
+   ): Promise<Materialization> {
+      return this.materializationRepo.update(id, updates);
+   }
+
+   async deleteMaterialization(id: string): Promise<void> {
+      return this.materializationRepo.deleteById(id);
+   }
+
+   // ==================== BUILD MANIFESTS ====================
+
+   async listManifestEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      return this.manifestRepo.listEntries(projectId, packageName);
+   }
+
+   async upsertManifestEntry(
+      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
+   ): Promise<ManifestEntry> {
+      return this.manifestRepo.upsertEntry(entry);
+   }
+
+   async deleteManifestEntry(id: string): Promise<void> {
+      return this.manifestRepo.deleteEntry(id);
+   }
 }

package/src/storage/duckdb/ManifestRepository.ts

@@ -0,0 +1,119 @@
+import { ManifestEntry } from "../DatabaseInterface";
+import { DuckDBConnection } from "./DuckDBConnection";
+
+/**
+ * Repository for build manifest entries stored in the `build_manifests` table.
+ *
+ * A build manifest records the materialized table produced by a specific build
+ * of a package, linking it back to the Malloy source and connection that
+ * generated it. Manifests are keyed by (project, package, build) and enable
+ * downstream consumers to discover which physical tables correspond to a given
+ * package version.
+ */
+export class ManifestRepository {
+   constructor(private db: DuckDBConnection) {}
+
+   private generateId(): string {
+      return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+   }
+
+   /** Returns all manifest entries for a package, most recent first. */
+   async listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      const rows = await this.db.all<Record<string, unknown>>(
+         "SELECT * FROM build_manifests WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC",
+         [projectId, packageName],
+      );
+      return rows.map(this.mapToEntry);
+   }
+
+   /** Looks up the manifest entry for a specific build, or null if none exists. */
+   async getEntryByBuildId(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+   ): Promise<ManifestEntry | null> {
+      const row = await this.db.get<Record<string, unknown>>(
+         "SELECT * FROM build_manifests WHERE project_id = ? AND package_name = ? AND build_id = ?",
+         [projectId, packageName, buildId],
+      );
+      return row ? this.mapToEntry(row) : null;
+   }
+
+   /**
+    * Inserts a new manifest entry, or updates the existing one if a row with
+    * the same (project, package, build) triple already exists. Uses INSERT ON
+    * CONFLICT to avoid the TOCTOU race of SELECT-then-INSERT/UPDATE.
+    */
+   async upsertEntry(
+      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
+   ): Promise<ManifestEntry> {
+      const id = this.generateId();
+      const now = new Date();
+      const iso = now.toISOString();
+
+      const rows = await this.db.all<Record<string, unknown>>(
+         `INSERT INTO build_manifests (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT (project_id, package_name, build_id)
+         DO UPDATE SET table_name = EXCLUDED.table_name,
+                       source_name = EXCLUDED.source_name,
+                       connection_name = EXCLUDED.connection_name,
+                       updated_at = EXCLUDED.updated_at
+         RETURNING *`,
+         [
+            id,
+            entry.projectId,
+            entry.packageName,
+            entry.buildId,
+            entry.tableName,
+            entry.sourceName,
+            entry.connectionName,
+            iso,
+            iso,
+         ],
+      );
+
+      return this.mapToEntry(rows[0]);
+   }
+
+   /** Deletes a single manifest entry by ID. */
+   async deleteEntry(id: string): Promise<void> {
+      await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
+   }
+
+   /** Removes all manifest entries belonging to a project (used on project deletion). */
+   async deleteEntriesByProjectId(projectId: string): Promise<void> {
+      await this.db.run("DELETE FROM build_manifests WHERE project_id = ?", [
+         projectId,
+      ]);
+   }
+
+   /** Removes all manifest entries for a specific package. */
+   async deleteEntriesByPackage(
+      projectId: string,
+      packageName: string,
+   ): Promise<void> {
+      await this.db.run(
+         "DELETE FROM build_manifests WHERE project_id = ? AND package_name = ?",
+         [projectId, packageName],
+      );
+   }
+
+   /** Maps a raw DB row (snake_case columns) to a {@link ManifestEntry}. */
+   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
+      return {
+         id: row.id as string,
+         projectId: row.project_id as string,
+         packageName: row.package_name as string,
+         buildId: row.build_id as string,
+         tableName: row.table_name as string,
+         sourceName: row.source_name as string,
+         connectionName: row.connection_name as string,
+         createdAt: new Date(row.created_at as string),
+         updatedAt: new Date(row.updated_at as string),
+      };
+   }
+}

package/src/storage/duckdb/MaterializationRepository.ts

@@ -0,0 +1,249 @@
+import { Materialization, MaterializationStatus } from "../DatabaseInterface";
+import { DuckDBConnection } from "./DuckDBConnection";
+
+const TERMINAL_STATUSES: ReadonlySet<MaterializationStatus> = new Set([
+   "SUCCESS",
+   "FAILED",
+   "CANCELLED",
+]);
+
+function activeKeyFor(projectId: string, packageName: string): string {
+   return `${projectId}|${packageName}`;
+}
+
+/**
+ * Thrown when an atomic insert loses a race on (project, package) active
+ * materialization. Surfaced separately from a generic DB error so the service
+ * layer can translate to `MaterializationConflictError`.
+ */
+export class DuplicateActiveMaterializationError extends Error {
+   constructor(projectId: string, packageName: string) {
+      super(
+         `Active materialization already exists for (${projectId}, ${packageName})`,
+      );
+   }
+}
+
+/**
+ * DuckDB-backed repository for package materializations.
+ *
+ * A Materialization tracks a single build run for a (project, package) pair
+ * through its lifecycle: PENDING -> RUNNING -> SUCCESS | FAILED | CANCELLED.
+ */
+export class MaterializationRepository {
+   constructor(private db: DuckDBConnection) {}
+
+   private generateId(): string {
+      return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+   }
+
+   private now(): Date {
+      return new Date();
+   }
+
+   async list(
+      projectId: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]> {
+      let sql =
+         "SELECT * FROM materializations WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC";
+      const params: unknown[] = [projectId, packageName];
+      if (options?.limit !== undefined) {
+         sql += " LIMIT ?";
+         params.push(options.limit);
+      }
+      if (options?.offset !== undefined) {
+         sql += " OFFSET ?";
+         params.push(options.offset);
+      }
+      const rows = await this.db.all<Record<string, unknown>>(sql, params);
+      return rows.map(this.mapRow);
+   }
+
+   async getById(id: string): Promise<Materialization | null> {
+      const row = await this.db.get<Record<string, unknown>>(
+         "SELECT * FROM materializations WHERE id = ?",
+         [id],
+      );
+      return row ? this.mapRow(row) : null;
+   }
+
+   async getActive(
+      projectId: string,
+      packageName: string,
+   ): Promise<Materialization | null> {
+      const row = await this.db.get<Record<string, unknown>>(
+         "SELECT * FROM materializations WHERE project_id = ? AND package_name = ? AND status IN ('PENDING', 'RUNNING')",
+         [projectId, packageName],
+      );
+      return row ? this.mapRow(row) : null;
+   }
+
+   async create(
+      projectId: string,
+      packageName: string,
+      status: MaterializationStatus = "PENDING",
+      metadata: Record<string, unknown> | null = null,
+   ): Promise<Materialization> {
+      const id = this.generateId();
+      const now = this.now();
+      const iso = now.toISOString();
+      // Set active_key iff the row is in a non-terminal state. The unique
+      // index on active_key makes the race-free conditional insert: a second
+      // concurrent create on the same (project, package) fails here rather
+      // than in a check-then-write window.
+      const activeKey = TERMINAL_STATUSES.has(status)
+         ? null
+         : activeKeyFor(projectId, packageName);
+      const metadataJson = metadata ? JSON.stringify(metadata) : null;
+
+      try {
+         const rows = await this.db.all<Record<string, unknown>>(
+            `INSERT INTO materializations (id, project_id, package_name, status, active_key, metadata, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            RETURNING *`,
+            [
+               id,
+               projectId,
+               packageName,
+               status,
+               activeKey,
+               metadataJson,
+               iso,
+               iso,
+            ],
+         );
+         return this.mapRow(rows[0]);
+      } catch (err) {
+         if (isUniqueViolation(err, "idx_materializations_active_key")) {
+            throw new DuplicateActiveMaterializationError(
+               projectId,
+               packageName,
+            );
+         }
+         throw err;
+      }
+   }
+
+   async update(
+      id: string,
+      updates: {
+         status?: MaterializationStatus;
+         startedAt?: Date;
+         completedAt?: Date;
+         error?: string | null;
+         metadata?: Record<string, unknown> | null;
+      },
+   ): Promise<Materialization> {
+      const now = this.now();
+      const setClauses: string[] = [];
+      const params: unknown[] = [];
+
+      if (updates.status !== undefined) {
+         setClauses.push(`status = ?`);
+         params.push(updates.status);
+         // Clear active_key on any transition to a terminal state; set it on
+         // any transition to a non-terminal state. The unique index
+         // guarantees we can never end up with two active rows for the same
+         // (project, package).
+         if (TERMINAL_STATUSES.has(updates.status)) {
+            setClauses.push(`active_key = NULL`);
+         } else {
+            setClauses.push(`active_key = project_id || '|' || package_name`);
+         }
+      }
+      if (updates.startedAt !== undefined) {
+         setClauses.push(`started_at = ?`);
+         params.push(updates.startedAt.toISOString());
+      }
+      if (updates.completedAt !== undefined) {
+         setClauses.push(`completed_at = ?`);
+         params.push(updates.completedAt.toISOString());
+      }
+      if (updates.error !== undefined) {
+         setClauses.push(`error = ?`);
+         params.push(updates.error);
+      }
+      if (updates.metadata !== undefined) {
+         setClauses.push(`metadata = ?`);
+         params.push(
+            updates.metadata ? JSON.stringify(updates.metadata) : null,
+         );
+      }
+
+      setClauses.push(`updated_at = ?`);
+      params.push(now.toISOString());
+      params.push(id);
+
+      await this.db.run(
+         `UPDATE materializations SET ${setClauses.join(", ")} WHERE id = ?`,
+         params,
+      );
+
+      const updated = await this.getById(id);
+      if (!updated) {
+         throw new Error(`Materialization ${id} not found after update`);
+      }
+      return updated;
+   }
+
+   async deleteByProjectId(projectId: string): Promise<void> {
+      await this.db.run("DELETE FROM materializations WHERE project_id = ?", [
+         projectId,
+      ]);
+   }
+
+   async deleteById(id: string): Promise<void> {
+      await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
+   }
+
+   async deleteByPackage(
+      projectId: string,
+      packageName: string,
+   ): Promise<void> {
+      await this.db.run(
+         "DELETE FROM materializations WHERE project_id = ? AND package_name = ?",
+         [projectId, packageName],
+      );
+   }
+
+   private mapRow(row: Record<string, unknown>): Materialization {
+      let metadata: Record<string, unknown> | null = null;
+      if (row.metadata) {
+         try {
+            metadata = JSON.parse(row.metadata as string);
+         } catch {
+            metadata = null;
+         }
+      }
+
+      return {
+         id: row.id as string,
+         projectId: row.project_id as string,
+         packageName: row.package_name as string,
+         status: row.status as MaterializationStatus,
+         startedAt: row.started_at ? new Date(row.started_at as string) : null,
+         completedAt: row.completed_at
+            ? new Date(row.completed_at as string)
+            : null,
+         error: row.error != null ? (row.error as string) : null,
+         metadata,
+         createdAt: new Date(row.created_at as string),
+         updatedAt: new Date(row.updated_at as string),
+      };
+   }
+}
+
+/**
+ * DuckDB surfaces unique-constraint violations as plain Errors whose message
+ * mentions the violated index. We match on the index name rather than a
+ * generic substring so we don't misclassify unrelated constraint errors.
+ */
+function isUniqueViolation(err: unknown, indexName: string): boolean {
+   if (!(err instanceof Error)) return false;
+   const msg = err.message;
+   return (
+      msg.includes(indexName) || /duplicate key|unique constraint/i.test(msg)
+   );
+}

package/src/storage/duckdb/manifest_store.spec.ts

@@ -0,0 +1,133 @@
+import { beforeEach, describe, expect, it } from "bun:test";
+import * as sinon from "sinon";
+import { ManifestEntry, ResourceRepository } from "../DatabaseInterface";
+import { DuckDBManifestStore } from "./DuckDBManifestStore";
+
+function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
+   return {
+      id: "entry-1",
+      projectId: "proj-1",
+      packageName: "pkg",
+      buildId: "build-abc",
+      tableName: "my_table",
+      sourceName: "my_source",
+      connectionName: "duckdb",
+      createdAt: new Date("2026-04-03"),
+      updatedAt: new Date("2026-04-03"),
+      ...overrides,
+   };
+}
+
+function createMocks() {
+   const sandbox = sinon.createSandbox();
+
+   const repository = {
+      listManifestEntries: sandbox.stub(),
+      upsertManifestEntry: sandbox.stub(),
+      deleteManifestEntry: sandbox.stub(),
+   };
+
+   const store = new DuckDBManifestStore(
+      repository as unknown as ResourceRepository,
+   );
+
+   return { sandbox, repository, store };
+}
+
+describe("DuckDBManifestStore", () => {
+   let ctx: ReturnType<typeof createMocks>;
+
+   beforeEach(() => {
+      ctx = createMocks();
+   });
+
+   describe("getManifest", () => {
+      it("should assemble a BuildManifest from repository entries", async () => {
+         ctx.repository.listManifestEntries.resolves([
+            makeEntry({ buildId: "b1", tableName: "tbl_a" }),
+            makeEntry({ buildId: "b2", tableName: "tbl_b" }),
+         ]);
+
+         const manifest = await ctx.store.getManifest("proj-1", "pkg");
+
+         expect(manifest.strict).toBe(false);
+         expect(manifest.entries).toEqual({
+            b1: { tableName: "tbl_a" },
+            b2: { tableName: "tbl_b" },
+         });
+      });
+
+      it("should return empty entries when no rows exist", async () => {
+         ctx.repository.listManifestEntries.resolves([]);
+
+         const manifest = await ctx.store.getManifest("proj-1", "pkg");
+
+         expect(manifest.strict).toBe(false);
+         expect(manifest.entries).toEqual({});
+      });
+   });
+
+   describe("writeEntry", () => {
+      it("should upsert an entry with all fields", async () => {
+         ctx.repository.upsertManifestEntry.resolves(makeEntry());
+
+         await ctx.store.writeEntry(
+            "proj-1",
+            "pkg",
+            "build-abc",
+            "tbl",
+            "src",
+            "conn",
+         );
+
+         expect(ctx.repository.upsertManifestEntry.calledOnce).toBe(true);
+         const arg = ctx.repository.upsertManifestEntry.firstCall.args[0];
+         expect(arg).toEqual({
+            projectId: "proj-1",
+            packageName: "pkg",
+            buildId: "build-abc",
+            tableName: "tbl",
+            sourceName: "src",
+            connectionName: "conn",
+         });
+      });
+   });
+
+   describe("deleteEntry", () => {
+      it("should delegate to repository", async () => {
+         ctx.repository.deleteManifestEntry.resolves();
+
+         await ctx.store.deleteEntry("entry-1");
+
+         expect(ctx.repository.deleteManifestEntry.calledOnce).toBe(true);
+         expect(ctx.repository.deleteManifestEntry.firstCall.args[0]).toBe(
+            "entry-1",
+         );
+      });
+   });
+
+   describe("listEntries", () => {
+      it("should return entries from the repository", async () => {
+         const entries = [
+            makeEntry(),
+            makeEntry({ id: "entry-2", buildId: "build-def" }),
+         ];
+         ctx.repository.listManifestEntries.resolves(entries);
+
+         const result = await ctx.store.listEntries("proj-1", "pkg");
+
+         expect(result).toEqual(entries);
+         expect(
+            ctx.repository.listManifestEntries.calledWith("proj-1", "pkg"),
+         ).toBe(true);
+      });
+
+      it("should return empty array when no entries exist", async () => {
+         ctx.repository.listManifestEntries.resolves([]);
+
+         const result = await ctx.store.listEntries("proj-1", "pkg");
+
+         expect(result).toEqual([]);
+      });
+   });
+});