@malloy-publisher/server 0.0.180 → 0.0.181-dev-v1

package/src/storage/duckdb/MaterializationRepository.ts

@@ -0,0 +1,249 @@
+import { Materialization, MaterializationStatus } from "../DatabaseInterface";
+import { DuckDBConnection } from "./DuckDBConnection";
+
+const TERMINAL_STATUSES: ReadonlySet<MaterializationStatus> = new Set([
+   "SUCCESS",
+   "FAILED",
+   "CANCELLED",
+]);
+
+function activeKeyFor(projectId: string, packageName: string): string {
+   return `${projectId}|${packageName}`;
+}
+
+/**
+ * Thrown when an atomic insert loses a race on (project, package) active
+ * materialization. Surfaced separately from a generic DB error so the service
+ * layer can translate to `MaterializationConflictError`.
+ */
+export class DuplicateActiveMaterializationError extends Error {
+   constructor(projectId: string, packageName: string) {
+      super(
+         `Active materialization already exists for (${projectId}, ${packageName})`,
+      );
+   }
+}
+
+/**
+ * DuckDB-backed repository for package materializations.
+ *
+ * A Materialization tracks a single build run for a (project, package) pair
+ * through its lifecycle: PENDING -> RUNNING -> SUCCESS | FAILED | CANCELLED.
+ */
+export class MaterializationRepository {
+   constructor(private db: DuckDBConnection) {}
+
+   private generateId(): string {
+      return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+   }
+
+   private now(): Date {
+      return new Date();
+   }
+
+   async list(
+      projectId: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]> {
+      let sql =
+         "SELECT * FROM materializations WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC";
+      const params: unknown[] = [projectId, packageName];
+      if (options?.limit !== undefined) {
+         sql += " LIMIT ?";
+         params.push(options.limit);
+      }
+      if (options?.offset !== undefined) {
+         sql += " OFFSET ?";
+         params.push(options.offset);
+      }
+      const rows = await this.db.all<Record<string, unknown>>(sql, params);
+      return rows.map(this.mapRow);
+   }
+
+   async getById(id: string): Promise<Materialization | null> {
+      const row = await this.db.get<Record<string, unknown>>(
+         "SELECT * FROM materializations WHERE id = ?",
+         [id],
+      );
+      return row ? this.mapRow(row) : null;
+   }
+
+   async getActive(
+      projectId: string,
+      packageName: string,
+   ): Promise<Materialization | null> {
+      const row = await this.db.get<Record<string, unknown>>(
+         "SELECT * FROM materializations WHERE project_id = ? AND package_name = ? AND status IN ('PENDING', 'RUNNING')",
+         [projectId, packageName],
+      );
+      return row ? this.mapRow(row) : null;
+   }
+
+   async create(
+      projectId: string,
+      packageName: string,
+      status: MaterializationStatus = "PENDING",
+      metadata: Record<string, unknown> | null = null,
+   ): Promise<Materialization> {
+      const id = this.generateId();
+      const now = this.now();
+      const iso = now.toISOString();
+      // Set active_key iff the row is in a non-terminal state. The unique
+      // index on active_key makes the race-free conditional insert: a second
+      // concurrent create on the same (project, package) fails here rather
+      // than in a check-then-write window.
+      const activeKey = TERMINAL_STATUSES.has(status)
+         ? null
+         : activeKeyFor(projectId, packageName);
+      const metadataJson = metadata ? JSON.stringify(metadata) : null;
+
+      try {
+         const rows = await this.db.all<Record<string, unknown>>(
+            `INSERT INTO materializations (id, project_id, package_name, status, active_key, metadata, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            RETURNING *`,
+            [
+               id,
+               projectId,
+               packageName,
+               status,
+               activeKey,
+               metadataJson,
+               iso,
+               iso,
+            ],
+         );
+         return this.mapRow(rows[0]);
+      } catch (err) {
+         if (isUniqueViolation(err, "idx_materializations_active_key")) {
+            throw new DuplicateActiveMaterializationError(
+               projectId,
+               packageName,
+            );
+         }
+         throw err;
+      }
+   }
+
+   async update(
+      id: string,
+      updates: {
+         status?: MaterializationStatus;
+         startedAt?: Date;
+         completedAt?: Date;
+         error?: string | null;
+         metadata?: Record<string, unknown> | null;
+      },
+   ): Promise<Materialization> {
+      const now = this.now();
+      const setClauses: string[] = [];
+      const params: unknown[] = [];
+
+      if (updates.status !== undefined) {
+         setClauses.push(`status = ?`);
+         params.push(updates.status);
+         // Clear active_key on any transition to a terminal state; set it on
+         // any transition to a non-terminal state. The unique index
+         // guarantees we can never end up with two active rows for the same
+         // (project, package).
+         if (TERMINAL_STATUSES.has(updates.status)) {
+            setClauses.push(`active_key = NULL`);
+         } else {
+            setClauses.push(`active_key = project_id || '|' || package_name`);
+         }
+      }
+      if (updates.startedAt !== undefined) {
+         setClauses.push(`started_at = ?`);
+         params.push(updates.startedAt.toISOString());
+      }
+      if (updates.completedAt !== undefined) {
+         setClauses.push(`completed_at = ?`);
+         params.push(updates.completedAt.toISOString());
+      }
+      if (updates.error !== undefined) {
+         setClauses.push(`error = ?`);
+         params.push(updates.error);
+      }
+      if (updates.metadata !== undefined) {
+         setClauses.push(`metadata = ?`);
+         params.push(
+            updates.metadata ? JSON.stringify(updates.metadata) : null,
+         );
+      }
+
+      setClauses.push(`updated_at = ?`);
+      params.push(now.toISOString());
+      params.push(id);
+
+      await this.db.run(
+         `UPDATE materializations SET ${setClauses.join(", ")} WHERE id = ?`,
+         params,
+      );
+
+      const updated = await this.getById(id);
+      if (!updated) {
+         throw new Error(`Materialization ${id} not found after update`);
+      }
+      return updated;
+   }
+
+   async deleteByProjectId(projectId: string): Promise<void> {
+      await this.db.run("DELETE FROM materializations WHERE project_id = ?", [
+         projectId,
+      ]);
+   }
+
+   async deleteById(id: string): Promise<void> {
+      await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
+   }
+
+   async deleteByPackage(
+      projectId: string,
+      packageName: string,
+   ): Promise<void> {
+      await this.db.run(
+         "DELETE FROM materializations WHERE project_id = ? AND package_name = ?",
+         [projectId, packageName],
+      );
+   }
+
+   private mapRow(row: Record<string, unknown>): Materialization {
+      let metadata: Record<string, unknown> | null = null;
+      if (row.metadata) {
+         try {
+            metadata = JSON.parse(row.metadata as string);
+         } catch {
+            metadata = null;
+         }
+      }
+
+      return {
+         id: row.id as string,
+         projectId: row.project_id as string,
+         packageName: row.package_name as string,
+         status: row.status as MaterializationStatus,
+         startedAt: row.started_at ? new Date(row.started_at as string) : null,
+         completedAt: row.completed_at
+            ? new Date(row.completed_at as string)
+            : null,
+         error: row.error != null ? (row.error as string) : null,
+         metadata,
+         createdAt: new Date(row.created_at as string),
+         updatedAt: new Date(row.updated_at as string),
+      };
+   }
+}
+
+/**
+ * DuckDB surfaces unique-constraint violations as plain Errors whose message
+ * mentions the violated index. We match on the index name rather than a
+ * generic substring so we don't misclassify unrelated constraint errors.
+ */
+function isUniqueViolation(err: unknown, indexName: string): boolean {
+   if (!(err instanceof Error)) return false;
+   const msg = err.message;
+   return (
+      msg.includes(indexName) || /duplicate key|unique constraint/i.test(msg)
+   );
+}

package/src/storage/duckdb/manifest_store.spec.ts

@@ -0,0 +1,133 @@
+import { beforeEach, describe, expect, it } from "bun:test";
+import * as sinon from "sinon";
+import { ManifestEntry, ResourceRepository } from "../DatabaseInterface";
+import { DuckDBManifestStore } from "./DuckDBManifestStore";
+
+function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
+   return {
+      id: "entry-1",
+      projectId: "proj-1",
+      packageName: "pkg",
+      buildId: "build-abc",
+      tableName: "my_table",
+      sourceName: "my_source",
+      connectionName: "duckdb",
+      createdAt: new Date("2026-04-03"),
+      updatedAt: new Date("2026-04-03"),
+      ...overrides,
+   };
+}
+
+function createMocks() {
+   const sandbox = sinon.createSandbox();
+
+   const repository = {
+      listManifestEntries: sandbox.stub(),
+      upsertManifestEntry: sandbox.stub(),
+      deleteManifestEntry: sandbox.stub(),
+   };
+
+   const store = new DuckDBManifestStore(
+      repository as unknown as ResourceRepository,
+   );
+
+   return { sandbox, repository, store };
+}
+
+describe("DuckDBManifestStore", () => {
+   let ctx: ReturnType<typeof createMocks>;
+
+   beforeEach(() => {
+      ctx = createMocks();
+   });
+
+   describe("getManifest", () => {
+      it("should assemble a BuildManifest from repository entries", async () => {
+         ctx.repository.listManifestEntries.resolves([
+            makeEntry({ buildId: "b1", tableName: "tbl_a" }),
+            makeEntry({ buildId: "b2", tableName: "tbl_b" }),
+         ]);
+
+         const manifest = await ctx.store.getManifest("proj-1", "pkg");
+
+         expect(manifest.strict).toBe(false);
+         expect(manifest.entries).toEqual({
+            b1: { tableName: "tbl_a" },
+            b2: { tableName: "tbl_b" },
+         });
+      });
+
+      it("should return empty entries when no rows exist", async () => {
+         ctx.repository.listManifestEntries.resolves([]);
+
+         const manifest = await ctx.store.getManifest("proj-1", "pkg");
+
+         expect(manifest.strict).toBe(false);
+         expect(manifest.entries).toEqual({});
+      });
+   });
+
+   describe("writeEntry", () => {
+      it("should upsert an entry with all fields", async () => {
+         ctx.repository.upsertManifestEntry.resolves(makeEntry());
+
+         await ctx.store.writeEntry(
+            "proj-1",
+            "pkg",
+            "build-abc",
+            "tbl",
+            "src",
+            "conn",
+         );
+
+         expect(ctx.repository.upsertManifestEntry.calledOnce).toBe(true);
+         const arg = ctx.repository.upsertManifestEntry.firstCall.args[0];
+         expect(arg).toEqual({
+            projectId: "proj-1",
+            packageName: "pkg",
+            buildId: "build-abc",
+            tableName: "tbl",
+            sourceName: "src",
+            connectionName: "conn",
+         });
+      });
+   });
+
+   describe("deleteEntry", () => {
+      it("should delegate to repository", async () => {
+         ctx.repository.deleteManifestEntry.resolves();
+
+         await ctx.store.deleteEntry("entry-1");
+
+         expect(ctx.repository.deleteManifestEntry.calledOnce).toBe(true);
+         expect(ctx.repository.deleteManifestEntry.firstCall.args[0]).toBe(
+            "entry-1",
+         );
+      });
+   });
+
+   describe("listEntries", () => {
+      it("should return entries from the repository", async () => {
+         const entries = [
+            makeEntry(),
+            makeEntry({ id: "entry-2", buildId: "build-def" }),
+         ];
+         ctx.repository.listManifestEntries.resolves(entries);
+
+         const result = await ctx.store.listEntries("proj-1", "pkg");
+
+         expect(result).toEqual(entries);
+         expect(
+            ctx.repository.listManifestEntries.calledWith("proj-1", "pkg"),
+         ).toBe(true);
+      });
+
+      it("should return empty array when no entries exist", async () => {
+         ctx.repository.listManifestEntries.resolves([]);
+
+         const result = await ctx.store.listEntries("proj-1", "pkg");
+
+         expect(result).toEqual([]);
+      });
+   });
+});

package/src/storage/duckdb/schema.ts

@@ -64,6 +64,49 @@ export async function initializeSchema(
     )
   `);
 
+   // Materializations table.
+   //
+   // `active_key` enforces at-most-one active (PENDING or RUNNING)
+   // materialization per (project, package) at the DB layer. It is set to
+   // `{project_id}|{package_name}` while the row is active and cleared
+   // to NULL on transition to any terminal state. A unique index on
+   // `active_key` (see below) makes the insert-then-check race impossible —
+   // a second concurrent create fails with a constraint violation, which the
+   // service layer translates to `MaterializationConflictError`.
+   await db.run(`
+    CREATE TABLE IF NOT EXISTS materializations (
+      id VARCHAR PRIMARY KEY,
+      project_id VARCHAR NOT NULL,
+      package_name VARCHAR NOT NULL,
+      status VARCHAR NOT NULL,
+      active_key VARCHAR,
+      started_at TIMESTAMP,
+      completed_at TIMESTAMP,
+      error TEXT,
+      metadata JSON,
+      created_at TIMESTAMP NOT NULL,
+      updated_at TIMESTAMP NOT NULL,
+      FOREIGN KEY (project_id) REFERENCES projects(id)
+    )
+  `);
+
+   // Build manifests table
+   await db.run(`
+    CREATE TABLE IF NOT EXISTS build_manifests (
+      id VARCHAR PRIMARY KEY,
+      project_id VARCHAR NOT NULL,
+      package_name VARCHAR NOT NULL,
+      build_id VARCHAR NOT NULL,
+      table_name VARCHAR NOT NULL,
+      source_name VARCHAR NOT NULL,
+      connection_name VARCHAR NOT NULL,
+      created_at TIMESTAMP NOT NULL,
+      updated_at TIMESTAMP NOT NULL,
+      FOREIGN KEY (project_id) REFERENCES projects(id),
+      UNIQUE (project_id, package_name, build_id)
+    )
+  `);
+
    // Create indexes for better query performance
    await db.run(
       "CREATE INDEX IF NOT EXISTS idx_packages_project_id ON packages(project_id)",
@@ -71,10 +114,25 @@ export async function initializeSchema(
    await db.run(
       "CREATE INDEX IF NOT EXISTS idx_connections_project_id ON connections(project_id)",
    );
+   await db.run(
+      "CREATE INDEX IF NOT EXISTS idx_materializations_project_package ON materializations(project_id, package_name)",
+   );
+   await db.run(
+      "CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
+   );
+   await db.run(
+      "CREATE INDEX IF NOT EXISTS idx_build_manifests_project_package ON build_manifests(project_id, package_name)",
+   );
 }
 
 async function dropAllTables(db: DuckDBConnection): Promise<void> {
-   const tables = ["packages", "connections", "projects"];
+   const tables = [
+      "build_manifests",
+      "materializations",
+      "packages",
+      "connections",
+      "projects",
+   ];
 
    logger.info("Dropping tables:", tables.join(", "));
 

package/src/storage/ducklake/DuckLakeManifestStore.ts

@@ -0,0 +1,146 @@
+import { logger } from "../../logger";
+import {
+   BuildManifest,
+   ManifestEntry,
+   ManifestStore,
+} from "../DatabaseInterface";
+import { DuckDBConnection } from "../duckdb/DuckDBConnection";
+
+/**
+ * DuckLake-backed ManifestStore used in orchestrated mode.
+ *
+ * Reads and writes manifest entries through a DuckLake catalog attached to
+ * the publisher's internal DuckDB. All workers sharing the same DuckLake
+ * catalog see the same manifest, enabling multi-worker coordination.
+ *
+ * The catalog is attached by {@link StorageManager} before this store is
+ * instantiated. The first worker to call {@link bootstrapSchema} creates the
+ * `build_manifests` table idempotently. Schema ownership lives in the
+ * publisher so that DDL and code co-evolve in the same repo.
+ *
+ * **Scope: manifest sync only.** Orchestrated mode only shares manifest
+ * state across workers. The active-materialization lock (the unique index
+ * on `materializations.active_key`) still lives in each worker's local
+ * DuckDB, so two workers can run builds concurrently and race on physical
+ * table names, staging table names, and manifest writes. Until a shared
+ * build lease lives in the DuckLake catalog, deployments running
+ * orchestrated mode must ensure builds are externally single-writer (e.g.,
+ * one designated build worker, or an external job scheduler). Other
+ * workers should only call `manifest?action=reload` to pick up manifests produced
+ * by the build worker.
+ */
+export class DuckLakeManifestStore implements ManifestStore {
+   private readonly table: string;
+
+   constructor(
+      private db: DuckDBConnection,
+      catalogName: string,
+   ) {
+      this.table = `${catalogName}.build_manifests`;
+   }
+
+   /**
+    * Idempotently creates the `build_manifests` table and indices in the
+    * DuckLake catalog. Safe to call from every worker on startup.
+    */
+   async bootstrapSchema(): Promise<void> {
+      await this.db.run(`
+         CREATE TABLE IF NOT EXISTS ${this.table} (
+            id VARCHAR,
+            project_id VARCHAR NOT NULL,
+            package_name VARCHAR NOT NULL,
+            build_id VARCHAR NOT NULL,
+            table_name VARCHAR NOT NULL,
+            source_name VARCHAR NOT NULL,
+            connection_name VARCHAR NOT NULL,
+            created_at TIMESTAMP NOT NULL,
+            updated_at TIMESTAMP NOT NULL
+         )
+      `);
+      logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
+   }
+
+   async getManifest(
+      projectId: string,
+      packageName: string,
+   ): Promise<BuildManifest> {
+      const rows = await this.db.all<Record<string, unknown>>(
+         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
+         [projectId, packageName],
+      );
+      const manifest: BuildManifest = { entries: {}, strict: false };
+      for (const row of rows) {
+         const buildId = row.build_id as string;
+         // Rows are ordered newest-first; keep only the latest per build_id
+         // to handle rare duplicates from cross-worker races.
+         if (!manifest.entries[buildId]) {
+            manifest.entries[buildId] = {
+               tableName: row.table_name as string,
+            };
+         }
+      }
+      return manifest;
+   }
+
+   /**
+    * Insert a manifest entry. If a row with the same `build_id` already
+    * exists (retry after crash), the duplicate is harmless:
+    * {@link getManifest} deduplicates by build_id keeping the newest row.
+    */
+   async writeEntry(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+      tableName: string,
+      sourceName: string,
+      connectionName: string,
+   ): Promise<void> {
+      const now = new Date().toISOString();
+      const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+
+      await this.db.run(
+         `INSERT INTO ${this.table} (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+         [
+            id,
+            projectId,
+            packageName,
+            buildId,
+            tableName,
+            sourceName,
+            connectionName,
+            now,
+            now,
+         ],
+      );
+   }
+
+   async deleteEntry(id: string): Promise<void> {
+      await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
+   }
+
+   async listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      const rows = await this.db.all<Record<string, unknown>>(
+         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
+         [projectId, packageName],
+      );
+      return rows.map(this.mapToEntry);
+   }
+
+   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
+      return {
+         id: row.id as string,
+         projectId: row.project_id as string,
+         packageName: row.package_name as string,
+         buildId: row.build_id as string,
+         tableName: row.table_name as string,
+         sourceName: row.source_name as string,
+         connectionName: row.connection_name as string,
+         createdAt: new Date(row.created_at as string),
+         updatedAt: new Date(row.updated_at as string),
+      };
+   }
+}

package/tests/fixtures/persist-test/data/orders.csv

@@ -0,0 +1,5 @@
+category,amount
+electronics,100
+clothing,50
+electronics,200
+clothing,75

package/tests/fixtures/persist-test/persist_test.malloy

@@ -0,0 +1,11 @@
+##! experimental.persistence
+
+source: raw_orders is duckdb.table('data/orders.csv')
+
+#@ persist name="order_summary"
+source: order_summary is raw_orders -> {
+  group_by: category
+  aggregate:
+    total_orders is count()
+    total_revenue is amount.sum()
+}

package/tests/fixtures/persist-test/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "persist-test",
+  "version": "1.0.0",
+  "description": "DuckDB persist test fixture"
+}

package/tests/fixtures/publisher.config.json

@@ -0,0 +1,15 @@
+{
+  "frozenConfig": false,
+  "projects": [
+    {
+      "name": "test-project",
+      "packages": [
+        {
+          "name": "persist-test",
+          "location": "./persist-test"
+        }
+      ],
+      "connections": []
+    }
+  ]
+}