@malloy-publisher/server 0.0.188 → 0.0.382-dev

package/src/storage/duckdb/schema.ts

@@ -64,6 +64,49 @@ export async function initializeSchema(
     )
   `);
 
+   // Materializations table.
+   //
+   // `active_key` enforces at-most-one active (PENDING or RUNNING)
+   // materialization per (project, package) at the DB layer. It is set to
+   // `{project_id}|{package_name}` while the row is active and cleared
+   // to NULL on transition to any terminal state. A unique index on
+   // `active_key` (see below) makes the insert-then-check race impossible —
+   // a second concurrent create fails with a constraint violation, which the
+   // service layer translates to `MaterializationConflictError`.
+   await db.run(`
+    CREATE TABLE IF NOT EXISTS materializations (
+      id VARCHAR PRIMARY KEY,
+      project_id VARCHAR NOT NULL,
+      package_name VARCHAR NOT NULL,
+      status VARCHAR NOT NULL,
+      active_key VARCHAR,
+      started_at TIMESTAMP,
+      completed_at TIMESTAMP,
+      error TEXT,
+      metadata JSON,
+      created_at TIMESTAMP NOT NULL,
+      updated_at TIMESTAMP NOT NULL,
+      FOREIGN KEY (project_id) REFERENCES projects(id)
+    )
+  `);
+
+   // Build manifests table
+   await db.run(`
+    CREATE TABLE IF NOT EXISTS build_manifests (
+      id VARCHAR PRIMARY KEY,
+      project_id VARCHAR NOT NULL,
+      package_name VARCHAR NOT NULL,
+      build_id VARCHAR NOT NULL,
+      table_name VARCHAR NOT NULL,
+      source_name VARCHAR NOT NULL,
+      connection_name VARCHAR NOT NULL,
+      created_at TIMESTAMP NOT NULL,
+      updated_at TIMESTAMP NOT NULL,
+      FOREIGN KEY (project_id) REFERENCES projects(id),
+      UNIQUE (project_id, package_name, build_id)
+    )
+  `);
+
    // Create indexes for better query performance
    await db.run(
       "CREATE INDEX IF NOT EXISTS idx_packages_project_id ON packages(project_id)",
@@ -71,10 +114,25 @@ export async function initializeSchema(
    await db.run(
       "CREATE INDEX IF NOT EXISTS idx_connections_project_id ON connections(project_id)",
    );
+   await db.run(
+      "CREATE INDEX IF NOT EXISTS idx_materializations_project_package ON materializations(project_id, package_name)",
+   );
+   await db.run(
+      "CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
+   );
+   await db.run(
+      "CREATE INDEX IF NOT EXISTS idx_build_manifests_project_package ON build_manifests(project_id, package_name)",
+   );
 }
 
 async function dropAllTables(db: DuckDBConnection): Promise<void> {
-   const tables = ["packages", "connections", "projects"];
+   const tables = [
+      "build_manifests",
+      "materializations",
+      "packages",
+      "connections",
+      "projects",
+   ];
 
    logger.info("Dropping tables:", tables.join(", "));
 

package/src/storage/ducklake/DuckLakeManifestStore.ts

@@ -0,0 +1,146 @@
+import { logger } from "../../logger";
+import {
+   BuildManifest,
+   ManifestEntry,
+   ManifestStore,
+} from "../DatabaseInterface";
+import { DuckDBConnection } from "../duckdb/DuckDBConnection";
+
+/**
+ * DuckLake-backed ManifestStore used in orchestrated mode.
+ *
+ * Reads and writes manifest entries through a DuckLake catalog attached to
+ * the publisher's internal DuckDB. All workers sharing the same DuckLake
+ * catalog see the same manifest, enabling multi-worker coordination.
+ *
+ * The catalog is attached by {@link StorageManager} before this store is
+ * instantiated. The first worker to call {@link bootstrapSchema} creates the
+ * `build_manifests` table idempotently. Schema ownership lives in the
+ * publisher so that DDL and code co-evolve in the same repo.
+ *
+ * **Scope: manifest sync only.** Orchestrated mode only shares manifest
+ * state across workers. The active-materialization lock (the unique index
+ * on `materializations.active_key`) still lives in each worker's local
+ * DuckDB, so two workers can run builds concurrently and race on physical
+ * table names, staging table names, and manifest writes. Until a shared
+ * build lease lives in the DuckLake catalog, deployments running
+ * orchestrated mode must ensure builds are externally single-writer (e.g.,
+ * one designated build worker, or an external job scheduler). Other
+ * workers should only call `manifest?action=reload` to pick up manifests produced
+ * by the build worker.
+ */
+export class DuckLakeManifestStore implements ManifestStore {
+   private readonly table: string;
+
+   constructor(
+      private db: DuckDBConnection,
+      catalogName: string,
+   ) {
+      this.table = `${catalogName}.build_manifests`;
+   }
+
+   /**
+    * Idempotently creates the `build_manifests` table and indices in the
+    * DuckLake catalog. Safe to call from every worker on startup.
+    */
+   async bootstrapSchema(): Promise<void> {
+      await this.db.run(`
+         CREATE TABLE IF NOT EXISTS ${this.table} (
+            id VARCHAR,
+            project_id VARCHAR NOT NULL,
+            package_name VARCHAR NOT NULL,
+            build_id VARCHAR NOT NULL,
+            table_name VARCHAR NOT NULL,
+            source_name VARCHAR NOT NULL,
+            connection_name VARCHAR NOT NULL,
+            created_at TIMESTAMP NOT NULL,
+            updated_at TIMESTAMP NOT NULL
+         )
+      `);
+      logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
+   }
+
+   async getManifest(
+      projectId: string,
+      packageName: string,
+   ): Promise<BuildManifest> {
+      const rows = await this.db.all<Record<string, unknown>>(
+         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
+         [projectId, packageName],
+      );
+      const manifest: BuildManifest = { entries: {}, strict: false };
+      for (const row of rows) {
+         const buildId = row.build_id as string;
+         // Rows are ordered newest-first; keep only the latest per build_id
+         // to handle rare duplicates from cross-worker races.
+         if (!manifest.entries[buildId]) {
+            manifest.entries[buildId] = {
+               tableName: row.table_name as string,
+            };
+         }
+      }
+      return manifest;
+   }
+
+   /**
+    * Insert a manifest entry. If a row with the same `build_id` already
+    * exists (retry after crash), the duplicate is harmless:
+    * {@link getManifest} deduplicates by build_id keeping the newest row.
+    */
+   async writeEntry(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+      tableName: string,
+      sourceName: string,
+      connectionName: string,
+   ): Promise<void> {
+      const now = new Date().toISOString();
+      const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+
+      await this.db.run(
+         `INSERT INTO ${this.table} (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+         [
+            id,
+            projectId,
+            packageName,
+            buildId,
+            tableName,
+            sourceName,
+            connectionName,
+            now,
+            now,
+         ],
+      );
+   }
+
+   async deleteEntry(id: string): Promise<void> {
+      await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
+   }
+
+   async listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      const rows = await this.db.all<Record<string, unknown>>(
+         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
+         [projectId, packageName],
+      );
+      return rows.map(this.mapToEntry);
+   }
+
+   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
+      return {
+         id: row.id as string,
+         projectId: row.project_id as string,
+         packageName: row.package_name as string,
+         buildId: row.build_id as string,
+         tableName: row.table_name as string,
+         sourceName: row.source_name as string,
+         connectionName: row.connection_name as string,
+         createdAt: new Date(row.created_at as string),
+         updatedAt: new Date(row.updated_at as string),
+      };
+   }
+}

package/tests/fixtures/persist-test/data/orders.csv

@@ -0,0 +1,5 @@
+category,amount
+electronics,100
+clothing,50
+electronics,200
+clothing,75

package/tests/fixtures/persist-test/persist_test.malloy

@@ -0,0 +1,11 @@
+##! experimental.persistence
+
+source: raw_orders is duckdb.table('data/orders.csv')
+
+#@ persist name="order_summary"
+source: order_summary is raw_orders -> {
+  group_by: category
+  aggregate:
+    total_orders is count()
+    total_revenue is amount.sum()
+}

package/tests/fixtures/persist-test/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "persist-test",
+  "version": "1.0.0",
+  "description": "DuckDB persist test fixture"
+}

package/tests/fixtures/publisher.config.json

@@ -0,0 +1,15 @@
+{
+  "frozenConfig": false,
+  "projects": [
+    {
+      "name": "test-project",
+      "packages": [
+        {
+          "name": "persist-test",
+          "location": "./persist-test"
+        }
+      ],
+      "connections": []
+    }
+  ]
+}

package/tests/harness/rest_e2e.ts

@@ -0,0 +1,68 @@
+import http from "http";
+
+export interface RestE2EEnv {
+   httpServer: http.Server;
+   baseUrl: string;
+}
+
+/**
+ * Spin up an HTTP server wrapping the real Express REST app.
+ *
+ * Works regardless of which test file first imported server.ts —
+ * reuses the cached Express app and binds on an OS-assigned port
+ * to avoid collisions.
+ *
+ * Callers are responsible for creating any test-specific projects
+ * via the REST API (POST /api/v0/projects) and cleaning them up.
+ */
+export async function startRestE2E(): Promise<
+   RestE2EEnv & { stop(): Promise<void> }
+> {
+   const { app } = await import("../../src/server");
+
+   const httpServer: http.Server = await new Promise<http.Server>(
+      (resolve, reject) => {
+         const srv = http
+            .createServer(app)
+            .listen(0, "127.0.0.1", () => resolve(srv));
+         srv.on("error", (err: NodeJS.ErrnoException) => {
+            console.error("[REST E2E] server listen error", err);
+            reject(err);
+         });
+      },
+   );
+
+   const addr = httpServer.address() as { port: number };
+   const baseUrl = `http://127.0.0.1:${addr.port}`;
+
+   const maxWait = 180_000;
+   const start = Date.now();
+   let ready = false;
+   while (!ready && Date.now() - start < maxWait) {
+      try {
+         const res = await fetch(`${baseUrl}/health/readiness`);
+         if (res.ok) {
+            const data = (await res.json()) as { status: string };
+            if (data.status === "UP") {
+               ready = true;
+               break;
+            }
+         }
+      } catch {
+         // server not ready yet
+      }
+      await new Promise((r) => setTimeout(r, 500));
+   }
+   if (!ready) {
+      httpServer.closeAllConnections?.();
+      await new Promise<void>((r) => httpServer.close(() => r()));
+      throw new Error("REST E2E server did not become ready in time");
+   }
+
+   const stop = async (): Promise<void> => {
+      httpServer.closeAllConnections?.();
+      await new Promise<void>((r) => httpServer.close(() => r()));
+   };
+
+   return { httpServer, baseUrl, stop };
+}