@malloy-publisher/server 0.0.205 → 0.0.207

package/src/storage/StorageManager.ts

@@ -1,35 +1,11 @@
-import { Mutex } from "async-mutex";
-import * as crypto from "crypto";
-import {
-   isCatalogVersionSupported,
-   SUPPORTED_CATALOG_VERSIONS,
-} from "../ducklake_version";
-import { ConnectionAuthError } from "../errors";
 import { logger } from "../logger";
-import {
-   handlePgAttachError,
-   pgConnectTimeoutSeconds,
-   redactPgSecrets,
-   withPgConnectTimeout,
-} from "../pg_helpers";
-import {
-   DatabaseConnection,
-   ManifestStore,
-   ResourceRepository,
-} from "./DatabaseInterface";
+import { DatabaseConnection, ResourceRepository } from "./DatabaseInterface";
 import { DuckDBConnection } from "./duckdb/DuckDBConnection";
-import { DuckDBManifestStore } from "./duckdb/DuckDBManifestStore";
 import { DuckDBRepository } from "./duckdb/DuckDBRepository";
 import { initializeSchema } from "./duckdb/schema";
-import { DuckLakeManifestStore } from "./ducklake/DuckLakeManifestStore";
 
 export type StorageType = "duckdb" | "postgres" | "mysql";
 
-export interface DuckLakeManifestConfig {
-   catalogUrl: string;
-   dataPath: string;
-}
-
 export interface StorageConfig {
    type: StorageType;
    duckdb?: {
@@ -51,98 +27,16 @@ export interface StorageConfig {
    };
 }
 
-function escapeSQL(value: string): string {
-   return value.replace(/'/g, "''");
-}
-
-function configKey(c: DuckLakeManifestConfig): string {
-   return `${c.catalogUrl}|${c.dataPath}`;
-}
-
-function catalogNameForConfig(c: DuckLakeManifestConfig): string {
-   const hash = crypto
-      .createHash("sha256")
-      .update(configKey(c))
-      .digest("hex")
-      .slice(0, 8);
-   return `manifest_lake_${hash}`;
-}
-
-// Read the catalog's recorded DuckLake format version from its
-// `ducklake_metadata` table via a plain postgres ATTACH (does NOT invoke
-// the DuckLake extension on the catalog). Returns the version string on
-// success, or `undefined` on any failure (missing table, query timeout,
-// connect failure) so the main ATTACH path stays the source of truth for
-// unrelated errors. Only meaningful for postgres-backed catalogs; the
-// caller must guard with `isPostgres`.
-async function readDuckLakeCatalogVersion(
-   connection: DuckDBConnection,
-   catalogUrl: string,
-   catalogName: string,
-): Promise<string | undefined> {
-   if (!catalogUrl.startsWith("postgres:")) {
-      return undefined;
-   }
-   const pgConnString = catalogUrl.slice("postgres:".length);
-   const tempDb = `${catalogName}_preflight`;
-   const escaped = escapeSQL(pgConnString);
-   try {
-      await connection.run(
-         `ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`,
-      );
-      const rows = await connection.all<{ value: string }>(
-         `SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`,
-      );
-      const value = rows[0]?.value;
-      return typeof value === "string" ? value : undefined;
-   } catch (error) {
-      logger.warn(
-         "DuckLake catalog version preflight failed; falling back to ATTACH",
-         {
-            catalogName,
-            error: redactPgSecrets(
-               error instanceof Error ? error.message : String(error),
-            ),
-         },
-      );
-      return undefined;
-   } finally {
-      try {
-         await connection.run(`DETACH ${tempDb};`);
-      } catch {
-         // ATTACH may have failed, so DETACH may have nothing to do.
-      }
-   }
-}
-
 /**
- * Manages the storage backend (DuckDB, Postgres, etc.) and per-environment
- * manifest stores. Environments without `materializationStorage` config use
- * the default DuckDB manifest store. Environments with the config get a
- * DuckLake-backed store attached lazily on first access.
+ * Manages the storage backend (DuckDB today) and exposes the resource
+ * repository. The publisher persists only its own control records
+ * (environments, packages, connections, materializations) here; build
+ * manifests are returned inline on the materialization resource and persisted
+ * by the control plane.
  */
 export class StorageManager {
    private connection: DatabaseConnection | null = null;
-   private duckDbConnection: DuckDBConnection | null = null;
    private repository: ResourceRepository | null = null;
-   private defaultManifestStore: ManifestStore | null = null;
-
-   /** Per-environment DuckLake manifest stores, keyed by environmentId. */
-   private environmentManifestStores = new Map<string, ManifestStore>();
-
-   /**
-    * Tracks attached DuckLake catalogs as `configKey -> catalogName`. Each
-    * unique materializationStorage config gets its own ATTACHment under a
-    * deterministic catalog name, so multiple configs can coexist on one worker.
-    */
-   private attachedCatalogs = new Map<string, string>();
-
-   // Serializes DuckLake catalog attaches. Concurrent POST /environments calls
-   // hitting the same DuckDB connection would otherwise race on extension
-   // autoload (httpfs/azure/etc.), where multiple connections download the
-   // extension to `.tmp-<uuid>` files in parallel; only one wins the rename
-   // and the rest crash with "Could not remove file ... No such file or directory".
-   private duckLakeAttachMutex: Mutex = new Mutex();
 
    private config: StorageConfig;
 
@@ -173,166 +67,12 @@ export class StorageManager {
    }
 
    private async initializeDuckDB(reinit: boolean): Promise<void> {
-      const dbPath = this.config.duckdb?.path;
-
-      const connection = new DuckDBConnection(dbPath);
-
+      const connection = new DuckDBConnection(this.config.duckdb?.path);
       await connection.initialize();
-
       await initializeSchema(connection, reinit);
 
       this.connection = connection;
-      this.duckDbConnection = connection;
       this.repository = new DuckDBRepository(connection);
-      this.defaultManifestStore = new DuckDBManifestStore(this.repository);
-   }
-
-   /**
-    * Lazily initializes a DuckLake manifest store for an environment.
-    *
-    * One shared catalog per materializationStorage config: every environment
-    * pointing at the same (catalogUrl, dataPath) shares one `build_manifests`
-    * table inside it, partitioned by `environment_id` (set to the environment's name
-    * so it's stable across worker replicas — required for cross-pod manifest
-    * visibility in orchestrated mode). Different configs (e.g. different
-    * orgs) attach as separate catalogs under distinct deterministic aliases.
-    */
-   async initializeDuckLakeForEnvironment(
-      environmentId: string,
-      environmentName: string,
-      config: DuckLakeManifestConfig,
-   ): Promise<void> {
-      if (!this.duckDbConnection) {
-         throw new Error("Storage not initialized. Call initialize() first.");
-      }
-
-      const key = configKey(config);
-      const catalogName = await this.duckLakeAttachMutex.runExclusive(
-         async () => {
-            const existing = this.attachedCatalogs.get(key);
-            if (existing) return existing;
-            // Catalog name derived from the config so multiple configs can coexist as
-            // separate ATTACHments without colliding on the name.
-            const name = catalogNameForConfig(config);
-            await this.attachDuckLakeCatalog(config, name);
-            this.attachedCatalogs.set(key, name);
-            return name;
-         },
-      );
-
-      const store = new DuckLakeManifestStore(
-         this.duckDbConnection,
-         catalogName,
-         environmentName,
-      );
-      await store.bootstrapSchema();
-
-      this.environmentManifestStores.set(environmentId, store);
-      logger.info("DuckLake manifest store initialized for environment", {
-         environmentId,
-         environmentName,
-         catalogName,
-      });
-   }
-
-   private async attachDuckLakeCatalog(
-      config: DuckLakeManifestConfig,
-      catalogName: string,
-   ): Promise<void> {
-      const connection = this.duckDbConnection!;
-
-      await connection.run("INSTALL ducklake; LOAD ducklake;");
-
-      const isPostgres = config.catalogUrl.startsWith("postgres:");
-      if (isPostgres) {
-         await connection.run("INSTALL postgres; LOAD postgres;");
-      }
-
-      // For PG-backed catalogs, inject connect_timeout so a wedged libpq
-      // handshake fails the caller in seconds rather than hanging the
-      // worker until the K8s liveness probe trips (the 2026-05 incident).
-      // Non-PG catalogs (e.g. SQLite, MySQL) pass through unchanged.
-      const catalogUrl = isPostgres
-         ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds())
-         : config.catalogUrl;
-
-      const escapedCatalogUrl = escapeSQL(catalogUrl);
-      const escapedDataPath = escapeSQL(config.dataPath);
-      const isCloudStorage =
-         config.dataPath.startsWith("gs://") ||
-         config.dataPath.startsWith("s3://");
-
-      // Pre-install httpfs explicitly so the ATTACH below doesn't trigger
-      // DuckDB's autoloader. The autoloader downloads extensions to
-      // `<ext>.tmp-<uuid>` and races when multiple connections within the
-      // same process hit it concurrently — losers fail with
-      // "Could not remove file ... No such file or directory" on cleanup
-      // of their .tmp file. INSTALL/LOAD here is idempotent and serialized
-      // by the caller's mutex.
-      if (isCloudStorage) {
-         await connection.run("INSTALL httpfs; LOAD httpfs;");
-      }
-
-      // Preflight: read the catalog's recorded format version via the
-      // postgres extension (not DuckLake) and fail fast with a non-retryable
-      // 422 if the baked DuckLake extension can't read it. Without this,
-      // an unsupported catalog would surface as a generic DuckDB error
-      // from the ATTACH below, which retry loops misclassify as transient.
-      if (isPostgres) {
-         const catalogVersion = await readDuckLakeCatalogVersion(
-            connection,
-            catalogUrl,
-            catalogName,
-         );
-         if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
-            const supportedMax =
-               SUPPORTED_CATALOG_VERSIONS[
-                  SUPPORTED_CATALOG_VERSIONS.length - 1
-               ];
-            throw new ConnectionAuthError(
-               `DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`,
-            );
-         }
-      }
-
-      let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
-      const attachOpts: string[] = [
-         `DATA_PATH '${escapedDataPath}'`,
-         // The manifest table is small relational metadata (one row per build).
-         // Set a high inlining limit so writes always land transactionally in
-         // the postgres catalog rather than as parquet files in object storage,
-         // sidestepping object-storage auth issues entirely for this path.
-         "DATA_INLINING_ROW_LIMIT 100000",
-      ];
-
-      if (isCloudStorage) {
-         attachOpts.push("OVERRIDE_DATA_PATH true");
-      }
-      attachCmd += ` (${attachOpts.join(", ")});`;
-
-      logger.info(
-         `Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`,
-      );
-      try {
-         await connection.run(attachCmd);
-      } catch (error) {
-         const outcome = handlePgAttachError(
-            error,
-            `DuckLake catalog credentials rejected for ${catalogName}`,
-         );
-         if (outcome.action === "swallow") {
-            logger.info(
-               `DuckLake catalog ${catalogName} is already attached, skipping`,
-            );
-            return;
-         }
-         if (outcome.error instanceof ConnectionAuthError) {
-            logger.warn("DuckLake catalog credentials rejected", {
-               catalogName,
-            });
-         }
-         throw outcome.error;
-      }
    }
 
    getRepository(): ResourceRepository {
@@ -342,34 +82,11 @@ export class StorageManager {
       return this.repository;
    }
 
-   /**
-    * Returns the manifest store for an environment. If the environment has a
-    * DuckLake store configured, returns that; otherwise returns the
-    * default DuckDB-backed store.
-    */
-   getManifestStore(environmentId?: string): ManifestStore {
-      if (environmentId) {
-         const environmentStore =
-            this.environmentManifestStores.get(environmentId);
-         if (environmentStore) {
-            return environmentStore;
-         }
-      }
-      if (!this.defaultManifestStore) {
-         throw new Error("Storage not initialized. Call initialize() first.");
-      }
-      return this.defaultManifestStore;
-   }
-
    async close(): Promise<void> {
       if (this.connection) {
          await this.connection.close();
          this.connection = null;
-         this.duckDbConnection = null;
          this.repository = null;
-         this.defaultManifestStore = null;
-         this.environmentManifestStores.clear();
-         this.attachedCatalogs.clear();
       }
    }
 

package/src/storage/duckdb/DuckDBRepository.ts

@@ -1,16 +1,15 @@
 import {
    Connection,
    Environment,
-   ManifestEntry,
    Materialization,
    MaterializationStatus,
+   MaterializationUpdate,
    Package,
    ResourceRepository,
 } from "../DatabaseInterface";
 import { ConnectionRepository } from "./ConnectionRepository";
 import { DuckDBConnection } from "./DuckDBConnection";
 import { EnvironmentRepository } from "./EnvironmentRepository";
-import { ManifestRepository } from "./ManifestRepository";
 import { MaterializationRepository } from "./MaterializationRepository";
 import { PackageRepository } from "./PackageRepository";
 
@@ -19,14 +18,12 @@ export class DuckDBRepository implements ResourceRepository {
    private packageRepo: PackageRepository;
    private connectionRepo: ConnectionRepository;
    private materializationRepo: MaterializationRepository;
-   private manifestRepo: ManifestRepository;
 
    constructor(public db: DuckDBConnection) {
       this.environmentRepo = new EnvironmentRepository(db);
       this.packageRepo = new PackageRepository(db);
       this.connectionRepo = new ConnectionRepository(db);
       this.materializationRepo = new MaterializationRepository(db);
-      this.manifestRepo = new ManifestRepository(db);
    }
 
    // ==================== ENVIRONMENTS ====================
@@ -57,7 +54,6 @@ export class DuckDBRepository implements ResourceRepository {
    }
 
    async deleteEnvironment(id: string): Promise<void> {
-      await this.manifestRepo.deleteEntriesByEnvironmentId(id);
       await this.materializationRepo.deleteByEnvironmentId(id);
       await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
       await this.packageRepo.deletePackagesByEnvironmentId(id);
@@ -97,10 +93,6 @@ export class DuckDBRepository implements ResourceRepository {
    async deletePackage(id: string): Promise<void> {
       const pkg = await this.packageRepo.getPackageById(id);
       if (pkg) {
-         await this.manifestRepo.deleteEntriesByPackage(
-            pkg.environmentId,
-            pkg.name,
-         );
          await this.materializationRepo.deleteByPackage(
             pkg.environmentId,
             pkg.name,
@@ -188,13 +180,7 @@ export class DuckDBRepository implements ResourceRepository {
 
    async updateMaterialization(
       id: string,
-      updates: {
-         status?: MaterializationStatus;
-         startedAt?: Date;
-         completedAt?: Date;
-         error?: string | null;
-         metadata?: Record<string, unknown> | null;
-      },
+      updates: MaterializationUpdate,
    ): Promise<Materialization> {
       return this.materializationRepo.update(id, updates);
    }
@@ -202,23 +188,4 @@ export class DuckDBRepository implements ResourceRepository {
    async deleteMaterialization(id: string): Promise<void> {
       return this.materializationRepo.deleteById(id);
    }
-
-   // ==================== BUILD MANIFESTS ====================
-
-   async listManifestEntries(
-      environmentId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      return this.manifestRepo.listEntries(environmentId, packageName);
-   }
-
-   async upsertManifestEntry(
-      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
-   ): Promise<ManifestEntry> {
-      return this.manifestRepo.upsertEntry(entry);
-   }
-
-   async deleteManifestEntry(id: string): Promise<void> {
-      return this.manifestRepo.deleteEntry(id);
-   }
 }

package/src/storage/duckdb/MaterializationRepository.ts

@@ -1,12 +1,25 @@
-import { Materialization, MaterializationStatus } from "../DatabaseInterface";
+import {
+   BuildManifestResult,
+   BuildPlan,
+   Materialization,
+   MaterializationStatus,
+   MaterializationUpdate,
+} from "../DatabaseInterface";
 import { DuckDBConnection } from "./DuckDBConnection";
 
 const TERMINAL_STATUSES: ReadonlySet<MaterializationStatus> = new Set([
-   "SUCCESS",
+   "MANIFEST_FILE_READY",
    "FAILED",
    "CANCELLED",
 ]);
 
+/** Non-terminal statuses count as "active" for the single-active guard. */
+const ACTIVE_STATUSES: readonly MaterializationStatus[] = [
+   "PENDING",
+   "BUILD_PLAN_READY",
+   "MANIFEST_ROWS_READY",
+];
+
 function activeKeyFor(environmentId: string, packageName: string): string {
    return `${environmentId}|${packageName}`;
 }
@@ -73,9 +86,10 @@ export class MaterializationRepository {
       environmentId: string,
       packageName: string,
    ): Promise<Materialization | null> {
+      const placeholders = ACTIVE_STATUSES.map(() => "?").join(", ");
       const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN ('PENDING', 'RUNNING')",
-         [environmentId, packageName],
+         `SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN (${placeholders})`,
+         [environmentId, packageName, ...ACTIVE_STATUSES],
       );
       return row ? this.mapRow(row) : null;
    }
@@ -100,8 +114,8 @@ export class MaterializationRepository {
 
       try {
          const rows = await this.db.all<Record<string, unknown>>(
-            `INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, created_at, updated_at)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            `INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
             RETURNING *`,
             [
                id,
@@ -128,13 +142,7 @@ export class MaterializationRepository {
 
    async update(
       id: string,
-      updates: {
-         status?: MaterializationStatus;
-         startedAt?: Date;
-         completedAt?: Date;
-         error?: string | null;
-         metadata?: Record<string, unknown> | null;
-      },
+      updates: MaterializationUpdate,
    ): Promise<Materialization> {
       const now = this.now();
       const setClauses: string[] = [];
@@ -173,6 +181,18 @@ export class MaterializationRepository {
             updates.metadata ? JSON.stringify(updates.metadata) : null,
          );
       }
+      if (updates.buildPlan !== undefined) {
+         setClauses.push(`build_plan = ?`);
+         params.push(
+            updates.buildPlan ? JSON.stringify(updates.buildPlan) : null,
+         );
+      }
+      if (updates.manifest !== undefined) {
+         setClauses.push(`manifest = ?`);
+         params.push(
+            updates.manifest ? JSON.stringify(updates.manifest) : null,
+         );
+      }
 
       setClauses.push(`updated_at = ?`);
       params.push(now.toISOString());
@@ -190,6 +210,10 @@ export class MaterializationRepository {
       return updated;
    }
 
+   async deleteById(id: string): Promise<void> {
+      await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
+   }
+
    async deleteByEnvironmentId(environmentId: string): Promise<void> {
       await this.db.run(
          "DELETE FROM materializations WHERE environment_id = ?",
@@ -197,10 +221,6 @@ export class MaterializationRepository {
       );
    }
 
-   async deleteById(id: string): Promise<void> {
-      await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
-   }
-
    async deleteByPackage(
       environmentId: string,
       packageName: string,
@@ -212,32 +232,37 @@ export class MaterializationRepository {
    }
 
    private mapRow(row: Record<string, unknown>): Materialization {
-      let metadata: Record<string, unknown> | null = null;
-      if (row.metadata) {
-         try {
-            metadata = JSON.parse(row.metadata as string);
-         } catch {
-            metadata = null;
-         }
-      }
-
+      const metadata = parseJsonColumn<Record<string, unknown>>(row.metadata);
       return {
          id: row.id as string,
          environmentId: row.environment_id as string,
          packageName: row.package_name as string,
+         pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
          status: row.status as MaterializationStatus,
+         buildPlan: parseJsonColumn<BuildPlan>(row.build_plan),
+         manifest: parseJsonColumn<BuildManifestResult>(row.manifest),
+         metadata,
          startedAt: row.started_at ? new Date(row.started_at as string) : null,
          completedAt: row.completed_at
             ? new Date(row.completed_at as string)
             : null,
          error: row.error != null ? (row.error as string) : null,
-         metadata,
          createdAt: new Date(row.created_at as string),
          updatedAt: new Date(row.updated_at as string),
       };
    }
 }
 
+/** Parse a nullable JSON text column, returning null on absence or parse error. */
+function parseJsonColumn<T>(value: unknown): T | null {
+   if (value == null) return null;
+   try {
+      return JSON.parse(value as string) as T;
+   } catch {
+      return null;
+   }
+}
+
 /**
  * DuckDB surfaces unique-constraint violations as plain Errors whose message
  * mentions the violated index. We match on the index name rather than a

package/src/storage/duckdb/schema.ts

@@ -82,6 +82,8 @@ export async function initializeSchema(
    // `active_key` (see below) makes the insert-then-check race impossible —
    // a second concurrent create fails with a constraint violation, which the
    // service layer translates to `MaterializationConflictError`.
+   // `build_plan` (Round 1) and `manifest` (Round 2) are JSON blobs holding
+   // the two-round protocol payloads returned inline on the resource.
    await db.run(`
     CREATE TABLE IF NOT EXISTS materializations (
       id VARCHAR PRIMARY KEY,
@@ -93,29 +95,14 @@ export async function initializeSchema(
       completed_at TIMESTAMP,
       error TEXT,
       metadata JSON,
+      build_plan JSON,
+      manifest JSON,
       created_at TIMESTAMP NOT NULL,
       updated_at TIMESTAMP NOT NULL,
       FOREIGN KEY (environment_id) REFERENCES environments(id)
     )
   `);
 
-   // Build manifests table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS build_manifests (
-      id VARCHAR PRIMARY KEY,
-      environment_id VARCHAR NOT NULL,
-      package_name VARCHAR NOT NULL,
-      build_id VARCHAR NOT NULL,
-      table_name VARCHAR NOT NULL,
-      source_name VARCHAR NOT NULL,
-      connection_name VARCHAR NOT NULL,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (environment_id) REFERENCES environments(id),
-      UNIQUE (environment_id, package_name, build_id)
-    )
-  `);
-
    // Create indexes for better query performance
    await db.run(
       "CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)",
@@ -129,9 +116,6 @@ export async function initializeSchema(
    await db.run(
       "CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
    );
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)",
-   );
 }
 
 // TODO: Remove this during projects cleanup