@malloy-publisher/server 0.0.204 → 0.0.206

package/src/storage/StorageManager.ts

@@ -1,35 +1,11 @@
-import { Mutex } from "async-mutex";
-import * as crypto from "crypto";
-import {
-   isCatalogVersionSupported,
-   SUPPORTED_CATALOG_VERSIONS,
-} from "../ducklake_version";
-import { ConnectionAuthError } from "../errors";
 import { logger } from "../logger";
-import {
-   handlePgAttachError,
-   pgConnectTimeoutSeconds,
-   redactPgSecrets,
-   withPgConnectTimeout,
-} from "../pg_helpers";
-import {
-   DatabaseConnection,
-   ManifestStore,
-   ResourceRepository,
-} from "./DatabaseInterface";
+import { DatabaseConnection, ResourceRepository } from "./DatabaseInterface";
 import { DuckDBConnection } from "./duckdb/DuckDBConnection";
-import { DuckDBManifestStore } from "./duckdb/DuckDBManifestStore";
 import { DuckDBRepository } from "./duckdb/DuckDBRepository";
 import { initializeSchema } from "./duckdb/schema";
-import { DuckLakeManifestStore } from "./ducklake/DuckLakeManifestStore";
 
 export type StorageType = "duckdb" | "postgres" | "mysql";
 
-export interface DuckLakeManifestConfig {
-   catalogUrl: string;
-   dataPath: string;
-}
-
 export interface StorageConfig {
    type: StorageType;
    duckdb?: {
@@ -51,98 +27,16 @@ export interface StorageConfig {
    };
 }
 
-function escapeSQL(value: string): string {
-   return value.replace(/'/g, "''");
-}
-
-function configKey(c: DuckLakeManifestConfig): string {
-   return `${c.catalogUrl}|${c.dataPath}`;
-}
-
-function catalogNameForConfig(c: DuckLakeManifestConfig): string {
-   const hash = crypto
-      .createHash("sha256")
-      .update(configKey(c))
-      .digest("hex")
-      .slice(0, 8);
-   return `manifest_lake_${hash}`;
-}
-
-// Read the catalog's recorded DuckLake format version from its
-// `ducklake_metadata` table via a plain postgres ATTACH (does NOT invoke
-// the DuckLake extension on the catalog). Returns the version string on
-// success, or `undefined` on any failure (missing table, query timeout,
-// connect failure) so the main ATTACH path stays the source of truth for
-// unrelated errors. Only meaningful for postgres-backed catalogs; the
-// caller must guard with `isPostgres`.
-async function readDuckLakeCatalogVersion(
-   connection: DuckDBConnection,
-   catalogUrl: string,
-   catalogName: string,
-): Promise<string | undefined> {
-   if (!catalogUrl.startsWith("postgres:")) {
-      return undefined;
-   }
-   const pgConnString = catalogUrl.slice("postgres:".length);
-   const tempDb = `${catalogName}_preflight`;
-   const escaped = escapeSQL(pgConnString);
-   try {
-      await connection.run(
-         `ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`,
-      );
-      const rows = await connection.all<{ value: string }>(
-         `SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`,
-      );
-      const value = rows[0]?.value;
-      return typeof value === "string" ? value : undefined;
-   } catch (error) {
-      logger.warn(
-         "DuckLake catalog version preflight failed; falling back to ATTACH",
-         {
-            catalogName,
-            error: redactPgSecrets(
-               error instanceof Error ? error.message : String(error),
-            ),
-         },
-      );
-      return undefined;
-   } finally {
-      try {
-         await connection.run(`DETACH ${tempDb};`);
-      } catch {
-         // ATTACH may have failed, so DETACH may have nothing to do.
-      }
-   }
-}
-
 /**
- * Manages the storage backend (DuckDB, Postgres, etc.) and per-environment
- * manifest stores. Environments without `materializationStorage` config use
- * the default DuckDB manifest store. Environments with the config get a
- * DuckLake-backed store attached lazily on first access.
+ * Manages the storage backend (DuckDB today) and exposes the resource
+ * repository. The publisher persists only its own control records
+ * (environments, packages, connections, materializations) here; build
+ * manifests are returned inline on the materialization resource and persisted
+ * by the control plane.
  */
 export class StorageManager {
    private connection: DatabaseConnection | null = null;
-   private duckDbConnection: DuckDBConnection | null = null;
    private repository: ResourceRepository | null = null;
-   private defaultManifestStore: ManifestStore | null = null;
-
-   /** Per-environment DuckLake manifest stores, keyed by environmentId. */
-   private environmentManifestStores = new Map<string, ManifestStore>();
-
-   /**
-    * Tracks attached DuckLake catalogs as `configKey -> catalogName`. Each
-    * unique materializationStorage config gets its own ATTACHment under a
-    * deterministic catalog name, so multiple configs can coexist on one worker.
-    */
-   private attachedCatalogs = new Map<string, string>();
-
-   // Serializes DuckLake catalog attaches. Concurrent POST /environments calls
-   // hitting the same DuckDB connection would otherwise race on extension
-   // autoload (httpfs/azure/etc.), where multiple connections download the
-   // extension to `.tmp-<uuid>` files in parallel; only one wins the rename
-   // and the rest crash with "Could not remove file ... No such file or directory".
-   private duckLakeAttachMutex: Mutex = new Mutex();
 
    private config: StorageConfig;
 
@@ -173,166 +67,12 @@ export class StorageManager {
    }
 
    private async initializeDuckDB(reinit: boolean): Promise<void> {
-      const dbPath = this.config.duckdb?.path;
-
-      const connection = new DuckDBConnection(dbPath);
-
+      const connection = new DuckDBConnection(this.config.duckdb?.path);
       await connection.initialize();
-
       await initializeSchema(connection, reinit);
 
       this.connection = connection;
-      this.duckDbConnection = connection;
       this.repository = new DuckDBRepository(connection);
-      this.defaultManifestStore = new DuckDBManifestStore(this.repository);
-   }
-
-   /**
-    * Lazily initializes a DuckLake manifest store for an environment.
-    *
-    * One shared catalog per materializationStorage config: every environment
-    * pointing at the same (catalogUrl, dataPath) shares one `build_manifests`
-    * table inside it, partitioned by `environment_id` (set to the environment's name
-    * so it's stable across worker replicas — required for cross-pod manifest
-    * visibility in orchestrated mode). Different configs (e.g. different
-    * orgs) attach as separate catalogs under distinct deterministic aliases.
-    */
-   async initializeDuckLakeForEnvironment(
-      environmentId: string,
-      environmentName: string,
-      config: DuckLakeManifestConfig,
-   ): Promise<void> {
-      if (!this.duckDbConnection) {
-         throw new Error("Storage not initialized. Call initialize() first.");
-      }
-
-      const key = configKey(config);
-      const catalogName = await this.duckLakeAttachMutex.runExclusive(
-         async () => {
-            const existing = this.attachedCatalogs.get(key);
-            if (existing) return existing;
-            // Catalog name derived from the config so multiple configs can coexist as
-            // separate ATTACHments without colliding on the name.
-            const name = catalogNameForConfig(config);
-            await this.attachDuckLakeCatalog(config, name);
-            this.attachedCatalogs.set(key, name);
-            return name;
-         },
-      );
-
-      const store = new DuckLakeManifestStore(
-         this.duckDbConnection,
-         catalogName,
-         environmentName,
-      );
-      await store.bootstrapSchema();
-
-      this.environmentManifestStores.set(environmentId, store);
-      logger.info("DuckLake manifest store initialized for environment", {
-         environmentId,
-         environmentName,
-         catalogName,
-      });
-   }
-
-   private async attachDuckLakeCatalog(
-      config: DuckLakeManifestConfig,
-      catalogName: string,
-   ): Promise<void> {
-      const connection = this.duckDbConnection!;
-
-      await connection.run("INSTALL ducklake; LOAD ducklake;");
-
-      const isPostgres = config.catalogUrl.startsWith("postgres:");
-      if (isPostgres) {
-         await connection.run("INSTALL postgres; LOAD postgres;");
-      }
-
-      // For PG-backed catalogs, inject connect_timeout so a wedged libpq
-      // handshake fails the caller in seconds rather than hanging the
-      // worker until the K8s liveness probe trips (the 2026-05 incident).
-      // Non-PG catalogs (e.g. SQLite, MySQL) pass through unchanged.
-      const catalogUrl = isPostgres
-         ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds())
-         : config.catalogUrl;
-
-      const escapedCatalogUrl = escapeSQL(catalogUrl);
-      const escapedDataPath = escapeSQL(config.dataPath);
-      const isCloudStorage =
-         config.dataPath.startsWith("gs://") ||
-         config.dataPath.startsWith("s3://");
-
-      // Pre-install httpfs explicitly so the ATTACH below doesn't trigger
-      // DuckDB's autoloader. The autoloader downloads extensions to
-      // `<ext>.tmp-<uuid>` and races when multiple connections within the
-      // same process hit it concurrently — losers fail with
-      // "Could not remove file ... No such file or directory" on cleanup
-      // of their .tmp file. INSTALL/LOAD here is idempotent and serialized
-      // by the caller's mutex.
-      if (isCloudStorage) {
-         await connection.run("INSTALL httpfs; LOAD httpfs;");
-      }
-
-      // Preflight: read the catalog's recorded format version via the
-      // postgres extension (not DuckLake) and fail fast with a non-retryable
-      // 422 if the baked DuckLake extension can't read it. Without this,
-      // an unsupported catalog would surface as a generic DuckDB error
-      // from the ATTACH below, which retry loops misclassify as transient.
-      if (isPostgres) {
-         const catalogVersion = await readDuckLakeCatalogVersion(
-            connection,
-            catalogUrl,
-            catalogName,
-         );
-         if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
-            const supportedMax =
-               SUPPORTED_CATALOG_VERSIONS[
-                  SUPPORTED_CATALOG_VERSIONS.length - 1
-               ];
-            throw new ConnectionAuthError(
-               `DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`,
-            );
-         }
-      }
-
-      let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
-      const attachOpts: string[] = [
-         `DATA_PATH '${escapedDataPath}'`,
-         // The manifest table is small relational metadata (one row per build).
-         // Set a high inlining limit so writes always land transactionally in
-         // the postgres catalog rather than as parquet files in object storage,
-         // sidestepping object-storage auth issues entirely for this path.
-         "DATA_INLINING_ROW_LIMIT 100000",
-      ];
-
-      if (isCloudStorage) {
-         attachOpts.push("OVERRIDE_DATA_PATH true");
-      }
-      attachCmd += ` (${attachOpts.join(", ")});`;
-
-      logger.info(
-         `Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`,
-      );
-      try {
-         await connection.run(attachCmd);
-      } catch (error) {
-         const outcome = handlePgAttachError(
-            error,
-            `DuckLake catalog credentials rejected for ${catalogName}`,
-         );
-         if (outcome.action === "swallow") {
-            logger.info(
-               `DuckLake catalog ${catalogName} is already attached, skipping`,
-            );
-            return;
-         }
-         if (outcome.error instanceof ConnectionAuthError) {
-            logger.warn("DuckLake catalog credentials rejected", {
-               catalogName,
-            });
-         }
-         throw outcome.error;
-      }
    }
 
    getRepository(): ResourceRepository {
@@ -342,34 +82,11 @@ export class StorageManager {
       return this.repository;
    }
 
-   /**
-    * Returns the manifest store for an environment. If the environment has a
-    * DuckLake store configured, returns that; otherwise returns the
-    * default DuckDB-backed store.
-    */
-   getManifestStore(environmentId?: string): ManifestStore {
-      if (environmentId) {
-         const environmentStore =
-            this.environmentManifestStores.get(environmentId);
-         if (environmentStore) {
-            return environmentStore;
-         }
-      }
-      if (!this.defaultManifestStore) {
-         throw new Error("Storage not initialized. Call initialize() first.");
-      }
-      return this.defaultManifestStore;
-   }
-
    async close(): Promise<void> {
       if (this.connection) {
          await this.connection.close();
          this.connection = null;
-         this.duckDbConnection = null;
          this.repository = null;
-         this.defaultManifestStore = null;
-         this.environmentManifestStores.clear();
-         this.attachedCatalogs.clear();
       }
    }
 

package/src/storage/duckdb/DuckDBConnection.ts

@@ -1,11 +1,30 @@
 import { Mutex } from "async-mutex";
-import duckdb from "duckdb";
+import {
+   DuckDBConnection as NeoConnection,
+   DuckDBInstance,
+   type DuckDBValue,
+} from "@duckdb/node-api";
 import * as path from "path";
 import { DatabaseConnection } from "../DatabaseInterface";
 
+/**
+ * Embedded persistence layer for the publisher's own metadata (environments,
+ * packages, connections, materializations, build manifests) in `publisher.db`.
+ *
+ * This is a plain DAO over a durable, exclusively-owned DuckDB handle -- it is
+ * deliberately NOT Malloy's `@malloydata/db-duckdb` connection, which is an
+ * analytical query connection (no prepared-statement parameter binding, a
+ * `:memory:` primary with ATTACH/DETACH/idle lifecycle, pooled/shared
+ * instances, and a poison-pill close). Those semantics are wrong for a
+ * source-of-truth store that must hold one handle open for the server's
+ * lifetime and run parameterized CRUD.
+ *
+ * It wraps `@duckdb/node-api` (the same DuckDB engine Malloy pulls in), so the
+ * repo carries a single DuckDB engine rather than a second, redundant driver.
+ */
 export class DuckDBConnection implements DatabaseConnection {
-   private db: duckdb.Database | null = null;
-   private connection: duckdb.Connection | null = null;
+   private instance: DuckDBInstance | null = null;
+   private connection: NeoConnection | null = null;
    private dbPath: string;
    private mutex: Mutex = new Mutex();
 
@@ -15,94 +34,47 @@ export class DuckDBConnection implements DatabaseConnection {
    }
 
    async initialize(): Promise<void> {
-      return new Promise((resolve, reject) => {
-         this.db = new duckdb.Database(this.dbPath, {}, (err) => {
-            if (err) {
-               console.error("Failed to create DuckDB database:", err);
-               reject(new Error(`Failed to initialize DuckDB: ${err.message}`));
-               return;
-            }
-
-            // Connect synchronously
-            this.connection = (
-               this.db as duckdb.Database & { connect(): duckdb.Connection }
-            ).connect();
-
-            if (!this.connection) {
-               reject(new Error("Failed to create connection object"));
-               return;
-            }
-
-            // Verify connection works
-            this.connection.all("SELECT 42 as answer", (testErr, _rows) => {
-               if (testErr) {
-                  console.error("Connection test failed:", testErr);
-                  reject(
-                     new Error(
-                        `Failed to verify DuckDB connection: ${testErr.message}`,
-                     ),
-                  );
-                  return;
-               }
-
-               resolve();
-            });
-         });
-      });
+      try {
+         this.instance = await DuckDBInstance.create(this.dbPath);
+         this.connection = await this.instance.connect();
+         // Verify the connection works
+         await this.connection.run("SELECT 42 as answer");
+      } catch (err) {
+         const message = err instanceof Error ? err.message : String(err);
+         console.error("Failed to create DuckDB database:", err);
+         throw new Error(`Failed to initialize DuckDB: ${message}`);
+      }
    }
 
    async close(): Promise<void> {
-      return new Promise((resolve, reject) => {
+      try {
          if (this.connection) {
-            this.connection.close((err) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Failed to close DuckDB connection: ${err.message}`,
-                     ),
-                  );
-                  return;
-               }
-
-               if (this.db) {
-                  this.db.close((dbErr) => {
-                     if (dbErr) {
-                        reject(
-                           new Error(
-                              `Failed to close DuckDB: ${dbErr.message}`,
-                           ),
-                        );
-                        return;
-                     }
-                     console.log("DuckDB connection closed");
-                     resolve();
-                  });
-               } else {
-                  resolve();
-               }
-            });
-         } else {
-            resolve();
+            this.connection.closeSync();
+            this.connection = null;
          }
-      });
+         if (this.instance) {
+            this.instance.closeSync();
+            this.instance = null;
+         }
+         console.log("DuckDB connection closed");
+      } catch (err) {
+         const message = err instanceof Error ? err.message : String(err);
+         throw new Error(`Failed to close DuckDB connection: ${message}`);
+      }
    }
 
    async isInitialized(): Promise<boolean> {
       if (!this.connection) return false;
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<boolean>((resolve) => {
-            this.connection!.all(
+         try {
+            const reader = await this.connection!.runAndReadAll(
                "SELECT name FROM sqlite_master WHERE type='table' AND name='environments'",
-               (err, rows) => {
-                  if (err) {
-                     resolve(false);
-                     return;
-                  }
-                  resolve(rows && rows.length > 0);
-               },
             );
-         });
+            return reader.getRowObjectsJS().length > 0;
+         } catch {
+            return false;
+         }
       });
    }
 
@@ -112,26 +84,14 @@ export class DuckDBConnection implements DatabaseConnection {
       }
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<void>((resolve, reject) => {
-            const callback = (err: Error | null) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Query execution failed: ${err.message}\nQuery: ${query}`,
-                     ),
-                  );
-                  return;
-               }
-               resolve();
-            };
-
-            // Pass params directly without the params argument if empty
-            if (params && params.length > 0) {
-               this.connection!.run(query, ...params, callback);
-            } else {
-               this.connection!.run(query, callback);
-            }
-         });
+         try {
+            await this.connection!.run(query, params as DuckDBValue[]);
+         } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new Error(
+               `Query execution failed: ${message}\nQuery: ${query}`,
+            );
+         }
       });
    }
 
@@ -141,25 +101,18 @@ export class DuckDBConnection implements DatabaseConnection {
       }
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<T[]>((resolve, reject) => {
-            const callback = (err: Error | null, rows: unknown[]) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Query execution failed: ${err.message}\nQuery: ${query}`,
-                     ),
-                  );
-                  return;
-               }
-               resolve((rows || []) as T[]);
-            };
-
-            if (params && params.length > 0) {
-               this.connection!.all(query, ...params, callback);
-            } else {
-               this.connection!.all(query, callback);
-            }
-         });
+         try {
+            const reader = await this.connection!.runAndReadAll(
+               query,
+               params as DuckDBValue[],
+            );
+            return reader.getRowObjectsJS() as T[];
+         } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new Error(
+               `Query execution failed: ${message}\nQuery: ${query}`,
+            );
+         }
       });
    }
 
@@ -167,11 +120,4 @@ export class DuckDBConnection implements DatabaseConnection {
       const rows = await this.all<T>(query, params);
       return rows.length > 0 ? rows[0] : null;
    }
-
-   getConnection(): duckdb.Connection {
-      if (!this.connection) {
-         throw new Error("Database not initialized");
-      }
-      return this.connection;
-   }
 }

package/src/storage/duckdb/DuckDBRepository.ts

@@ -1,16 +1,15 @@
 import {
    Connection,
    Environment,
-   ManifestEntry,
    Materialization,
    MaterializationStatus,
+   MaterializationUpdate,
    Package,
    ResourceRepository,
 } from "../DatabaseInterface";
 import { ConnectionRepository } from "./ConnectionRepository";
 import { DuckDBConnection } from "./DuckDBConnection";
 import { EnvironmentRepository } from "./EnvironmentRepository";
-import { ManifestRepository } from "./ManifestRepository";
 import { MaterializationRepository } from "./MaterializationRepository";
 import { PackageRepository } from "./PackageRepository";
 
@@ -19,14 +18,12 @@ export class DuckDBRepository implements ResourceRepository {
    private packageRepo: PackageRepository;
    private connectionRepo: ConnectionRepository;
    private materializationRepo: MaterializationRepository;
-   private manifestRepo: ManifestRepository;
 
    constructor(public db: DuckDBConnection) {
       this.environmentRepo = new EnvironmentRepository(db);
       this.packageRepo = new PackageRepository(db);
       this.connectionRepo = new ConnectionRepository(db);
       this.materializationRepo = new MaterializationRepository(db);
-      this.manifestRepo = new ManifestRepository(db);
    }
 
    // ==================== ENVIRONMENTS ====================
@@ -57,7 +54,6 @@ export class DuckDBRepository implements ResourceRepository {
    }
 
    async deleteEnvironment(id: string): Promise<void> {
-      await this.manifestRepo.deleteEntriesByEnvironmentId(id);
       await this.materializationRepo.deleteByEnvironmentId(id);
       await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
       await this.packageRepo.deletePackagesByEnvironmentId(id);
@@ -97,10 +93,6 @@ export class DuckDBRepository implements ResourceRepository {
    async deletePackage(id: string): Promise<void> {
       const pkg = await this.packageRepo.getPackageById(id);
       if (pkg) {
-         await this.manifestRepo.deleteEntriesByPackage(
-            pkg.environmentId,
-            pkg.name,
-         );
          await this.materializationRepo.deleteByPackage(
             pkg.environmentId,
             pkg.name,
@@ -188,13 +180,7 @@ export class DuckDBRepository implements ResourceRepository {
 
    async updateMaterialization(
       id: string,
-      updates: {
-         status?: MaterializationStatus;
-         startedAt?: Date;
-         completedAt?: Date;
-         error?: string | null;
-         metadata?: Record<string, unknown> | null;
-      },
+      updates: MaterializationUpdate,
    ): Promise<Materialization> {
       return this.materializationRepo.update(id, updates);
    }
@@ -202,23 +188,4 @@ export class DuckDBRepository implements ResourceRepository {
    async deleteMaterialization(id: string): Promise<void> {
       return this.materializationRepo.deleteById(id);
    }
-
-   // ==================== BUILD MANIFESTS ====================
-
-   async listManifestEntries(
-      environmentId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      return this.manifestRepo.listEntries(environmentId, packageName);
-   }
-
-   async upsertManifestEntry(
-      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
-   ): Promise<ManifestEntry> {
-      return this.manifestRepo.upsertEntry(entry);
-   }
-
-   async deleteManifestEntry(id: string): Promise<void> {
-      return this.manifestRepo.deleteEntry(id);
-   }
 }