@malloy-publisher/server 0.0.181 → 0.0.183-dev

package/src/storage/DatabaseInterface.ts

@@ -34,7 +34,7 @@ export interface ResourceRepository {
    getConnectionById(id: string): Promise<Connection | null>;
    getConnectionByName(
       projectId: string,
-      id: string,
+      name: string,
    ): Promise<Connection | null>;
    createConnection(
       connection: Omit<Connection, "id" | "createdAt" | "updatedAt">,
@@ -44,6 +44,44 @@ export interface ResourceRepository {
       updates: Partial<Connection>,
    ): Promise<Connection>;
    deleteConnection(id: string): Promise<void>;
+
+   // Materializations
+   listMaterializations(
+      projectId: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]>;
+   getMaterializationById(id: string): Promise<Materialization | null>;
+   getActiveMaterialization(
+      projectId: string,
+      packageName: string,
+   ): Promise<Materialization | null>;
+   createMaterialization(
+      projectId: string,
+      packageName: string,
+      status?: MaterializationStatus,
+      metadata?: Record<string, unknown> | null,
+   ): Promise<Materialization>;
+   updateMaterialization(
+      id: string,
+      updates: {
+         status?: MaterializationStatus;
+         startedAt?: Date;
+         completedAt?: Date;
+         error?: string | null;
+         metadata?: Record<string, unknown> | null;
+      },
+   ): Promise<Materialization>;
+   deleteMaterialization(id: string): Promise<void>;
+   // Build Manifests
+   listManifestEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]>;
+   upsertManifestEntry(
+      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
+   ): Promise<ManifestEntry>;
+   deleteManifestEntry(id: string): Promise<void>;
 }
 
 export interface Project {
@@ -76,3 +114,67 @@ export interface Connection {
    createdAt: Date;
    updatedAt: Date;
 }
+
+export type MaterializationStatus =
+   | "PENDING"
+   | "RUNNING"
+   | "SUCCESS"
+   | "FAILED"
+   | "CANCELLED";
+
+export interface Materialization {
+   id: string;
+   projectId: string;
+   packageName: string;
+   status: MaterializationStatus;
+   startedAt: Date | null;
+   completedAt: Date | null;
+   error: string | null;
+   metadata: Record<string, unknown> | null;
+   createdAt: Date;
+   updatedAt: Date;
+}
+
+export interface ManifestEntry {
+   id: string;
+   projectId: string;
+   packageName: string;
+   buildId: string;
+   tableName: string;
+   sourceName: string;
+   connectionName: string;
+   createdAt: Date;
+   updatedAt: Date;
+}
+
+// ==================== MANIFEST STORE ====================
+
+export interface BuildManifestEntry {
+   tableName: string;
+}
+
+export interface BuildManifest {
+   entries: Record<string, BuildManifestEntry>;
+   strict?: boolean;
+}
+
+/**
+ * Abstraction for manifest storage. Standalone mode uses DuckDB;
+ * orchestrated mode swaps in a DuckLakeManifestStore.
+ */
+export interface ManifestStore {
+   getManifest(projectId: string, packageName: string): Promise<BuildManifest>;
+   writeEntry(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+      tableName: string,
+      sourceName: string,
+      connectionName: string,
+   ): Promise<void>;
+   deleteEntry(id: string): Promise<void>;
+   listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]>;
+}

package/src/storage/{StorageManager.spec.ts → StorageManager.mock.ts}

@@ -47,4 +47,13 @@ export class StorageManager {
          deleteConnection: async (): Promise<void> => {},
       };
    }
+
+   getManifestStore() {
+      return {
+         getManifest: async () => ({ entries: {}, strict: false }),
+         writeEntry: async () => {},
+         deleteEntry: async () => {},
+         listEntries: async () => [],
+      };
+   }
 }

package/src/storage/StorageManager.ts

@@ -1,11 +1,22 @@
 import { logger } from "../logger";
-import { DatabaseConnection, ResourceRepository } from "./DatabaseInterface";
+import {
+   DatabaseConnection,
+   ManifestStore,
+   ResourceRepository,
+} from "./DatabaseInterface";
 import { DuckDBConnection } from "./duckdb/DuckDBConnection";
+import { DuckDBManifestStore } from "./duckdb/DuckDBManifestStore";
 import { DuckDBRepository } from "./duckdb/DuckDBRepository";
 import { initializeSchema } from "./duckdb/schema";
+import { DuckLakeManifestStore } from "./ducklake/DuckLakeManifestStore";
 
 export type StorageType = "duckdb" | "postgres" | "mysql";
 
+export interface DuckLakeManifestConfig {
+   catalogUrl: string;
+   dataPath: string;
+}
+
 export interface StorageConfig {
    type: StorageType;
    duckdb?: {
@@ -27,9 +38,28 @@ export interface StorageConfig {
    };
 }
 
+function escapeSQL(value: string): string {
+   return value.replace(/'/g, "''");
+}
+
+/**
+ * Manages the storage backend (DuckDB, Postgres, etc.) and per-project
+ * manifest stores. Projects without `materializationStorage` config use
+ * the default DuckDB manifest store. Projects with the config get a
+ * DuckLake-backed store attached lazily on first access.
+ */
 export class StorageManager {
    private connection: DatabaseConnection | null = null;
+   private duckDbConnection: DuckDBConnection | null = null;
    private repository: ResourceRepository | null = null;
+   private defaultManifestStore: ManifestStore | null = null;
+
+   /** Per-project DuckLake manifest stores, keyed by projectId. */
+   private projectManifestStores = new Map<string, ManifestStore>();
+
+   /** Tracks which catalogs have been attached to avoid duplicate ATTACHes. */
+   private attachedCatalogs = new Set<string>();
+
    private config: StorageConfig;
 
    constructor(config: StorageConfig) {
@@ -68,7 +98,73 @@ export class StorageManager {
       await initializeSchema(connection, reinit);
 
       this.connection = connection;
+      this.duckDbConnection = connection;
       this.repository = new DuckDBRepository(connection);
+      this.defaultManifestStore = new DuckDBManifestStore(this.repository);
+   }
+
+   /**
+    * Lazily initializes a DuckLake manifest store for a project.
+    * The DuckLake catalog is attached to the shared DuckDB connection
+    * and persists for the lifetime of the process.
+    */
+   async initializeDuckLakeForProject(
+      projectId: string,
+      config: DuckLakeManifestConfig,
+   ): Promise<void> {
+      if (!this.duckDbConnection) {
+         throw new Error("Storage not initialized. Call initialize() first.");
+      }
+
+      const catalogName = `manifest_lake_${projectId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+
+      if (!this.attachedCatalogs.has(catalogName)) {
+         await this.attachDuckLakeCatalog(config, catalogName);
+      }
+
+      const store = new DuckLakeManifestStore(
+         this.duckDbConnection,
+         catalogName,
+      );
+      await store.bootstrapSchema();
+
+      this.projectManifestStores.set(projectId, store);
+      logger.info("DuckLake manifest store initialized for project", {
+         projectId,
+         catalogName,
+      });
+   }
+
+   private async attachDuckLakeCatalog(
+      config: DuckLakeManifestConfig,
+      catalogName: string,
+   ): Promise<void> {
+      const connection = this.duckDbConnection!;
+
+      await connection.run("INSTALL ducklake; LOAD ducklake;");
+
+      const isPostgres = config.catalogUrl.startsWith("postgres:");
+      if (isPostgres) {
+         await connection.run("INSTALL postgres; LOAD postgres;");
+      }
+
+      const escapedCatalogUrl = escapeSQL(config.catalogUrl);
+      const escapedDataPath = escapeSQL(config.dataPath);
+      const isCloudStorage =
+         config.dataPath.startsWith("gs://") ||
+         config.dataPath.startsWith("s3://");
+
+      let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
+      const attachOpts: string[] = [`DATA_PATH '${escapedDataPath}'`];
+      if (isCloudStorage) {
+         attachOpts.push("OVERRIDE_DATA_PATH true");
+      }
+      attachCmd += ` (${attachOpts.join(", ")});`;
+
+      logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
+      await connection.run(attachCmd);
+
+      this.attachedCatalogs.add(catalogName);
    }
 
    getRepository(): ResourceRepository {
@@ -78,11 +174,33 @@ export class StorageManager {
       return this.repository;
    }
 
+   /**
+    * Returns the manifest store for a project. If the project has a
+    * DuckLake store configured, returns that; otherwise returns the
+    * default DuckDB-backed store.
+    */
+   getManifestStore(projectId?: string): ManifestStore {
+      if (projectId) {
+         const projectStore = this.projectManifestStores.get(projectId);
+         if (projectStore) {
+            return projectStore;
+         }
+      }
+      if (!this.defaultManifestStore) {
+         throw new Error("Storage not initialized. Call initialize() first.");
+      }
+      return this.defaultManifestStore;
+   }
+
    async close(): Promise<void> {
       if (this.connection) {
          await this.connection.close();
          this.connection = null;
+         this.duckDbConnection = null;
          this.repository = null;
+         this.defaultManifestStore = null;
+         this.projectManifestStores.clear();
+         this.attachedCatalogs.clear();
       }
    }
 

package/src/storage/duckdb/DuckDBConnection.ts

@@ -1,5 +1,5 @@
 import { Mutex } from "async-mutex";
-import * as duckdb from "duckdb";
+import duckdb from "duckdb";
 import * as path from "path";
 import { DatabaseConnection } from "../DatabaseInterface";
 

package/src/storage/duckdb/DuckDBManifestStore.ts

@@ -0,0 +1,70 @@
+import {
+   BuildManifest,
+   ManifestEntry,
+   ManifestStore,
+   ResourceRepository,
+} from "../DatabaseInterface";
+
+/**
+ * DuckDB-backed ManifestStore that delegates to the local ResourceRepository.
+ *
+ * In standalone mode this is the active store. Orchestrated deployments swap
+ * in a DuckLake-backed implementation that reads/writes manifests through the
+ * shared catalog instead of the local DuckDB build_manifests table.
+ */
+export class DuckDBManifestStore implements ManifestStore {
+   constructor(private repository: ResourceRepository) {}
+
+   /**
+    * Assembles a {@link BuildManifest} by folding all manifest rows for the
+    * given package into a build-ID-keyed map. `strict: false` lets the Malloy
+    * runtime fall back to executing the underlying query when a persist
+    * reference has no manifest entry (e.g. before the first materialization).
+    */
+   async getManifest(
+      projectId: string,
+      packageName: string,
+   ): Promise<BuildManifest> {
+      const entries = await this.repository.listManifestEntries(
+         projectId,
+         packageName,
+      );
+      const manifest: BuildManifest = {
+         entries: {},
+         strict: false,
+      };
+      for (const entry of entries) {
+         manifest.entries[entry.buildId] = { tableName: entry.tableName };
+      }
+      return manifest;
+   }
+
+   async writeEntry(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+      tableName: string,
+      sourceName: string,
+      connectionName: string,
+   ): Promise<void> {
+      await this.repository.upsertManifestEntry({
+         projectId,
+         packageName,
+         buildId,
+         tableName,
+         sourceName,
+         connectionName,
+      });
+   }
+
+   async deleteEntry(id: string): Promise<void> {
+      await this.repository.deleteManifestEntry(id);
+   }
+
+   async listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      return this.repository.listManifestEntries(projectId, packageName);
+   }
+}

package/src/storage/duckdb/DuckDBRepository.ts

@@ -1,23 +1,32 @@
 import {
-   ResourceRepository,
-   Project,
-   Package,
    Connection,
+   ManifestEntry,
+   Materialization,
+   MaterializationStatus,
+   Package,
+   Project,
+   ResourceRepository,
 } from "../DatabaseInterface";
+import { ConnectionRepository } from "./ConnectionRepository";
 import { DuckDBConnection } from "./DuckDBConnection";
-import { ProjectRepository } from "./ProjectRepository";
+import { ManifestRepository } from "./ManifestRepository";
+import { MaterializationRepository } from "./MaterializationRepository";
 import { PackageRepository } from "./PackageRepository";
-import { ConnectionRepository } from "./ConnectionRepository";
+import { ProjectRepository } from "./ProjectRepository";
 
 export class DuckDBRepository implements ResourceRepository {
    private projectRepo: ProjectRepository;
    private packageRepo: PackageRepository;
    private connectionRepo: ConnectionRepository;
+   private materializationRepo: MaterializationRepository;
+   private manifestRepo: ManifestRepository;
 
    constructor(public db: DuckDBConnection) {
       this.projectRepo = new ProjectRepository(db);
       this.packageRepo = new PackageRepository(db);
       this.connectionRepo = new ConnectionRepository(db);
+      this.materializationRepo = new MaterializationRepository(db);
+      this.manifestRepo = new ManifestRepository(db);
    }
 
    // ==================== PROJECTS ====================
@@ -48,11 +57,10 @@ export class DuckDBRepository implements ResourceRepository {
    }
 
    async deleteProject(id: string): Promise<void> {
-      // Delete related connections and packages first
+      await this.manifestRepo.deleteEntriesByProjectId(id);
+      await this.materializationRepo.deleteByProjectId(id);
       await this.connectionRepo.deleteConnectionsByProjectId(id);
       await this.packageRepo.deletePackagesByProjectId(id);
-
-      // Then delete the project
       await this.projectRepo.deleteProject(id);
    }
 
@@ -87,7 +95,18 @@ export class DuckDBRepository implements ResourceRepository {
    }
 
    async deletePackage(id: string): Promise<void> {
-      return this.packageRepo.deletePackage(id);
+      const pkg = await this.packageRepo.getPackageById(id);
+      if (pkg) {
+         await this.manifestRepo.deleteEntriesByPackage(
+            pkg.projectId,
+            pkg.name,
+         );
+         await this.materializationRepo.deleteByPackage(
+            pkg.projectId,
+            pkg.name,
+         );
+      }
+      await this.packageRepo.deletePackage(id);
    }
 
    async deletePackagesByProjectId(id: string): Promise<void> {
@@ -131,4 +150,75 @@ export class DuckDBRepository implements ResourceRepository {
    async deleteConnectionsByProjectId(id: string): Promise<void> {
       return this.connectionRepo.deleteConnectionsByProjectId(id);
    }
+
+   // ==================== MATERIALIZATIONS ====================
+
+   async listMaterializations(
+      projectId: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]> {
+      return this.materializationRepo.list(projectId, packageName, options);
+   }
+
+   async getMaterializationById(id: string): Promise<Materialization | null> {
+      return this.materializationRepo.getById(id);
+   }
+
+   async getActiveMaterialization(
+      projectId: string,
+      packageName: string,
+   ): Promise<Materialization | null> {
+      return this.materializationRepo.getActive(projectId, packageName);
+   }
+
+   async createMaterialization(
+      projectId: string,
+      packageName: string,
+      status: MaterializationStatus = "PENDING",
+      metadata: Record<string, unknown> | null = null,
+   ): Promise<Materialization> {
+      return this.materializationRepo.create(
+         projectId,
+         packageName,
+         status,
+         metadata,
+      );
+   }
+
+   async updateMaterialization(
+      id: string,
+      updates: {
+         status?: MaterializationStatus;
+         startedAt?: Date;
+         completedAt?: Date;
+         error?: string | null;
+         metadata?: Record<string, unknown> | null;
+      },
+   ): Promise<Materialization> {
+      return this.materializationRepo.update(id, updates);
+   }
+
+   async deleteMaterialization(id: string): Promise<void> {
+      return this.materializationRepo.deleteById(id);
+   }
+
+   // ==================== BUILD MANIFESTS ====================
+
+   async listManifestEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      return this.manifestRepo.listEntries(projectId, packageName);
+   }
+
+   async upsertManifestEntry(
+      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
+   ): Promise<ManifestEntry> {
+      return this.manifestRepo.upsertEntry(entry);
+   }
+
+   async deleteManifestEntry(id: string): Promise<void> {
+      return this.manifestRepo.deleteEntry(id);
+   }
 }

package/src/storage/duckdb/ManifestRepository.ts

@@ -0,0 +1,119 @@
+import { ManifestEntry } from "../DatabaseInterface";
+import { DuckDBConnection } from "./DuckDBConnection";
+
+/**
+ * Repository for build manifest entries stored in the `build_manifests` table.
+ *
+ * A build manifest records the materialized table produced by a specific build
+ * of a package, linking it back to the Malloy source and connection that
+ * generated it. Manifests are keyed by (project, package, build) and enable
+ * downstream consumers to discover which physical tables correspond to a given
+ * package version.
+ */
+export class ManifestRepository {
+   constructor(private db: DuckDBConnection) {}
+
+   private generateId(): string {
+      return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
+   }
+
+   /** Returns all manifest entries for a package, most recent first. */
+   async listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      const rows = await this.db.all<Record<string, unknown>>(
+         "SELECT * FROM build_manifests WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC",
+         [projectId, packageName],
+      );
+      return rows.map(this.mapToEntry);
+   }
+
+   /** Looks up the manifest entry for a specific build, or null if none exists. */
+   async getEntryByBuildId(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+   ): Promise<ManifestEntry | null> {
+      const row = await this.db.get<Record<string, unknown>>(
+         "SELECT * FROM build_manifests WHERE project_id = ? AND package_name = ? AND build_id = ?",
+         [projectId, packageName, buildId],
+      );
+      return row ? this.mapToEntry(row) : null;
+   }
+
+   /**
+    * Inserts a new manifest entry, or updates the existing one if a row with
+    * the same (project, package, build) triple already exists. Uses INSERT ON
+    * CONFLICT to avoid the TOCTOU race of SELECT-then-INSERT/UPDATE.
+    */
+   async upsertEntry(
+      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
+   ): Promise<ManifestEntry> {
+      const id = this.generateId();
+      const now = new Date();
+      const iso = now.toISOString();
+
+      const rows = await this.db.all<Record<string, unknown>>(
+         `INSERT INTO build_manifests (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT (project_id, package_name, build_id)
+         DO UPDATE SET table_name = EXCLUDED.table_name,
+                       source_name = EXCLUDED.source_name,
+                       connection_name = EXCLUDED.connection_name,
+                       updated_at = EXCLUDED.updated_at
+         RETURNING *`,
+         [
+            id,
+            entry.projectId,
+            entry.packageName,
+            entry.buildId,
+            entry.tableName,
+            entry.sourceName,
+            entry.connectionName,
+            iso,
+            iso,
+         ],
+      );
+
+      return this.mapToEntry(rows[0]);
+   }
+
+   /** Deletes a single manifest entry by ID. */
+   async deleteEntry(id: string): Promise<void> {
+      await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
+   }
+
+   /** Removes all manifest entries belonging to a project (used on project deletion). */
+   async deleteEntriesByProjectId(projectId: string): Promise<void> {
+      await this.db.run("DELETE FROM build_manifests WHERE project_id = ?", [
+         projectId,
+      ]);
+   }
+
+   /** Removes all manifest entries for a specific package. */
+   async deleteEntriesByPackage(
+      projectId: string,
+      packageName: string,
+   ): Promise<void> {
+      await this.db.run(
+         "DELETE FROM build_manifests WHERE project_id = ? AND package_name = ?",
+         [projectId, packageName],
+      );
+   }
+
+   /** Maps a raw DB row (snake_case columns) to a {@link ManifestEntry}. */
+   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
+      return {
+         id: row.id as string,
+         projectId: row.project_id as string,
+         packageName: row.package_name as string,
+         buildId: row.build_id as string,
+         tableName: row.table_name as string,
+         sourceName: row.source_name as string,
+         connectionName: row.connection_name as string,
+         createdAt: new Date(row.created_at as string),
+         updatedAt: new Date(row.updated_at as string),
+      };
+   }
+}