@malloy-publisher/server 0.0.382-dev → 0.2.0

package/src/storage/duckdb/DuckDBConnection.ts

@@ -1,177 +0,0 @@
-import { Mutex } from "async-mutex";
-import * as duckdb from "duckdb";
-import * as path from "path";
-import { DatabaseConnection } from "../DatabaseInterface";
-
-export class DuckDBConnection implements DatabaseConnection {
-   private db: duckdb.Database | null = null;
-   private connection: duckdb.Connection | null = null;
-   private dbPath: string;
-   private mutex: Mutex = new Mutex();
-
-   constructor(dbPath?: string) {
-      // Default to storing in the server root directory
-      this.dbPath = dbPath || path.join(process.cwd(), "publisher.db");
-   }
-
-   async initialize(): Promise<void> {
-      return new Promise((resolve, reject) => {
-         this.db = new duckdb.Database(this.dbPath, {}, (err) => {
-            if (err) {
-               console.error("Failed to create DuckDB database:", err);
-               reject(new Error(`Failed to initialize DuckDB: ${err.message}`));
-               return;
-            }
-
-            // Connect synchronously
-            this.connection = (
-               this.db as duckdb.Database & { connect(): duckdb.Connection }
-            ).connect();
-
-            if (!this.connection) {
-               reject(new Error("Failed to create connection object"));
-               return;
-            }
-
-            // Verify connection works
-            this.connection.all("SELECT 42 as answer", (testErr, _rows) => {
-               if (testErr) {
-                  console.error("Connection test failed:", testErr);
-                  reject(
-                     new Error(
-                        `Failed to verify DuckDB connection: ${testErr.message}`,
-                     ),
-                  );
-                  return;
-               }
-
-               resolve();
-            });
-         });
-      });
-   }
-
-   async close(): Promise<void> {
-      return new Promise((resolve, reject) => {
-         if (this.connection) {
-            this.connection.close((err) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Failed to close DuckDB connection: ${err.message}`,
-                     ),
-                  );
-                  return;
-               }
-
-               if (this.db) {
-                  this.db.close((dbErr) => {
-                     if (dbErr) {
-                        reject(
-                           new Error(
-                              `Failed to close DuckDB: ${dbErr.message}`,
-                           ),
-                        );
-                        return;
-                     }
-                     console.log("DuckDB connection closed");
-                     resolve();
-                  });
-               } else {
-                  resolve();
-               }
-            });
-         } else {
-            resolve();
-         }
-      });
-   }
-
-   async isInitialized(): Promise<boolean> {
-      if (!this.connection) return false;
-
-      return this.mutex.runExclusive(async () => {
-         return new Promise<boolean>((resolve) => {
-            this.connection!.all(
-               "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
-               (err, rows) => {
-                  if (err) {
-                     resolve(false);
-                     return;
-                  }
-                  resolve(rows && rows.length > 0);
-               },
-            );
-         });
-      });
-   }
-
-   async run(query: string, params?: unknown[]): Promise<void> {
-      if (!this.connection) {
-         throw new Error("Database not initialized");
-      }
-
-      return this.mutex.runExclusive(async () => {
-         return new Promise<void>((resolve, reject) => {
-            const callback = (err: Error | null) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Query execution failed: ${err.message}\nQuery: ${query}`,
-                     ),
-                  );
-                  return;
-               }
-               resolve();
-            };
-
-            // Pass params directly without the params argument if empty
-            if (params && params.length > 0) {
-               this.connection!.run(query, ...params, callback);
-            } else {
-               this.connection!.run(query, callback);
-            }
-         });
-      });
-   }
-
-   async all<T>(query: string, params?: unknown[]): Promise<T[]> {
-      if (!this.connection) {
-         throw new Error("Database not initialized");
-      }
-
-      return this.mutex.runExclusive(async () => {
-         return new Promise<T[]>((resolve, reject) => {
-            const callback = (err: Error | null, rows: unknown[]) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Query execution failed: ${err.message}\nQuery: ${query}`,
-                     ),
-                  );
-                  return;
-               }
-               resolve((rows || []) as T[]);
-            };
-
-            if (params && params.length > 0) {
-               this.connection!.all(query, ...params, callback);
-            } else {
-               this.connection!.all(query, callback);
-            }
-         });
-      });
-   }
-
-   async get<T>(query: string, params?: unknown[]): Promise<T | null> {
-      const rows = await this.all<T>(query, params);
-      return rows.length > 0 ? rows[0] : null;
-   }
-
-   getConnection(): duckdb.Connection {
-      if (!this.connection) {
-         throw new Error("Database not initialized");
-      }
-      return this.connection;
-   }
-}

package/src/storage/duckdb/DuckDBManifestStore.ts

@@ -1,70 +0,0 @@
-import {
-   BuildManifest,
-   ManifestEntry,
-   ManifestStore,
-   ResourceRepository,
-} from "../DatabaseInterface";
-
-/**
- * DuckDB-backed ManifestStore that delegates to the local ResourceRepository.
- *
- * In standalone mode this is the active store. Orchestrated deployments swap
- * in a DuckLake-backed implementation that reads/writes manifests through the
- * shared catalog instead of the local DuckDB build_manifests table.
- */
-export class DuckDBManifestStore implements ManifestStore {
-   constructor(private repository: ResourceRepository) {}
-
-   /**
-    * Assembles a {@link BuildManifest} by folding all manifest rows for the
-    * given package into a build-ID-keyed map. `strict: false` lets the Malloy
-    * runtime fall back to executing the underlying query when a persist
-    * reference has no manifest entry (e.g. before the first materialization).
-    */
-   async getManifest(
-      projectId: string,
-      packageName: string,
-   ): Promise<BuildManifest> {
-      const entries = await this.repository.listManifestEntries(
-         projectId,
-         packageName,
-      );
-      const manifest: BuildManifest = {
-         entries: {},
-         strict: false,
-      };
-      for (const entry of entries) {
-         manifest.entries[entry.buildId] = { tableName: entry.tableName };
-      }
-      return manifest;
-   }
-
-   async writeEntry(
-      projectId: string,
-      packageName: string,
-      buildId: string,
-      tableName: string,
-      sourceName: string,
-      connectionName: string,
-   ): Promise<void> {
-      await this.repository.upsertManifestEntry({
-         projectId,
-         packageName,
-         buildId,
-         tableName,
-         sourceName,
-         connectionName,
-      });
-   }
-
-   async deleteEntry(id: string): Promise<void> {
-      await this.repository.deleteManifestEntry(id);
-   }
-
-   async listEntries(
-      projectId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      return this.repository.listManifestEntries(projectId, packageName);
-   }
-}

package/src/storage/duckdb/DuckDBRepository.ts

@@ -1,224 +0,0 @@
-import {
-   Connection,
-   ManifestEntry,
-   Materialization,
-   MaterializationStatus,
-   Package,
-   Project,
-   ResourceRepository,
-} from "../DatabaseInterface";
-import { ConnectionRepository } from "./ConnectionRepository";
-import { DuckDBConnection } from "./DuckDBConnection";
-import { ManifestRepository } from "./ManifestRepository";
-import { MaterializationRepository } from "./MaterializationRepository";
-import { PackageRepository } from "./PackageRepository";
-import { ProjectRepository } from "./ProjectRepository";
-
-export class DuckDBRepository implements ResourceRepository {
-   private projectRepo: ProjectRepository;
-   private packageRepo: PackageRepository;
-   private connectionRepo: ConnectionRepository;
-   private materializationRepo: MaterializationRepository;
-   private manifestRepo: ManifestRepository;
-
-   constructor(public db: DuckDBConnection) {
-      this.projectRepo = new ProjectRepository(db);
-      this.packageRepo = new PackageRepository(db);
-      this.connectionRepo = new ConnectionRepository(db);
-      this.materializationRepo = new MaterializationRepository(db);
-      this.manifestRepo = new ManifestRepository(db);
-   }
-
-   // ==================== PROJECTS ====================
-
-   async listProjects(): Promise<Project[]> {
-      return this.projectRepo.listProjects();
-   }
-
-   async getProjectById(id: string): Promise<Project | null> {
-      return this.projectRepo.getProjectById(id);
-   }
-
-   async getProjectByName(name: string): Promise<Project | null> {
-      return this.projectRepo.getProjectByName(name);
-   }
-
-   async createProject(
-      project: Omit<Project, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Project> {
-      return this.projectRepo.createProject(project);
-   }
-
-   async updateProject(
-      id: string,
-      updates: Partial<Project>,
-   ): Promise<Project> {
-      return this.projectRepo.updateProject(id, updates);
-   }
-
-   async deleteProject(id: string): Promise<void> {
-      await this.manifestRepo.deleteEntriesByProjectId(id);
-      await this.materializationRepo.deleteByProjectId(id);
-      await this.connectionRepo.deleteConnectionsByProjectId(id);
-      await this.packageRepo.deletePackagesByProjectId(id);
-      await this.projectRepo.deleteProject(id);
-   }
-
-   // ==================== PACKAGES ====================
-
-   async listPackages(projectId: string): Promise<Package[]> {
-      return this.packageRepo.listPackages(projectId);
-   }
-
-   async getPackageById(id: string): Promise<Package | null> {
-      return this.packageRepo.getPackageById(id);
-   }
-
-   async getPackageByName(
-      projectId: string,
-      name: string,
-   ): Promise<Package | null> {
-      return this.packageRepo.getPackageByName(projectId, name);
-   }
-
-   async createPackage(
-      pkg: Omit<Package, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Package> {
-      return this.packageRepo.createPackage(pkg);
-   }
-
-   async updatePackage(
-      id: string,
-      updates: Partial<Package>,
-   ): Promise<Package> {
-      return this.packageRepo.updatePackage(id, updates);
-   }
-
-   async deletePackage(id: string): Promise<void> {
-      const pkg = await this.packageRepo.getPackageById(id);
-      if (pkg) {
-         await this.manifestRepo.deleteEntriesByPackage(
-            pkg.projectId,
-            pkg.name,
-         );
-         await this.materializationRepo.deleteByPackage(
-            pkg.projectId,
-            pkg.name,
-         );
-      }
-      await this.packageRepo.deletePackage(id);
-   }
-
-   async deletePackagesByProjectId(id: string): Promise<void> {
-      return this.packageRepo.deletePackagesByProjectId(id);
-   }
-
-   // ==================== CONNECTIONS ====================
-
-   async listConnections(projectId: string): Promise<Connection[]> {
-      return this.connectionRepo.listConnections(projectId);
-   }
-
-   async getConnectionById(id: string): Promise<Connection | null> {
-      return this.connectionRepo.getConnectionById(id);
-   }
-
-   async getConnectionByName(
-      projectId: string,
-      name: string,
-   ): Promise<Connection | null> {
-      return this.connectionRepo.getConnectionByName(projectId, name);
-   }
-
-   async createConnection(
-      connection: Omit<Connection, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Connection> {
-      return this.connectionRepo.createConnection(connection);
-   }
-
-   async updateConnection(
-      id: string,
-      updates: Partial<Connection>,
-   ): Promise<Connection> {
-      return this.connectionRepo.updateConnection(id, updates);
-   }
-
-   async deleteConnection(id: string): Promise<void> {
-      return this.connectionRepo.deleteConnection(id);
-   }
-
-   async deleteConnectionsByProjectId(id: string): Promise<void> {
-      return this.connectionRepo.deleteConnectionsByProjectId(id);
-   }
-
-   // ==================== MATERIALIZATIONS ====================
-
-   async listMaterializations(
-      projectId: string,
-      packageName: string,
-      options?: { limit?: number; offset?: number },
-   ): Promise<Materialization[]> {
-      return this.materializationRepo.list(projectId, packageName, options);
-   }
-
-   async getMaterializationById(id: string): Promise<Materialization | null> {
-      return this.materializationRepo.getById(id);
-   }
-
-   async getActiveMaterialization(
-      projectId: string,
-      packageName: string,
-   ): Promise<Materialization | null> {
-      return this.materializationRepo.getActive(projectId, packageName);
-   }
-
-   async createMaterialization(
-      projectId: string,
-      packageName: string,
-      status: MaterializationStatus = "PENDING",
-      metadata: Record<string, unknown> | null = null,
-   ): Promise<Materialization> {
-      return this.materializationRepo.create(
-         projectId,
-         packageName,
-         status,
-         metadata,
-      );
-   }
-
-   async updateMaterialization(
-      id: string,
-      updates: {
-         status?: MaterializationStatus;
-         startedAt?: Date;
-         completedAt?: Date;
-         error?: string | null;
-         metadata?: Record<string, unknown> | null;
-      },
-   ): Promise<Materialization> {
-      return this.materializationRepo.update(id, updates);
-   }
-
-   async deleteMaterialization(id: string): Promise<void> {
-      return this.materializationRepo.deleteById(id);
-   }
-
-   // ==================== BUILD MANIFESTS ====================
-
-   async listManifestEntries(
-      projectId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      return this.manifestRepo.listEntries(projectId, packageName);
-   }
-
-   async upsertManifestEntry(
-      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
-   ): Promise<ManifestEntry> {
-      return this.manifestRepo.upsertEntry(entry);
-   }
-
-   async deleteManifestEntry(id: string): Promise<void> {
-      return this.manifestRepo.deleteEntry(id);
-   }
-}

package/src/storage/duckdb/ManifestRepository.ts

@@ -1,119 +0,0 @@
-import { ManifestEntry } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-/**
- * Repository for build manifest entries stored in the `build_manifests` table.
- *
- * A build manifest records the materialized table produced by a specific build
- * of a package, linking it back to the Malloy source and connection that
- * generated it. Manifests are keyed by (project, package, build) and enable
- * downstream consumers to discover which physical tables correspond to a given
- * package version.
- */
-export class ManifestRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
-   }
-
-   /** Returns all manifest entries for a package, most recent first. */
-   async listEntries(
-      projectId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         "SELECT * FROM build_manifests WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC",
-         [projectId, packageName],
-      );
-      return rows.map(this.mapToEntry);
-   }
-
-   /** Looks up the manifest entry for a specific build, or null if none exists. */
-   async getEntryByBuildId(
-      projectId: string,
-      packageName: string,
-      buildId: string,
-   ): Promise<ManifestEntry | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM build_manifests WHERE project_id = ? AND package_name = ? AND build_id = ?",
-         [projectId, packageName, buildId],
-      );
-      return row ? this.mapToEntry(row) : null;
-   }
-
-   /**
-    * Inserts a new manifest entry, or updates the existing one if a row with
-    * the same (project, package, build) triple already exists. Uses INSERT ON
-    * CONFLICT to avoid the TOCTOU race of SELECT-then-INSERT/UPDATE.
-    */
-   async upsertEntry(
-      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
-   ): Promise<ManifestEntry> {
-      const id = this.generateId();
-      const now = new Date();
-      const iso = now.toISOString();
-
-      const rows = await this.db.all<Record<string, unknown>>(
-         `INSERT INTO build_manifests (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-         ON CONFLICT (project_id, package_name, build_id)
-         DO UPDATE SET table_name = EXCLUDED.table_name,
-                       source_name = EXCLUDED.source_name,
-                       connection_name = EXCLUDED.connection_name,
-                       updated_at = EXCLUDED.updated_at
-         RETURNING *`,
-         [
-            id,
-            entry.projectId,
-            entry.packageName,
-            entry.buildId,
-            entry.tableName,
-            entry.sourceName,
-            entry.connectionName,
-            iso,
-            iso,
-         ],
-      );
-
-      return this.mapToEntry(rows[0]);
-   }
-
-   /** Deletes a single manifest entry by ID. */
-   async deleteEntry(id: string): Promise<void> {
-      await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
-   }
-
-   /** Removes all manifest entries belonging to a project (used on project deletion). */
-   async deleteEntriesByProjectId(projectId: string): Promise<void> {
-      await this.db.run("DELETE FROM build_manifests WHERE project_id = ?", [
-         projectId,
-      ]);
-   }
-
-   /** Removes all manifest entries for a specific package. */
-   async deleteEntriesByPackage(
-      projectId: string,
-      packageName: string,
-   ): Promise<void> {
-      await this.db.run(
-         "DELETE FROM build_manifests WHERE project_id = ? AND package_name = ?",
-         [projectId, packageName],
-      );
-   }
-
-   /** Maps a raw DB row (snake_case columns) to a {@link ManifestEntry}. */
-   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
-      return {
-         id: row.id as string,
-         projectId: row.project_id as string,
-         packageName: row.package_name as string,
-         buildId: row.build_id as string,
-         tableName: row.table_name as string,
-         sourceName: row.source_name as string,
-         connectionName: row.connection_name as string,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}