@malloy-publisher/server 0.0.235 → 0.0.237

package/src/storage/StorageManager.ts

@@ -1,113 +0,0 @@
-import { logger } from "../logger";
-import { DatabaseConnection, ResourceRepository } from "./DatabaseInterface";
-import { DuckDBConnection } from "./duckdb/DuckDBConnection";
-import { DuckDBRepository } from "./duckdb/DuckDBRepository";
-import { initializeSchema } from "./duckdb/schema";
-
-export type StorageType = "duckdb" | "postgres" | "mysql";
-
-export interface StorageConfig {
-   type: StorageType;
-   duckdb?: {
-      path?: string;
-   };
-   postgres?: {
-      host: string;
-      port: number;
-      database: string;
-      user: string;
-      password: string;
-   };
-   mysql?: {
-      host: string;
-      port: number;
-      database: string;
-      user: string;
-      password: string;
-   };
-}
-
-/**
- * Manages the storage backend (DuckDB today) and exposes the resource
- * repository. The publisher persists only its own control records
- * (environments, packages, connections, materializations) here; build
- * manifests are returned inline on the materialization resource and persisted
- * by the control plane.
- */
-export class StorageManager {
-   private connection: DatabaseConnection | null = null;
-   private repository: ResourceRepository | null = null;
-
-   private config: StorageConfig;
-
-   constructor(config: StorageConfig) {
-      this.config = config;
-   }
-
-   async initialize(reinit: boolean = false): Promise<void> {
-      if (reinit) {
-         logger.info(
-            "Reinitialization mode: Database will be dropped and recreated",
-         );
-      } else {
-         logger.info("Normal mode: Loading from existing database");
-      }
-
-      switch (this.config.type) {
-         case "duckdb":
-            await this.initializeDuckDB(reinit);
-            break;
-         case "postgres":
-            throw new Error("PostgreSQL storage not yet implemented");
-         case "mysql":
-            throw new Error("MySQL storage not yet implemented");
-         default:
-            throw new Error(`Unsupported storage type: ${this.config.type}`);
-      }
-   }
-
-   private async initializeDuckDB(reinit: boolean): Promise<void> {
-      const connection = new DuckDBConnection(this.config.duckdb?.path);
-      await connection.initialize();
-      await initializeSchema(connection, reinit);
-
-      this.connection = connection;
-      this.repository = new DuckDBRepository(connection);
-   }
-
-   /**
-    * Access the underlying DuckDB connection for callers that need raw
-    * SQL access (e.g. the ThemeStore singleton) and don't fit the
-    * structured `ResourceRepository` interface. Use sparingly; prefer
-    * `getRepository()` for anything that maps cleanly onto the existing
-    * repository methods.
-    */
-   getDuckDbConnection(): DuckDBConnection {
-      if (!this.connection) {
-         throw new Error("Storage not initialized");
-      }
-      if (!(this.connection instanceof DuckDBConnection)) {
-         throw new Error("Underlying storage connection is not DuckDB");
-      }
-      return this.connection;
-   }
-
-   getRepository(): ResourceRepository {
-      if (!this.repository) {
-         throw new Error("Storage not initialized. Call initialize() first.");
-      }
-      return this.repository;
-   }
-
-   async close(): Promise<void> {
-      if (this.connection) {
-         await this.connection.close();
-         this.connection = null;
-         this.repository = null;
-      }
-   }
-
-   isInitialized(): boolean {
-      return this.connection !== null && this.repository !== null;
-   }
-}

package/src/storage/duckdb/ConnectionRepository.ts

@@ -1,143 +0,0 @@
-import { Connection } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-export class ConnectionRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
-   }
-
-   private now(): Date {
-      return new Date();
-   }
-
-   async listConnections(environmentId: string): Promise<Connection[]> {
-      try {
-         const rows = await this.db.all<Record<string, unknown>>(
-            "SELECT * FROM connections WHERE environment_id = ? ORDER BY name",
-            [environmentId],
-         );
-         return rows.map(this.mapToConnection);
-      } catch (err: unknown) {
-         const error = err as Error;
-         console.error("Failed to get connections:", error.message);
-         throw error;
-      }
-   }
-
-   async getConnectionById(id: string): Promise<Connection | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM connections WHERE id = ?",
-         [id],
-      );
-      return row ? this.mapToConnection(row) : null;
-   }
-
-   async getConnectionByName(
-      environmentId: string,
-      name: string,
-   ): Promise<Connection | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM connections WHERE environment_id = ? AND name = ?",
-         [environmentId, name],
-      );
-      return row ? this.mapToConnection(row) : null;
-   }
-
-   async createConnection(
-      connection: Omit<Connection, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Connection> {
-      const id = this.generateId();
-      const now = this.now();
-
-      try {
-         const configJson = JSON.stringify(connection.config);
-
-         await this.db.run(
-            `INSERT INTO connections (id, environment_id, name, type, config, created_at, updated_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?)`,
-            [
-               id,
-               connection.environmentId,
-               connection.name,
-               connection.type,
-               configJson,
-               now.toISOString(),
-               now.toISOString(),
-            ],
-         );
-
-         return {
-            id,
-            ...connection,
-            createdAt: now,
-            updatedAt: now,
-         };
-      } catch (err: unknown) {
-         const error = err as Error;
-         console.error("Failed to create connection:", error.message);
-         throw error;
-      }
-   }
-
-   async updateConnection(
-      id: string,
-      updates: Partial<Connection>,
-   ): Promise<Connection> {
-      const existing = await this.getConnectionById(id);
-      if (!existing) {
-         throw new Error(`Connection with id ${id} not found`);
-      }
-
-      const now = this.now();
-      const setClauses: string[] = [];
-      const params: unknown[] = [];
-
-      if (updates.name !== undefined) {
-         setClauses.push(`name = ?`);
-         params.push(updates.name);
-      }
-      if (updates.type !== undefined) {
-         setClauses.push(`type = ?`);
-         params.push(updates.type);
-      }
-      if (updates.config !== undefined) {
-         setClauses.push(`config = ?`);
-         params.push(JSON.stringify(updates.config));
-      }
-
-      setClauses.push(`updated_at = ?`);
-      params.push(now.toISOString());
-      params.push(id);
-
-      await this.db.run(
-         `UPDATE connections SET ${setClauses.join(", ")} WHERE id = ?`,
-         params,
-      );
-
-      return this.getConnectionById(id) as Promise<Connection>;
-   }
-
-   async deleteConnection(id: string): Promise<void> {
-      await this.db.run("DELETE FROM connections WHERE id = ?", [id]);
-   }
-
-   async deleteConnectionsByEnvironmentId(id: string): Promise<void> {
-      await this.db.run("DELETE FROM connections WHERE environment_id = ?", [
-         id,
-      ]);
-   }
-
-   private mapToConnection(row: Record<string, unknown>): Connection {
-      return {
-         id: row.id as string,
-         environmentId: row.environment_id as string,
-         name: row.name as string,
-         type: row.type as Connection["type"],
-         config: JSON.parse(row.config as string),
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}

package/src/storage/duckdb/DuckDBConnection.ts

@@ -1,123 +0,0 @@
-import { Mutex } from "async-mutex";
-import {
-   DuckDBConnection as NeoConnection,
-   DuckDBInstance,
-   type DuckDBValue,
-} from "@duckdb/node-api";
-import * as path from "path";
-import { DatabaseConnection } from "../DatabaseInterface";
-
-/**
- * Embedded persistence layer for the publisher's own metadata (environments,
- * packages, connections, materializations, build manifests) in `publisher.db`.
- *
- * This is a plain DAO over a durable, exclusively-owned DuckDB handle -- it is
- * deliberately NOT Malloy's `@malloydata/db-duckdb` connection, which is an
- * analytical query connection (no prepared-statement parameter binding, a
- * `:memory:` primary with ATTACH/DETACH/idle lifecycle, pooled/shared
- * instances, and a poison-pill close). Those semantics are wrong for a
- * source-of-truth store that must hold one handle open for the server's
- * lifetime and run parameterized CRUD.
- *
- * It wraps `@duckdb/node-api` (the same DuckDB engine Malloy pulls in), so the
- * repo carries a single DuckDB engine rather than a second, redundant driver.
- */
-export class DuckDBConnection implements DatabaseConnection {
-   private instance: DuckDBInstance | null = null;
-   private connection: NeoConnection | null = null;
-   private dbPath: string;
-   private mutex: Mutex = new Mutex();
-
-   constructor(dbPath?: string) {
-      // Default to storing in the server root directory
-      this.dbPath = dbPath || path.join(process.cwd(), "publisher.db");
-   }
-
-   async initialize(): Promise<void> {
-      try {
-         this.instance = await DuckDBInstance.create(this.dbPath);
-         this.connection = await this.instance.connect();
-         // Verify the connection works
-         await this.connection.run("SELECT 42 as answer");
-      } catch (err) {
-         const message = err instanceof Error ? err.message : String(err);
-         console.error("Failed to create DuckDB database:", err);
-         throw new Error(`Failed to initialize DuckDB: ${message}`);
-      }
-   }
-
-   async close(): Promise<void> {
-      try {
-         if (this.connection) {
-            this.connection.closeSync();
-            this.connection = null;
-         }
-         if (this.instance) {
-            this.instance.closeSync();
-            this.instance = null;
-         }
-         console.log("DuckDB connection closed");
-      } catch (err) {
-         const message = err instanceof Error ? err.message : String(err);
-         throw new Error(`Failed to close DuckDB connection: ${message}`);
-      }
-   }
-
-   async isInitialized(): Promise<boolean> {
-      if (!this.connection) return false;
-
-      return this.mutex.runExclusive(async () => {
-         try {
-            const reader = await this.connection!.runAndReadAll(
-               "SELECT name FROM sqlite_master WHERE type='table' AND name='environments'",
-            );
-            return reader.getRowObjectsJS().length > 0;
-         } catch {
-            return false;
-         }
-      });
-   }
-
-   async run(query: string, params?: unknown[]): Promise<void> {
-      if (!this.connection) {
-         throw new Error("Database not initialized");
-      }
-
-      return this.mutex.runExclusive(async () => {
-         try {
-            await this.connection!.run(query, params as DuckDBValue[]);
-         } catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            throw new Error(
-               `Query execution failed: ${message}\nQuery: ${query}`,
-            );
-         }
-      });
-   }
-
-   async all<T>(query: string, params?: unknown[]): Promise<T[]> {
-      if (!this.connection) {
-         throw new Error("Database not initialized");
-      }
-
-      return this.mutex.runExclusive(async () => {
-         try {
-            const reader = await this.connection!.runAndReadAll(
-               query,
-               params as DuckDBValue[],
-            );
-            return reader.getRowObjectsJS() as T[];
-         } catch (err) {
-            const message = err instanceof Error ? err.message : String(err);
-            throw new Error(
-               `Query execution failed: ${message}\nQuery: ${query}`,
-            );
-         }
-      });
-   }
-
-   async get<T>(query: string, params?: unknown[]): Promise<T | null> {
-      const rows = await this.all<T>(query, params);
-      return rows.length > 0 ? rows[0] : null;
-   }
-}

package/src/storage/duckdb/DuckDBRepository.ts

@@ -1,208 +0,0 @@
-import {
-   Connection,
-   Environment,
-   Materialization,
-   MaterializationStatus,
-   MaterializationUpdate,
-   Package,
-   ResourceRepository,
-} from "../DatabaseInterface";
-import { ConnectionRepository } from "./ConnectionRepository";
-import { DuckDBConnection } from "./DuckDBConnection";
-import { EnvironmentRepository } from "./EnvironmentRepository";
-import { MaterializationRepository } from "./MaterializationRepository";
-import { PackageRepository } from "./PackageRepository";
-
-export class DuckDBRepository implements ResourceRepository {
-   private environmentRepo: EnvironmentRepository;
-   private packageRepo: PackageRepository;
-   private connectionRepo: ConnectionRepository;
-   private materializationRepo: MaterializationRepository;
-
-   constructor(public db: DuckDBConnection) {
-      this.environmentRepo = new EnvironmentRepository(db);
-      this.packageRepo = new PackageRepository(db);
-      this.connectionRepo = new ConnectionRepository(db);
-      this.materializationRepo = new MaterializationRepository(db);
-   }
-
-   // ==================== ENVIRONMENTS ====================
-
-   async listEnvironments(): Promise<Environment[]> {
-      return this.environmentRepo.listEnvironments();
-   }
-
-   async getEnvironmentById(id: string): Promise<Environment | null> {
-      return this.environmentRepo.getEnvironmentById(id);
-   }
-
-   async getEnvironmentByName(name: string): Promise<Environment | null> {
-      return this.environmentRepo.getEnvironmentByName(name);
-   }
-
-   async createEnvironment(
-      environment: Omit<Environment, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Environment> {
-      return this.environmentRepo.createEnvironment(environment);
-   }
-
-   async updateEnvironment(
-      id: string,
-      updates: Partial<Environment>,
-   ): Promise<Environment> {
-      return this.environmentRepo.updateEnvironment(id, updates);
-   }
-
-   async deleteEnvironment(id: string): Promise<void> {
-      await this.materializationRepo.deleteByEnvironmentId(id);
-      await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
-      await this.packageRepo.deletePackagesByEnvironmentId(id);
-      await this.environmentRepo.deleteEnvironment(id);
-   }
-
-   // ==================== PACKAGES ====================
-
-   async listPackages(environmentId: string): Promise<Package[]> {
-      return this.packageRepo.listPackages(environmentId);
-   }
-
-   async getPackageById(id: string): Promise<Package | null> {
-      return this.packageRepo.getPackageById(id);
-   }
-
-   async getPackageByName(
-      environmentId: string,
-      name: string,
-   ): Promise<Package | null> {
-      return this.packageRepo.getPackageByName(environmentId, name);
-   }
-
-   async createPackage(
-      pkg: Omit<Package, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Package> {
-      return this.packageRepo.createPackage(pkg);
-   }
-
-   async updatePackage(
-      id: string,
-      updates: Partial<Package>,
-   ): Promise<Package> {
-      return this.packageRepo.updatePackage(id, updates);
-   }
-
-   async deletePackage(id: string): Promise<void> {
-      const pkg = await this.packageRepo.getPackageById(id);
-      if (pkg) {
-         await this.materializationRepo.deleteByPackage(
-            pkg.environmentId,
-            pkg.name,
-         );
-      }
-      await this.packageRepo.deletePackage(id);
-   }
-
-   async deletePackagesByEnvironmentId(id: string): Promise<void> {
-      return this.packageRepo.deletePackagesByEnvironmentId(id);
-   }
-
-   // ==================== CONNECTIONS ====================
-
-   async listConnections(environmentId: string): Promise<Connection[]> {
-      return this.connectionRepo.listConnections(environmentId);
-   }
-
-   async getConnectionById(id: string): Promise<Connection | null> {
-      return this.connectionRepo.getConnectionById(id);
-   }
-
-   async getConnectionByName(
-      environmentId: string,
-      name: string,
-   ): Promise<Connection | null> {
-      return this.connectionRepo.getConnectionByName(environmentId, name);
-   }
-
-   async createConnection(
-      connection: Omit<Connection, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Connection> {
-      return this.connectionRepo.createConnection(connection);
-   }
-
-   async updateConnection(
-      id: string,
-      updates: Partial<Connection>,
-   ): Promise<Connection> {
-      return this.connectionRepo.updateConnection(id, updates);
-   }
-
-   async deleteConnection(id: string): Promise<void> {
-      return this.connectionRepo.deleteConnection(id);
-   }
-
-   async deleteConnectionsByEnvironmentId(id: string): Promise<void> {
-      return this.connectionRepo.deleteConnectionsByEnvironmentId(id);
-   }
-
-   // ==================== MATERIALIZATIONS ====================
-
-   async listMaterializations(
-      environmentId: string,
-      packageName: string,
-      options?: { limit?: number; offset?: number },
-   ): Promise<Materialization[]> {
-      return this.materializationRepo.list(environmentId, packageName, options);
-   }
-
-   async listMaterializationsByEnvironment(
-      environmentId: string,
-      options?: { limit?: number; offset?: number },
-   ): Promise<Materialization[]> {
-      return this.materializationRepo.listByEnvironment(environmentId, options);
-   }
-
-   async getLatestScheduledFireAt(
-      environmentId: string,
-      packageName: string,
-   ): Promise<Date | null> {
-      return this.materializationRepo.getLatestScheduledFireAt(
-         environmentId,
-         packageName,
-      );
-   }
-
-   async getMaterializationById(id: string): Promise<Materialization | null> {
-      return this.materializationRepo.getById(id);
-   }
-
-   async getActiveMaterialization(
-      environmentId: string,
-      packageName: string,
-   ): Promise<Materialization | null> {
-      return this.materializationRepo.getActive(environmentId, packageName);
-   }
-
-   async createMaterialization(
-      environmentId: string,
-      packageName: string,
-      status: MaterializationStatus = "PENDING",
-      metadata: Record<string, unknown> | null = null,
-   ): Promise<Materialization> {
-      return this.materializationRepo.create(
-         environmentId,
-         packageName,
-         status,
-         metadata,
-      );
-   }
-
-   async updateMaterialization(
-      id: string,
-      updates: MaterializationUpdate,
-   ): Promise<Materialization> {
-      return this.materializationRepo.update(id, updates);
-   }
-
-   async deleteMaterialization(id: string): Promise<void> {
-      return this.materializationRepo.deleteById(id);
-   }
-}