@malloy-publisher/server 0.0.382-dev → 0.2.1

package/src/service/quoting.ts

@@ -1,41 +0,0 @@
-/**
- * Minimal identifier-quoting surface. Every `Dialect` in `@malloydata/malloy`
- * implements this; we accept the duck type so tests can inject a fake without
- * instantiating a full dialect.
- */
-export interface Quoter {
-   quoteTablePath(seg: string): string;
-}
-
-/**
- * Quote a potentially schema-qualified table path (e.g. "schema.table")
- * by quoting each segment individually with the dialect's quoteTablePath.
- */
-export function quoteTablePath(path: string, dialect: Quoter): string {
-   return path
-      .split(".")
-      .map((seg) => dialect.quoteTablePath(seg))
-      .join(".");
-}
-
-/**
- * Split a possibly schema-qualified table name into its schema prefix
- * (including the trailing dot) and the bare table name.
- *
- * Examples:
- *   "my_schema.my_table" -> { schemaPrefix: "my_schema.", bareName: "my_table" }
- *   "my_table"           -> { schemaPrefix: "", bareName: "my_table" }
- */
-export function splitTablePath(tableName: string): {
-   schemaPrefix: string;
-   bareName: string;
-} {
-   const lastDot = tableName.lastIndexOf(".");
-   if (lastDot >= 0) {
-      return {
-         schemaPrefix: tableName.substring(0, lastDot + 1),
-         bareName: tableName.substring(lastDot + 1),
-      };
-   }
-   return { schemaPrefix: "", bareName: tableName };
-}

package/src/service/resolve_project.ts

@@ -1,13 +0,0 @@
-import { ProjectNotFoundError } from "../errors";
-import { ResourceRepository } from "../storage/DatabaseInterface";
-
-export async function resolveProjectId(
-   repository: ResourceRepository,
-   projectName: string,
-): Promise<string> {
-   const dbProject = await repository.getProjectByName(projectName);
-   if (!dbProject) {
-      throw new ProjectNotFoundError(`Project '${projectName}' not found`);
-   }
-   return dbProject.id;
-}

package/src/storage/BaseRepository.ts

@@ -1,31 +0,0 @@
-import { DatabaseConnection } from "./DatabaseInterface";
-
-export abstract class BaseRepository {
-   constructor(protected db: DatabaseConnection) {}
-
-   protected generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
-   }
-
-   protected now(): Date {
-      return new Date();
-   }
-
-   protected async executeQuery<T>(
-      _query: string,
-      _params?: unknown[],
-   ): Promise<T[]> {
-      // This will be implemented by specific database implementations
-      throw new Error(
-         "executeQuery must be implemented by database-specific class",
-      );
-   }
-
-   protected async executeOne<T>(
-      query: string,
-      params?: unknown[],
-   ): Promise<T | null> {
-      const results = await this.executeQuery<T>(query, params);
-      return results.length > 0 ? results[0] : null;
-   }
-}

package/src/storage/DatabaseInterface.ts

@@ -1,180 +0,0 @@
-/**
- * Generic database interface for storage abstraction
- * This allows switching between DuckDB, PostgreSQL, MySQL, etc.
- */
-export interface DatabaseConnection {
-   initialize(): Promise<void>;
-   close(): Promise<void>;
-   isInitialized(): Promise<boolean>;
-}
-
-export interface ResourceRepository {
-   // Projects
-   listProjects(): Promise<Project[]>;
-   getProjectById(id: string): Promise<Project | null>;
-   getProjectByName(name: string): Promise<Project | null>;
-   createProject(
-      project: Omit<Project, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Project>;
-   updateProject(id: string, updates: Partial<Project>): Promise<Project>;
-   deleteProject(id: string): Promise<void>;
-
-   // Packages
-   listPackages(projectId: string): Promise<Package[]>;
-   getPackageById(id: string): Promise<Package | null>;
-   getPackageByName(projectId: string, name: string): Promise<Package | null>;
-   createPackage(
-      pkg: Omit<Package, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Package>;
-   updatePackage(id: string, updates: Partial<Package>): Promise<Package>;
-   deletePackage(id: string): Promise<void>;
-
-   // Connections
-   listConnections(projectId: string): Promise<Connection[]>;
-   getConnectionById(id: string): Promise<Connection | null>;
-   getConnectionByName(
-      projectId: string,
-      name: string,
-   ): Promise<Connection | null>;
-   createConnection(
-      connection: Omit<Connection, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Connection>;
-   updateConnection(
-      id: string,
-      updates: Partial<Connection>,
-   ): Promise<Connection>;
-   deleteConnection(id: string): Promise<void>;
-
-   // Materializations
-   listMaterializations(
-      projectId: string,
-      packageName: string,
-      options?: { limit?: number; offset?: number },
-   ): Promise<Materialization[]>;
-   getMaterializationById(id: string): Promise<Materialization | null>;
-   getActiveMaterialization(
-      projectId: string,
-      packageName: string,
-   ): Promise<Materialization | null>;
-   createMaterialization(
-      projectId: string,
-      packageName: string,
-      status?: MaterializationStatus,
-      metadata?: Record<string, unknown> | null,
-   ): Promise<Materialization>;
-   updateMaterialization(
-      id: string,
-      updates: {
-         status?: MaterializationStatus;
-         startedAt?: Date;
-         completedAt?: Date;
-         error?: string | null;
-         metadata?: Record<string, unknown> | null;
-      },
-   ): Promise<Materialization>;
-   deleteMaterialization(id: string): Promise<void>;
-   // Build Manifests
-   listManifestEntries(
-      projectId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]>;
-   upsertManifestEntry(
-      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
-   ): Promise<ManifestEntry>;
-   deleteManifestEntry(id: string): Promise<void>;
-}
-
-export interface Project {
-   id: string;
-   name: string;
-   path: string;
-   description?: string;
-   createdAt: Date;
-   updatedAt: Date;
-   metadata?: Record<string, unknown>;
-}
-
-export interface Package {
-   id: string;
-   projectId: string;
-   name: string;
-   description?: string;
-   manifestPath: string;
-   createdAt: Date;
-   updatedAt: Date;
-   metadata?: Record<string, unknown>;
-}
-
-export interface Connection {
-   id: string;
-   projectId: string;
-   name: string;
-   type: "bigquery" | "postgres" | "duckdb" | "mysql" | "snowflake" | "trino";
-   config: Record<string, unknown>;
-   createdAt: Date;
-   updatedAt: Date;
-}
-
-export type MaterializationStatus =
-   | "PENDING"
-   | "RUNNING"
-   | "SUCCESS"
-   | "FAILED"
-   | "CANCELLED";
-
-export interface Materialization {
-   id: string;
-   projectId: string;
-   packageName: string;
-   status: MaterializationStatus;
-   startedAt: Date | null;
-   completedAt: Date | null;
-   error: string | null;
-   metadata: Record<string, unknown> | null;
-   createdAt: Date;
-   updatedAt: Date;
-}
-
-export interface ManifestEntry {
-   id: string;
-   projectId: string;
-   packageName: string;
-   buildId: string;
-   tableName: string;
-   sourceName: string;
-   connectionName: string;
-   createdAt: Date;
-   updatedAt: Date;
-}
-
-// ==================== MANIFEST STORE ====================
-
-export interface BuildManifestEntry {
-   tableName: string;
-}
-
-export interface BuildManifest {
-   entries: Record<string, BuildManifestEntry>;
-   strict?: boolean;
-}
-
-/**
- * Abstraction for manifest storage. Standalone mode uses DuckDB;
- * orchestrated mode swaps in a DuckLakeManifestStore.
- */
-export interface ManifestStore {
-   getManifest(projectId: string, packageName: string): Promise<BuildManifest>;
-   writeEntry(
-      projectId: string,
-      packageName: string,
-      buildId: string,
-      tableName: string,
-      sourceName: string,
-      connectionName: string,
-   ): Promise<void>;
-   deleteEntry(id: string): Promise<void>;
-   listEntries(
-      projectId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]>;
-}

package/src/storage/StorageManager.mock.ts

@@ -1,59 +0,0 @@
-type MockData = Record<string, unknown>;
-
-export class StorageManager {
-   async initialize(): Promise<void> {
-      return;
-   }
-
-   getRepository() {
-      return {
-         listProjects: async (): Promise<unknown[]> => [],
-         createProject: async (data: MockData): Promise<MockData> => ({
-            id: "test-id",
-            ...data,
-         }),
-         updateProject: async (
-            id: string,
-            data: MockData,
-         ): Promise<MockData> => ({
-            id,
-            ...data,
-         }),
-         getPackages: async (): Promise<unknown[]> => [],
-         createPackage: async (data: MockData): Promise<MockData> => ({
-            id: "test-id",
-            ...data,
-         }),
-         updatePackage: async (
-            id: string,
-            data: MockData,
-         ): Promise<MockData> => ({
-            id,
-            ...data,
-         }),
-         deletePackage: async (): Promise<void> => {},
-         getConnections: async (): Promise<unknown[]> => [],
-         createConnection: async (data: MockData): Promise<MockData> => ({
-            id: "test-id",
-            ...data,
-         }),
-         updateConnection: async (
-            id: string,
-            data: MockData,
-         ): Promise<MockData> => ({
-            id,
-            ...data,
-         }),
-         deleteConnection: async (): Promise<void> => {},
-      };
-   }
-
-   getManifestStore() {
-      return {
-         getManifest: async () => ({ entries: {}, strict: false }),
-         writeEntry: async () => {},
-         deleteEntry: async () => {},
-         listEntries: async () => [],
-      };
-   }
-}

package/src/storage/StorageManager.ts

@@ -1,210 +0,0 @@
-import { logger } from "../logger";
-import {
-   DatabaseConnection,
-   ManifestStore,
-   ResourceRepository,
-} from "./DatabaseInterface";
-import { DuckDBConnection } from "./duckdb/DuckDBConnection";
-import { DuckDBManifestStore } from "./duckdb/DuckDBManifestStore";
-import { DuckDBRepository } from "./duckdb/DuckDBRepository";
-import { initializeSchema } from "./duckdb/schema";
-import { DuckLakeManifestStore } from "./ducklake/DuckLakeManifestStore";
-
-export type StorageType = "duckdb" | "postgres" | "mysql";
-
-export interface DuckLakeManifestConfig {
-   catalogUrl: string;
-   dataPath: string;
-}
-
-export interface StorageConfig {
-   type: StorageType;
-   duckdb?: {
-      path?: string;
-   };
-   postgres?: {
-      host: string;
-      port: number;
-      database: string;
-      user: string;
-      password: string;
-   };
-   mysql?: {
-      host: string;
-      port: number;
-      database: string;
-      user: string;
-      password: string;
-   };
-}
-
-function escapeSQL(value: string): string {
-   return value.replace(/'/g, "''");
-}
-
-/**
- * Manages the storage backend (DuckDB, Postgres, etc.) and per-project
- * manifest stores. Projects without `materializationStorage` config use
- * the default DuckDB manifest store. Projects with the config get a
- * DuckLake-backed store attached lazily on first access.
- */
-export class StorageManager {
-   private connection: DatabaseConnection | null = null;
-   private duckDbConnection: DuckDBConnection | null = null;
-   private repository: ResourceRepository | null = null;
-   private defaultManifestStore: ManifestStore | null = null;
-
-   /** Per-project DuckLake manifest stores, keyed by projectId. */
-   private projectManifestStores = new Map<string, ManifestStore>();
-
-   /** Tracks which catalogs have been attached to avoid duplicate ATTACHes. */
-   private attachedCatalogs = new Set<string>();
-
-   private config: StorageConfig;
-
-   constructor(config: StorageConfig) {
-      this.config = config;
-   }
-
-   async initialize(reinit: boolean = false): Promise<void> {
-      if (reinit) {
-         logger.info(
-            "Reinitialization mode: Database will be dropped and recreated",
-         );
-      } else {
-         logger.info("Normal mode: Loading from existing database");
-      }
-
-      switch (this.config.type) {
-         case "duckdb":
-            await this.initializeDuckDB(reinit);
-            break;
-         case "postgres":
-            throw new Error("PostgreSQL storage not yet implemented");
-         case "mysql":
-            throw new Error("MySQL storage not yet implemented");
-         default:
-            throw new Error(`Unsupported storage type: ${this.config.type}`);
-      }
-   }
-
-   private async initializeDuckDB(reinit: boolean): Promise<void> {
-      const dbPath = this.config.duckdb?.path;
-
-      const connection = new DuckDBConnection(dbPath);
-
-      await connection.initialize();
-
-      await initializeSchema(connection, reinit);
-
-      this.connection = connection;
-      this.duckDbConnection = connection;
-      this.repository = new DuckDBRepository(connection);
-      this.defaultManifestStore = new DuckDBManifestStore(this.repository);
-   }
-
-   /**
-    * Lazily initializes a DuckLake manifest store for a project.
-    * The DuckLake catalog is attached to the shared DuckDB connection
-    * and persists for the lifetime of the process.
-    */
-   async initializeDuckLakeForProject(
-      projectId: string,
-      config: DuckLakeManifestConfig,
-   ): Promise<void> {
-      if (!this.duckDbConnection) {
-         throw new Error("Storage not initialized. Call initialize() first.");
-      }
-
-      const catalogName = `manifest_lake_${projectId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
-
-      if (!this.attachedCatalogs.has(catalogName)) {
-         await this.attachDuckLakeCatalog(config, catalogName);
-      }
-
-      const store = new DuckLakeManifestStore(
-         this.duckDbConnection,
-         catalogName,
-      );
-      await store.bootstrapSchema();
-
-      this.projectManifestStores.set(projectId, store);
-      logger.info("DuckLake manifest store initialized for project", {
-         projectId,
-         catalogName,
-      });
-   }
-
-   private async attachDuckLakeCatalog(
-      config: DuckLakeManifestConfig,
-      catalogName: string,
-   ): Promise<void> {
-      const connection = this.duckDbConnection!;
-
-      await connection.run("INSTALL ducklake; LOAD ducklake;");
-
-      const isPostgres = config.catalogUrl.startsWith("postgres:");
-      if (isPostgres) {
-         await connection.run("INSTALL postgres; LOAD postgres;");
-      }
-
-      const escapedCatalogUrl = escapeSQL(config.catalogUrl);
-      const escapedDataPath = escapeSQL(config.dataPath);
-      const isCloudStorage =
-         config.dataPath.startsWith("gs://") ||
-         config.dataPath.startsWith("s3://");
-
-      let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
-      const attachOpts: string[] = [`DATA_PATH '${escapedDataPath}'`];
-      if (isCloudStorage) {
-         attachOpts.push("OVERRIDE_DATA_PATH true");
-      }
-      attachCmd += ` (${attachOpts.join(", ")});`;
-
-      logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
-      await connection.run(attachCmd);
-
-      this.attachedCatalogs.add(catalogName);
-   }
-
-   getRepository(): ResourceRepository {
-      if (!this.repository) {
-         throw new Error("Storage not initialized. Call initialize() first.");
-      }
-      return this.repository;
-   }
-
-   /**
-    * Returns the manifest store for a project. If the project has a
-    * DuckLake store configured, returns that; otherwise returns the
-    * default DuckDB-backed store.
-    */
-   getManifestStore(projectId?: string): ManifestStore {
-      if (projectId) {
-         const projectStore = this.projectManifestStores.get(projectId);
-         if (projectStore) {
-            return projectStore;
-         }
-      }
-      if (!this.defaultManifestStore) {
-         throw new Error("Storage not initialized. Call initialize() first.");
-      }
-      return this.defaultManifestStore;
-   }
-
-   async close(): Promise<void> {
-      if (this.connection) {
-         await this.connection.close();
-         this.connection = null;
-         this.duckDbConnection = null;
-         this.repository = null;
-         this.defaultManifestStore = null;
-         this.projectManifestStores.clear();
-         this.attachedCatalogs.clear();
-      }
-   }
-
-   isInitialized(): boolean {
-      return this.connection !== null && this.repository !== null;
-   }
-}

package/src/storage/duckdb/ConnectionRepository.ts

@@ -1,141 +0,0 @@
-import { Connection } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-export class ConnectionRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
-   }
-
-   private now(): Date {
-      return new Date();
-   }
-
-   async listConnections(projectId: string): Promise<Connection[]> {
-      try {
-         const rows = await this.db.all<Record<string, unknown>>(
-            "SELECT * FROM connections WHERE project_id = ? ORDER BY name",
-            [projectId],
-         );
-         return rows.map(this.mapToConnection);
-      } catch (err: unknown) {
-         const error = err as Error;
-         console.error("Failed to get connections:", error.message);
-         throw error;
-      }
-   }
-
-   async getConnectionById(id: string): Promise<Connection | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM connections WHERE id = ?",
-         [id],
-      );
-      return row ? this.mapToConnection(row) : null;
-   }
-
-   async getConnectionByName(
-      projectId: string,
-      name: string,
-   ): Promise<Connection | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM connections WHERE project_id = ? AND name = ?",
-         [projectId, name],
-      );
-      return row ? this.mapToConnection(row) : null;
-   }
-
-   async createConnection(
-      connection: Omit<Connection, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Connection> {
-      const id = this.generateId();
-      const now = this.now();
-
-      try {
-         const configJson = JSON.stringify(connection.config);
-
-         await this.db.run(
-            `INSERT INTO connections (id, project_id, name, type, config, created_at, updated_at)
-        VALUES (?, ?, ?, ?, ?, ?, ?)`,
-            [
-               id,
-               connection.projectId,
-               connection.name,
-               connection.type,
-               configJson,
-               now.toISOString(),
-               now.toISOString(),
-            ],
-         );
-
-         return {
-            id,
-            ...connection,
-            createdAt: now,
-            updatedAt: now,
-         };
-      } catch (err: unknown) {
-         const error = err as Error;
-         console.error("Failed to create connection:", error.message);
-         throw error;
-      }
-   }
-
-   async updateConnection(
-      id: string,
-      updates: Partial<Connection>,
-   ): Promise<Connection> {
-      const existing = await this.getConnectionById(id);
-      if (!existing) {
-         throw new Error(`Connection with id ${id} not found`);
-      }
-
-      const now = this.now();
-      const setClauses: string[] = [];
-      const params: unknown[] = [];
-
-      if (updates.name !== undefined) {
-         setClauses.push(`name = ?`);
-         params.push(updates.name);
-      }
-      if (updates.type !== undefined) {
-         setClauses.push(`type = ?`);
-         params.push(updates.type);
-      }
-      if (updates.config !== undefined) {
-         setClauses.push(`config = ?`);
-         params.push(JSON.stringify(updates.config));
-      }
-
-      setClauses.push(`updated_at = ?`);
-      params.push(now.toISOString());
-      params.push(id);
-
-      await this.db.run(
-         `UPDATE connections SET ${setClauses.join(", ")} WHERE id = ?`,
-         params,
-      );
-
-      return this.getConnectionById(id) as Promise<Connection>;
-   }
-
-   async deleteConnection(id: string): Promise<void> {
-      await this.db.run("DELETE FROM connections WHERE id = ?", [id]);
-   }
-
-   async deleteConnectionsByProjectId(id: string): Promise<void> {
-      await this.db.run("DELETE FROM connections WHERE project_id = ?", [id]);
-   }
-
-   private mapToConnection(row: Record<string, unknown>): Connection {
-      return {
-         id: row.id as string,
-         projectId: row.project_id as string,
-         name: row.name as string,
-         type: row.type as Connection["type"],
-         config: JSON.parse(row.config as string),
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}