@malloy-publisher/server 0.0.382-dev → 0.2.0

package/src/storage/duckdb/schema.ts

@@ -1,147 +0,0 @@
-import { logger } from "../../logger";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-export async function initializeSchema(
-   db: DuckDBConnection,
-   force: boolean = false,
-): Promise<void> {
-   const initialized = await db.isInitialized();
-
-   if (initialized && !force) {
-      return;
-   }
-
-   if (force) {
-      logger.info(
-         "Reinitializing database schema dropping and recreating all tables",
-      );
-      await dropAllTables(db);
-   } else {
-      logger.info("Creating database schema for the first time...");
-   }
-
-   // Projects table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS projects (
-      id VARCHAR PRIMARY KEY,
-      name VARCHAR NOT NULL UNIQUE,
-      path VARCHAR NOT NULL,
-      description VARCHAR,
-      metadata JSON,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL
-    )
-  `);
-
-   // Packages table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS packages (
-      id VARCHAR PRIMARY KEY,
-      project_id VARCHAR NOT NULL,
-      name VARCHAR NOT NULL,
-      description VARCHAR,
-      manifest_path VARCHAR NOT NULL,
-      metadata JSON,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (project_id) REFERENCES projects(id),
-      UNIQUE (project_id, name)
-    )
-  `);
-
-   // Connections table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS connections (
-      id VARCHAR PRIMARY KEY,
-      project_id VARCHAR NOT NULL,
-      name VARCHAR NOT NULL,
-      type VARCHAR NOT NULL,
-      config JSON NOT NULL,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (project_id) REFERENCES projects(id),
-      UNIQUE (project_id, name)
-    )
-  `);
-
-   // Materializations table.
-   //
-   // `active_key` enforces at-most-one active (PENDING or RUNNING)
-   // materialization per (project, package) at the DB layer. It is set to
-   // `{project_id}|{package_name}` while the row is active and cleared
-   // to NULL on transition to any terminal state. A unique index on
-   // `active_key` (see below) makes the insert-then-check race impossible —
-   // a second concurrent create fails with a constraint violation, which the
-   // service layer translates to `MaterializationConflictError`.
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS materializations (
-      id VARCHAR PRIMARY KEY,
-      project_id VARCHAR NOT NULL,
-      package_name VARCHAR NOT NULL,
-      status VARCHAR NOT NULL,
-      active_key VARCHAR,
-      started_at TIMESTAMP,
-      completed_at TIMESTAMP,
-      error TEXT,
-      metadata JSON,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (project_id) REFERENCES projects(id)
-    )
-  `);
-
-   // Build manifests table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS build_manifests (
-      id VARCHAR PRIMARY KEY,
-      project_id VARCHAR NOT NULL,
-      package_name VARCHAR NOT NULL,
-      build_id VARCHAR NOT NULL,
-      table_name VARCHAR NOT NULL,
-      source_name VARCHAR NOT NULL,
-      connection_name VARCHAR NOT NULL,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (project_id) REFERENCES projects(id),
-      UNIQUE (project_id, package_name, build_id)
-    )
-  `);
-
-   // Create indexes for better query performance
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_packages_project_id ON packages(project_id)",
-   );
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_connections_project_id ON connections(project_id)",
-   );
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_materializations_project_package ON materializations(project_id, package_name)",
-   );
-   await db.run(
-      "CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
-   );
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_build_manifests_project_package ON build_manifests(project_id, package_name)",
-   );
-}
-
-async function dropAllTables(db: DuckDBConnection): Promise<void> {
-   const tables = [
-      "build_manifests",
-      "materializations",
-      "packages",
-      "connections",
-      "projects",
-   ];
-
-   logger.info("Dropping tables:", tables.join(", "));
-
-   for (const table of tables) {
-      try {
-         await db.run(`DROP TABLE IF EXISTS ${table} `);
-         logger.info(`Dropped table: ${table}`);
-      } catch (err) {
-         logger.warn(` Warning: Could not drop table ${table}:`, err);
-      }
-   }
-}

package/src/storage/ducklake/DuckLakeManifestStore.ts

@@ -1,146 +0,0 @@
-import { logger } from "../../logger";
-import {
-   BuildManifest,
-   ManifestEntry,
-   ManifestStore,
-} from "../DatabaseInterface";
-import { DuckDBConnection } from "../duckdb/DuckDBConnection";
-
-/**
- * DuckLake-backed ManifestStore used in orchestrated mode.
- *
- * Reads and writes manifest entries through a DuckLake catalog attached to
- * the publisher's internal DuckDB. All workers sharing the same DuckLake
- * catalog see the same manifest, enabling multi-worker coordination.
- *
- * The catalog is attached by {@link StorageManager} before this store is
- * instantiated. The first worker to call {@link bootstrapSchema} creates the
- * `build_manifests` table idempotently. Schema ownership lives in the
- * publisher so that DDL and code co-evolve in the same repo.
- *
- * **Scope: manifest sync only.** Orchestrated mode only shares manifest
- * state across workers. The active-materialization lock (the unique index
- * on `materializations.active_key`) still lives in each worker's local
- * DuckDB, so two workers can run builds concurrently and race on physical
- * table names, staging table names, and manifest writes. Until a shared
- * build lease lives in the DuckLake catalog, deployments running
- * orchestrated mode must ensure builds are externally single-writer (e.g.,
- * one designated build worker, or an external job scheduler). Other
- * workers should only call `manifest?action=reload` to pick up manifests produced
- * by the build worker.
- */
-export class DuckLakeManifestStore implements ManifestStore {
-   private readonly table: string;
-
-   constructor(
-      private db: DuckDBConnection,
-      catalogName: string,
-   ) {
-      this.table = `${catalogName}.build_manifests`;
-   }
-
-   /**
-    * Idempotently creates the `build_manifests` table and indices in the
-    * DuckLake catalog. Safe to call from every worker on startup.
-    */
-   async bootstrapSchema(): Promise<void> {
-      await this.db.run(`
-         CREATE TABLE IF NOT EXISTS ${this.table} (
-            id VARCHAR,
-            project_id VARCHAR NOT NULL,
-            package_name VARCHAR NOT NULL,
-            build_id VARCHAR NOT NULL,
-            table_name VARCHAR NOT NULL,
-            source_name VARCHAR NOT NULL,
-            connection_name VARCHAR NOT NULL,
-            created_at TIMESTAMP NOT NULL,
-            updated_at TIMESTAMP NOT NULL
-         )
-      `);
-      logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
-   }
-
-   async getManifest(
-      projectId: string,
-      packageName: string,
-   ): Promise<BuildManifest> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
-         [projectId, packageName],
-      );
-      const manifest: BuildManifest = { entries: {}, strict: false };
-      for (const row of rows) {
-         const buildId = row.build_id as string;
-         // Rows are ordered newest-first; keep only the latest per build_id
-         // to handle rare duplicates from cross-worker races.
-         if (!manifest.entries[buildId]) {
-            manifest.entries[buildId] = {
-               tableName: row.table_name as string,
-            };
-         }
-      }
-      return manifest;
-   }
-
-   /**
-    * Insert a manifest entry. If a row with the same `build_id` already
-    * exists (retry after crash), the duplicate is harmless:
-    * {@link getManifest} deduplicates by build_id keeping the newest row.
-    */
-   async writeEntry(
-      projectId: string,
-      packageName: string,
-      buildId: string,
-      tableName: string,
-      sourceName: string,
-      connectionName: string,
-   ): Promise<void> {
-      const now = new Date().toISOString();
-      const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
-
-      await this.db.run(
-         `INSERT INTO ${this.table} (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
-          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-         [
-            id,
-            projectId,
-            packageName,
-            buildId,
-            tableName,
-            sourceName,
-            connectionName,
-            now,
-            now,
-         ],
-      );
-   }
-
-   async deleteEntry(id: string): Promise<void> {
-      await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
-   }
-
-   async listEntries(
-      projectId: string,
-      packageName: string,
-   ): Promise<ManifestEntry[]> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
-         [projectId, packageName],
-      );
-      return rows.map(this.mapToEntry);
-   }
-
-   private mapToEntry(row: Record<string, unknown>): ManifestEntry {
-      return {
-         id: row.id as string,
-         projectId: row.project_id as string,
-         packageName: row.package_name as string,
-         buildId: row.build_id as string,
-         tableName: row.table_name as string,
-         sourceName: row.source_name as string,
-         connectionName: row.connection_name as string,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}

package/src/utils.ts

@@ -1,13 +0,0 @@
-import { URLReader } from "@malloydata/malloy";
-import * as fs from "fs";
-import { fileURLToPath } from "url";
-
-export const URL_READER: URLReader = {
-   readURL: (url: URL) => {
-      let path = url.toString();
-      if (url.protocol == "file:") {
-         path = fileURLToPath(url);
-      }
-      return fs.promises.readFile(path, "utf8");
-   },
-};

package/tests/fixtures/persist-test/data/orders.csv

@@ -1,5 +0,0 @@
-category,amount
-electronics,100
-clothing,50
-electronics,200
-clothing,75

package/tests/fixtures/persist-test/persist_test.malloy

@@ -1,11 +0,0 @@
-##! experimental.persistence
-
-source: raw_orders is duckdb.table('data/orders.csv')
-
-#@ persist name="order_summary"
-source: order_summary is raw_orders -> {
-  group_by: category
-  aggregate:
-    total_orders is count()
-    total_revenue is amount.sum()
-}

package/tests/fixtures/persist-test/publisher.json

@@ -1,5 +0,0 @@
-{
-  "name": "persist-test",
-  "version": "1.0.0",
-  "description": "DuckDB persist test fixture"
-}

package/tests/fixtures/publisher.config.json

@@ -1,15 +0,0 @@
-{
-  "frozenConfig": false,
-  "projects": [
-    {
-      "name": "test-project",
-      "packages": [
-        {
-          "name": "persist-test",
-          "location": "./persist-test"
-        }
-      ],
-      "connections": []
-    }
-  ]
-}

package/tests/harness/e2e.ts

@@ -1,96 +0,0 @@
-import { Client } from "@modelcontextprotocol/sdk/client/index.js";
-import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
-import type {
-   Notification,
-   Request,
-   Result,
-} from "@modelcontextprotocol/sdk/types.js";
-import http from "http";
-import path from "path";
-import { fileURLToPath } from "url";
-import { URL } from "url";
-
-/**
- * E2E environment descriptor returned by {@link startE2E}.
- */
-export interface E2EEnv {
-   httpServer: http.Server;
-   serverUrl: string;
-   mcpClient: Client<Request, Notification, Result>;
-}
-
-let originalServerRoot: string | undefined;
-
-/**
- * Spin-up the real Express app (via mcp/server.ts), expose its base URL and
- * return a fully-connected MCP client. The caller **must** invoke {@link stop}
- * once its suite finishes.
- */
-export async function startE2E(): Promise<E2EEnv & { stop(): Promise<void> }> {
-   //--------------------------------------------------------------------------
-   // 1.  Set SERVER_ROOT so ProjectStore loader finds publisher.config.json
-   //--------------------------------------------------------------------------
-   originalServerRoot = process.env.SERVER_ROOT;
-   // Use import.meta.url for cross-platform compatibility (works on Windows)
-   const __filename = fileURLToPath(import.meta.url);
-   const __dirname = path.dirname(__filename);
-   const serverPackageDir = path.resolve(__dirname, "../../../"); // packages/server
-   process.env.SERVER_ROOT = serverPackageDir;
-
-   //--------------------------------------------------------------------------
-   // 2.  Dynamically import the configured Express app AFTER env var mutation
-   //--------------------------------------------------------------------------
-   const { mcpApp } = await import("../../src/server");
-
-   //--------------------------------------------------------------------------
-   // 3.  Start HTTP server on a predictable high port (4042 by default)
-   //--------------------------------------------------------------------------
-   const TEST_PORT = Number(process.env.MCP_PORT || 4040) + 2;
-
-   const httpServer: http.Server = await new Promise<http.Server>(
-      (resolve, reject) => {
-         const srv = http
-            .createServer(mcpApp)
-            .listen(TEST_PORT, "127.0.0.1", () => resolve(srv));
-
-         srv.on("error", (err: NodeJS.ErrnoException) => {
-            /* c8 ignore next 3 */
-            console.error("[E2E] server listen error", err);
-            reject(err);
-         });
-      },
-   );
-
-   const serverUrl = `http://127.0.0.1:${TEST_PORT}`;
-
-   //--------------------------------------------------------------------------
-   // 4.  Connect MCP client over HTTP streaming bridge
-   //--------------------------------------------------------------------------
-   const mcpClient = new Client<Request, Notification, Result>({
-      name: "mcp-e2e-client",
-      version: "1.0",
-   });
-
-   const transport = new StreamableHTTPClientTransport(
-      new URL(`${serverUrl}/mcp`),
-   );
-
-   await mcpClient.connect(transport);
-
-   //--------------------------------------------------------------------------
-   // 5.  Return env + graceful stop helper
-   //--------------------------------------------------------------------------
-   const stop = async (): Promise<void> => {
-      try {
-         await mcpClient.close();
-      } finally {
-         httpServer.closeAllConnections?.();
-         await new Promise<void>((r) => httpServer.close(() => r()));
-         // Restore SERVER_ROOT
-         if (originalServerRoot === undefined) delete process.env.SERVER_ROOT;
-         else process.env.SERVER_ROOT = originalServerRoot;
-      }
-   };
-
-   return { httpServer, serverUrl, mcpClient, stop };
-}