@malloy-publisher/server 0.0.194 → 0.0.196-dev

package/dist/server.mjs

@@ -220381,6 +220381,7 @@ import fs from "fs/promises";
 import path2 from "path";
 
 // src/service/connection_config.ts
+import { createPrivateKey } from "crypto";
 import path from "path";
 var PUBLISHER_DUCKDB_API_FIELDS = new Set(["attachedDatabases"]);
 function normalizeSnowflakePrivateKey(privateKey) {
@@ -220399,6 +220400,12 @@ function normalizeSnowflakePrivateKey(privateKey) {
         endRegex: /-----END\s+PRIVATE\s+KEY-----/i,
         beginMarker: "-----BEGIN PRIVATE KEY-----",
         endMarker: "-----END PRIVATE KEY-----"
+      },
+      {
+        beginRegex: /-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i,
+        endRegex: /-----END\s+RSA\s+PRIVATE\s+KEY-----/i,
+        beginMarker: "-----BEGIN RSA PRIVATE KEY-----",
+        endMarker: "-----END RSA PRIVATE KEY-----"
       }
     ];
     for (const pattern of keyPatterns) {
@@ -220425,6 +220432,21 @@ ${pattern.endMarker}
     privateKeyContent += `
 `;
   }
+  if (/-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i.test(privateKeyContent)) {
+    try {
+      privateKeyContent = createPrivateKey({
+        key: privateKeyContent,
+        format: "pem"
+      }).export({ type: "pkcs8", format: "pem" }).toString();
+    } catch (err) {
+      throw new Error(`Failed to convert Snowflake RSA private key (PKCS#1) to PKCS#8: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!privateKeyContent.endsWith(`
+`)) {
+      privateKeyContent += `
+`;
+    }
+  }
   return privateKeyContent;
 }
 function validateDuckdbApiSurface(connection) {
@@ -224989,7 +225011,7 @@ class Mutex {
 }
 
 // src/service/project_store.ts
-import crypto3 from "crypto";
+import crypto4 from "crypto";
 import * as fs7 from "fs";
 import * as path9 from "path";
 
@@ -229231,6 +229253,9 @@ function registerHealthEndpoints(app) {
   });
 }
 
+// src/storage/StorageManager.ts
+import * as crypto3 from "crypto";
+
 // src/storage/duckdb/DuckDBConnection.ts
 import duckdb from "duckdb";
 import * as path5 from "path";
@@ -230146,15 +230171,17 @@ async function dropAllTables(db) {
 class DuckLakeManifestStore {
   db;
   table;
-  constructor(db, catalogName) {
+  projectName;
+  constructor(db, catalogName, projectName) {
     this.db = db;
     this.table = `${catalogName}.build_manifests`;
+    this.projectName = projectName;
   }
   async bootstrapSchema() {
     await this.db.run(`
          CREATE TABLE IF NOT EXISTS ${this.table} (
             id VARCHAR,
-            project_id VARCHAR NOT NULL,
+            project_name VARCHAR NOT NULL,
             package_name VARCHAR NOT NULL,
             build_id VARCHAR NOT NULL,
             table_name VARCHAR NOT NULL,
@@ -230166,8 +230193,8 @@ class DuckLakeManifestStore {
       `);
     logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
   }
-  async getManifest(projectId, packageName) {
-    const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`, [projectId, packageName]);
+  async getManifest(_projectId, packageName) {
+    const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.projectName, packageName]);
     const manifest = { entries: {}, strict: false };
     for (const row of rows) {
       const buildId = row.build_id;
@@ -230179,13 +230206,13 @@ class DuckLakeManifestStore {
     }
     return manifest;
   }
-  async writeEntry(projectId, packageName, buildId, tableName, sourceName, connectionName) {
+  async writeEntry(_projectId, packageName, buildId, tableName, sourceName, connectionName) {
     const now = new Date().toISOString();
     const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
-    await this.db.run(`INSERT INTO ${this.table} (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
+    await this.db.run(`INSERT INTO ${this.table} (id, project_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
       id,
-      projectId,
+      this.projectName,
       packageName,
       buildId,
       tableName,
@@ -230198,14 +230225,14 @@ class DuckLakeManifestStore {
   async deleteEntry(id) {
     await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
   }
-  async listEntries(projectId, packageName) {
-    const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`, [projectId, packageName]);
+  async listEntries(_projectId, packageName) {
+    const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.projectName, packageName]);
     return rows.map(this.mapToEntry);
   }
   mapToEntry(row) {
     return {
       id: row.id,
-      projectId: row.project_id,
+      projectId: row.project_name,
       packageName: row.package_name,
       buildId: row.build_id,
       tableName: row.table_name,
@@ -230221,6 +230248,13 @@ class DuckLakeManifestStore {
 function escapeSQL2(value) {
   return value.replace(/'/g, "''");
 }
+function configKey(c) {
+  return `${c.catalogUrl}|${c.dataPath}`;
+}
+function catalogNameForConfig(c) {
+  const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
+  return `manifest_lake_${hash}`;
+}
 
 class StorageManager {
   connection = null;
@@ -230228,7 +230262,7 @@ class StorageManager {
   repository = null;
   defaultManifestStore = null;
   projectManifestStores = new Map;
-  attachedCatalogs = new Set;
+  attachedCatalogs = new Map;
   config;
   constructor(config) {
     this.config = config;
@@ -230261,19 +230295,23 @@ class StorageManager {
     this.repository = new DuckDBRepository(connection);
     this.defaultManifestStore = new DuckDBManifestStore(this.repository);
   }
-  async initializeDuckLakeForProject(projectId, config) {
+  async initializeDuckLakeForProject(projectId, projectName, config) {
     if (!this.duckDbConnection) {
       throw new Error("Storage not initialized. Call initialize() first.");
     }
-    const catalogName = `manifest_lake_${projectId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
-    if (!this.attachedCatalogs.has(catalogName)) {
+    const key = configKey(config);
+    let catalogName = this.attachedCatalogs.get(key);
+    if (!catalogName) {
+      catalogName = catalogNameForConfig(config);
       await this.attachDuckLakeCatalog(config, catalogName);
+      this.attachedCatalogs.set(key, catalogName);
     }
-    const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName);
+    const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, projectName);
     await store.bootstrapSchema();
     this.projectManifestStores.set(projectId, store);
     logger.info("DuckLake manifest store initialized for project", {
       projectId,
+      projectName,
       catalogName
     });
   }
@@ -230288,14 +230326,16 @@ class StorageManager {
     const escapedDataPath = escapeSQL2(config.dataPath);
     const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
     let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
-    const attachOpts = [`DATA_PATH '${escapedDataPath}'`];
+    const attachOpts = [
+      `DATA_PATH '${escapedDataPath}'`,
+      "DATA_INLINING_ROW_LIMIT 100000"
+    ];
     if (isCloudStorage) {
       attachOpts.push("OVERRIDE_DATA_PATH true");
     }
     attachCmd += ` (${attachOpts.join(", ")});`;
     logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
     await connection.run(attachCmd);
-    this.attachedCatalogs.add(catalogName);
   }
   getRepository() {
     if (!this.repository) {
@@ -230379,8 +230419,8 @@ import {
   MalloySQLParser,
   MalloySQLStatementType
 } from "@malloydata/malloy-sql";
-import { createRequire as createRequire2 } from "module";
 import * as fs4 from "fs/promises";
+import { createRequire as createRequire2 } from "module";
 import * as path6 from "path";
 
 // src/data_styles.ts
@@ -231010,7 +231050,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
     }
     const modelURL = new URL(`file://${fullModelPath}`);
     const baseUrl = new URL(".", modelURL);
-    const importBaseURL = new URL(baseUrl.pathname + "/", "file:");
+    const importBaseURL = baseUrl;
     const urlReader = new HackyDataStylesAccumulator(URL_READER);
     const runtime = new Runtime({
       urlReader,
@@ -231556,6 +231596,9 @@ class Project {
       this.metadata.readme = payload.readme;
       await this.writeProjectReadme(payload.readme);
     }
+    if (payload.materializationStorage !== undefined) {
+      this.metadata.materializationStorage = payload.materializationStorage;
+    }
     if (payload.connections) {
       const payloadConnections = payload.connections;
       await this.runConnectionUpdateExclusive(async () => {
@@ -232131,7 +232174,7 @@ class ProjectStore {
     }
     const materializationStorage = project.metadata?.materializationStorage;
     if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
-      await this.storageManager.initializeDuckLakeForProject(dbProject.id, {
+      await this.storageManager.initializeDuckLakeForProject(dbProject.id, dbProject.name, {
         catalogUrl: materializationStorage.catalogUrl,
         dataPath: materializationStorage.dataPath
       });
@@ -232407,6 +232450,9 @@ class ProjectStore {
     if (!newProject.metadata)
       newProject.metadata = {};
     newProject.metadata.location = absoluteProjectPath;
+    if (project.materializationStorage !== undefined) {
+      newProject.metadata.materializationStorage = project.materializationStorage;
+    }
     this.projects.set(projectName, newProject);
     project?.packages?.forEach((_package) => {
       if (_package.name) {
@@ -232561,7 +232607,7 @@ class ProjectStore {
       });
     }
     for (const [groupedLocation, packagesForLocation] of locationGroups) {
-      const locationHash = crypto3.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
+      const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
       const tempDownloadPath = `${absoluteTargetPath}/.temp_${locationHash}`;
       await fs7.promises.mkdir(tempDownloadPath, { recursive: true });
       logger.info(`Created temporary directory: ${tempDownloadPath}`);

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.194",
+   "version": "0.0.196-dev",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -16,7 +16,7 @@
    "scripts": {
       "test": "bun run test:unit && bun run test:integration",
       "test:unit": "bun test --timeout 100000 src",
-      "test:integration": "bun test --timeout 100000 tests",
+      "test:integration": "bun test --timeout 200000 tests --max-workers=1",
       "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
       "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
       "start": "NODE_ENV=production bun run ./dist/server.mjs",

package/src/service/connection_config.spec.ts

@@ -1,5 +1,9 @@
+import { generateKeyPairSync } from "crypto";
 import { describe, expect, it } from "bun:test";
-import { assembleProjectConnections } from "./connection_config";
+import {
+   assembleProjectConnections,
+   normalizeSnowflakePrivateKey,
+} from "./connection_config";
 import { components } from "../api";
 
 type ApiConnection = components["schemas"]["Connection"];
@@ -121,3 +125,44 @@ describe("assembleProjectConnections — databricks", () => {
       );
    });
 });
+
+describe("normalizeSnowflakePrivateKey", () => {
+   const { privateKey: pkcs8Pem } = generateKeyPairSync("rsa", {
+      modulusLength: 2048,
+      privateKeyEncoding: { type: "pkcs8", format: "pem" },
+      publicKeyEncoding: { type: "spki", format: "pem" },
+   });
+   const { privateKey: pkcs1Pem } = generateKeyPairSync("rsa", {
+      modulusLength: 2048,
+      privateKeyEncoding: { type: "pkcs1", format: "pem" },
+      publicKeyEncoding: { type: "pkcs1", format: "pem" },
+   });
+
+   it("passes a multi-line PKCS#8 key through and adds a trailing newline", () => {
+      const trimmed = (pkcs8Pem as string).trimEnd();
+      const result = normalizeSnowflakePrivateKey(trimmed);
+      expect(result).toContain("-----BEGIN PRIVATE KEY-----");
+      expect(result.endsWith("\n")).toBe(true);
+   });
+
+   it("converts a multi-line PKCS#1 RSA key to PKCS#8", () => {
+      const result = normalizeSnowflakePrivateKey(pkcs1Pem as string);
+      expect(result).toContain("-----BEGIN PRIVATE KEY-----");
+      expect(result).not.toContain("BEGIN RSA PRIVATE KEY");
+      expect(result.endsWith("\n")).toBe(true);
+   });
+
+   it("converts a single-line PKCS#1 RSA key to PKCS#8", () => {
+      const singleLine = (pkcs1Pem as string).replace(/\n/g, "");
+      const result = normalizeSnowflakePrivateKey(singleLine);
+      expect(result).toContain("-----BEGIN PRIVATE KEY-----");
+      expect(result).not.toContain("BEGIN RSA PRIVATE KEY");
+   });
+
+   it("reconstructs a single-line PKCS#8 key without conversion", () => {
+      const singleLine = (pkcs8Pem as string).replace(/\n/g, "");
+      const result = normalizeSnowflakePrivateKey(singleLine);
+      expect(result.startsWith("-----BEGIN PRIVATE KEY-----\n")).toBe(true);
+      expect(result.endsWith("-----END PRIVATE KEY-----\n")).toBe(true);
+   });
+});

package/src/service/connection_config.ts

@@ -1,3 +1,4 @@
+import { createPrivateKey } from "crypto";
 import path from "path";
 import { components } from "../api";
 
@@ -48,6 +49,12 @@ export function normalizeSnowflakePrivateKey(privateKey: string): string {
             beginMarker: "-----BEGIN PRIVATE KEY-----",
             endMarker: "-----END PRIVATE KEY-----",
          },
+         {
+            beginRegex: /-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i,
+            endRegex: /-----END\s+RSA\s+PRIVATE\s+KEY-----/i,
+            beginMarker: "-----BEGIN RSA PRIVATE KEY-----",
+            endMarker: "-----END RSA PRIVATE KEY-----",
+         },
       ];
 
       for (const pattern of keyPatterns) {
@@ -73,6 +80,28 @@ export function normalizeSnowflakePrivateKey(privateKey: string): string {
       privateKeyContent += "\n";
    }
 
+   // Snowflake's Node SDK requires PKCS#8 ("BEGIN PRIVATE KEY"). Convert
+   // PKCS#1 ("BEGIN RSA PRIVATE KEY") so users can paste either format.
+   if (/-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i.test(privateKeyContent)) {
+      try {
+         privateKeyContent = createPrivateKey({
+            key: privateKeyContent,
+            format: "pem",
+         })
+            .export({ type: "pkcs8", format: "pem" })
+            .toString();
+      } catch (err) {
+         throw new Error(
+            `Failed to convert Snowflake RSA private key (PKCS#1) to PKCS#8: ${
+               err instanceof Error ? err.message : String(err)
+            }`,
+         );
+      }
+      if (!privateKeyContent.endsWith("\n")) {
+         privateKeyContent += "\n";
+      }
+   }
+
    return privateKeyContent;
 }
 

package/src/service/model.ts

@@ -22,10 +22,10 @@ import {
    MalloySQLParser,
    MalloySQLStatementType,
 } from "@malloydata/malloy-sql";
-import { createRequire } from "module";
 import { DataStyles } from "@malloydata/render";
 import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
+import { createRequire } from "module";
 import * as path from "path";
 import { components } from "../api";
 import {
@@ -703,7 +703,7 @@ export class Model {
 
       const modelURL = new URL(`file://${fullModelPath}`);
       const baseUrl = new URL(".", modelURL);
-      const importBaseURL = new URL(baseUrl.pathname + "/", "file:");
+      const importBaseURL = baseUrl;
       const urlReader = new HackyDataStylesAccumulator(URL_READER);
 
       // Request runtimes borrow the cached package MalloyConfig. The package

package/src/service/project.ts

@@ -99,6 +99,10 @@ export class Project {
          await this.writeProjectReadme(payload.readme);
       }
 
+      if (payload.materializationStorage !== undefined) {
+         this.metadata.materializationStorage = payload.materializationStorage;
+      }
+
       // Handle connections update
       // TODO: Update project connections should have its own API endpoint
       if (payload.connections) {

package/src/service/project_store.spec.ts

@@ -10,6 +10,12 @@ import { ProjectStore } from "./project_store";
 
 type MockData = Record<string, unknown>;
 
+const initializeDuckLakeCalls: Array<{
+   projectId: string;
+   projectName: string;
+   config: { catalogUrl: string; dataPath: string };
+}> = [];
+
 mock.module("../storage/StorageManager", () => {
    return {
       StorageManager: class MockStorageManager {
@@ -17,6 +23,14 @@ mock.module("../storage/StorageManager", () => {
             return;
          }
 
+         async initializeDuckLakeForProject(
+            projectId: string,
+            projectName: string,
+            config: { catalogUrl: string; dataPath: string },
+         ): Promise<void> {
+            initializeDuckLakeCalls.push({ projectId, projectName, config });
+         }
+
          getRepository() {
             return {
                // ===== PROJECT METHODS =====
@@ -467,6 +481,69 @@ describe("ProjectStore Service", () => {
       expect(readmeContent).toBe("Updated README content");
    });
 
+   it("should propagate materializationStorage on addProject for new project", async () => {
+      writeFileSync(
+         path.join(serverRootPath, "publisher.config.json"),
+         JSON.stringify({ frozenConfig: false, projects: [] }),
+      );
+
+      await projectStore.finishedInitialization;
+
+      const materializationStorage = {
+         catalogUrl:
+            "postgres:host=localhost port=5432 dbname=ducklake user=u password=p",
+         dataPath: "gs://test-bucket",
+      };
+
+      initializeDuckLakeCalls.length = 0;
+      const project = await projectStore.addProject({
+         name: projectName,
+         materializationStorage,
+      });
+
+      expect(project.metadata.materializationStorage).toEqual(
+         materializationStorage,
+      );
+      expect(initializeDuckLakeCalls).toHaveLength(1);
+      expect(initializeDuckLakeCalls[0].config).toEqual(materializationStorage);
+   });
+
+   it("should propagate materializationStorage on update", async () => {
+      const projectPath = path.join(serverRootPath, projectName);
+      mkdirSync(projectPath, { recursive: true });
+      writeFileSync(
+         path.join(projectPath, "publisher.json"),
+         JSON.stringify({ name: projectName, description: "Test package" }),
+      );
+      writeFileSync(
+         path.join(serverRootPath, "publisher.config.json"),
+         JSON.stringify({
+            frozenConfig: false,
+            projects: [
+               {
+                  name: projectName,
+                  packages: [{ name: projectName, location: projectPath }],
+               },
+            ],
+         }),
+      );
+
+      await projectStore.finishedInitialization;
+      const project = await projectStore.getProject(projectName);
+
+      const materializationStorage = {
+         catalogUrl:
+            "postgres:host=localhost port=5432 dbname=ducklake user=u password=p",
+         dataPath: "gs://test-bucket",
+      };
+
+      await project.update({ name: projectName, materializationStorage });
+
+      expect(project.metadata.materializationStorage).toEqual(
+         materializationStorage,
+      );
+   });
+
    it(
       "should handle project reload",
       async () => {

package/src/service/project_store.ts

@@ -359,10 +359,14 @@ export class ProjectStore {
          materializationStorage?.catalogUrl &&
          materializationStorage?.dataPath
       ) {
-         await this.storageManager.initializeDuckLakeForProject(dbProject.id, {
-            catalogUrl: materializationStorage.catalogUrl,
-            dataPath: materializationStorage.dataPath,
-         });
+         await this.storageManager.initializeDuckLakeForProject(
+            dbProject.id,
+            dbProject.name,
+            {
+               catalogUrl: materializationStorage.catalogUrl,
+               dataPath: materializationStorage.dataPath,
+            },
+         );
       }
 
       return dbProject;
@@ -807,6 +811,10 @@ export class ProjectStore {
 
       if (!newProject.metadata) newProject.metadata = {};
       newProject.metadata.location = absoluteProjectPath;
+      if (project.materializationStorage !== undefined) {
+         newProject.metadata.materializationStorage =
+            project.materializationStorage;
+      }
 
       this.projects.set(projectName, newProject);
 

package/src/storage/StorageManager.ts

@@ -1,3 +1,4 @@
+import * as crypto from "crypto";
 import { logger } from "../logger";
 import {
    DatabaseConnection,
@@ -42,6 +43,19 @@ function escapeSQL(value: string): string {
    return value.replace(/'/g, "''");
 }
 
+function configKey(c: DuckLakeManifestConfig): string {
+   return `${c.catalogUrl}|${c.dataPath}`;
+}
+
+function catalogNameForConfig(c: DuckLakeManifestConfig): string {
+   const hash = crypto
+      .createHash("sha256")
+      .update(configKey(c))
+      .digest("hex")
+      .slice(0, 8);
+   return `manifest_lake_${hash}`;
+}
+
 /**
  * Manages the storage backend (DuckDB, Postgres, etc.) and per-project
  * manifest stores. Projects without `materializationStorage` config use
@@ -57,8 +71,12 @@ export class StorageManager {
    /** Per-project DuckLake manifest stores, keyed by projectId. */
    private projectManifestStores = new Map<string, ManifestStore>();
 
-   /** Tracks which catalogs have been attached to avoid duplicate ATTACHes. */
-   private attachedCatalogs = new Set<string>();
+   /**
+    * Tracks attached DuckLake catalogs as `configKey -> catalogName`. Each
+    * unique materializationStorage config gets its own ATTACHment under a
+    * deterministic catalog name, so multiple configs can coexist on one worker.
+    */
+   private attachedCatalogs = new Map<string, string>();
 
    private config: StorageConfig;
 
@@ -105,32 +123,44 @@ export class StorageManager {
 
    /**
     * Lazily initializes a DuckLake manifest store for a project.
-    * The DuckLake catalog is attached to the shared DuckDB connection
-    * and persists for the lifetime of the process.
+    *
+    * One shared catalog per materializationStorage config: every project
+    * pointing at the same (catalogUrl, dataPath) shares one `build_manifests`
+    * table inside it, partitioned by `project_id` (set to the project's name
+    * so it's stable across worker replicas — required for cross-pod manifest
+    * visibility in orchestrated mode). Different configs (e.g. different
+    * orgs) attach as separate catalogs under distinct deterministic aliases.
     */
    async initializeDuckLakeForProject(
       projectId: string,
+      projectName: string,
       config: DuckLakeManifestConfig,
    ): Promise<void> {
       if (!this.duckDbConnection) {
          throw new Error("Storage not initialized. Call initialize() first.");
       }
 
-      const catalogName = `manifest_lake_${projectId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
-
-      if (!this.attachedCatalogs.has(catalogName)) {
+      const key = configKey(config);
+      let catalogName = this.attachedCatalogs.get(key);
+      if (!catalogName) {
+         // Catalog name derived from the config so multiple configs can coexist as
+         // separate ATTACHments without colliding on the name.
+         catalogName = catalogNameForConfig(config);
          await this.attachDuckLakeCatalog(config, catalogName);
+         this.attachedCatalogs.set(key, catalogName);
       }
 
       const store = new DuckLakeManifestStore(
          this.duckDbConnection,
          catalogName,
+         projectName,
       );
       await store.bootstrapSchema();
 
       this.projectManifestStores.set(projectId, store);
       logger.info("DuckLake manifest store initialized for project", {
          projectId,
+         projectName,
          catalogName,
       });
    }
@@ -155,7 +185,14 @@ export class StorageManager {
          config.dataPath.startsWith("s3://");
 
       let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
-      const attachOpts: string[] = [`DATA_PATH '${escapedDataPath}'`];
+      const attachOpts: string[] = [
+         `DATA_PATH '${escapedDataPath}'`,
+         // The manifest table is small relational metadata (one row per build).
+         // Set a high inlining limit so writes always land transactionally in
+         // the postgres catalog rather than as parquet files in object storage,
+         // sidestepping object-storage auth issues entirely for this path.
+         "DATA_INLINING_ROW_LIMIT 100000",
+      ];
       if (isCloudStorage) {
          attachOpts.push("OVERRIDE_DATA_PATH true");
       }
@@ -163,8 +200,6 @@ export class StorageManager {
 
       logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
       await connection.run(attachCmd);
-
-      this.attachedCatalogs.add(catalogName);
    }
 
    getRepository(): ResourceRepository {

package/src/storage/ducklake/DuckLakeManifestStore.ts

@@ -31,23 +31,32 @@ import { DuckDBConnection } from "../duckdb/DuckDBConnection";
  */
 export class DuckLakeManifestStore implements ManifestStore {
    private readonly table: string;
+   private readonly projectName: string;
 
    constructor(
       private db: DuckDBConnection,
       catalogName: string,
+      projectName: string,
    ) {
       this.table = `${catalogName}.build_manifests`;
+      this.projectName = projectName;
    }
 
    /**
     * Idempotently creates the `build_manifests` table and indices in the
     * DuckLake catalog. Safe to call from every worker on startup.
+    *
+    * Note: this table uses `project_name` for the partition column, unlike
+    * the local DuckDB `build_manifests` table (in `storage/duckdb/schema.ts`)
+    * which uses `project_id` and FK-references `projects.id`. The two stores
+    * partition by different identifiers — local random id vs cross-pod-stable
+    * project name — so they intentionally diverge here.
     */
    async bootstrapSchema(): Promise<void> {
       await this.db.run(`
          CREATE TABLE IF NOT EXISTS ${this.table} (
             id VARCHAR,
-            project_id VARCHAR NOT NULL,
+            project_name VARCHAR NOT NULL,
             package_name VARCHAR NOT NULL,
             build_id VARCHAR NOT NULL,
             table_name VARCHAR NOT NULL,
@@ -61,12 +70,12 @@ export class DuckLakeManifestStore implements ManifestStore {
    }
 
    async getManifest(
-      projectId: string,
+      _projectId: string,
       packageName: string,
    ): Promise<BuildManifest> {
       const rows = await this.db.all<Record<string, unknown>>(
-         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
-         [projectId, packageName],
+         `SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`,
+         [this.projectName, packageName],
       );
       const manifest: BuildManifest = { entries: {}, strict: false };
       for (const row of rows) {
@@ -88,7 +97,7 @@ export class DuckLakeManifestStore implements ManifestStore {
     * {@link getManifest} deduplicates by build_id keeping the newest row.
     */
    async writeEntry(
-      projectId: string,
+      _projectId: string,
       packageName: string,
       buildId: string,
       tableName: string,
@@ -99,11 +108,11 @@ export class DuckLakeManifestStore implements ManifestStore {
       const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
 
       await this.db.run(
-         `INSERT INTO ${this.table} (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
+         `INSERT INTO ${this.table} (id, project_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
          [
             id,
-            projectId,
+            this.projectName,
             packageName,
             buildId,
             tableName,
@@ -120,12 +129,12 @@ export class DuckLakeManifestStore implements ManifestStore {
    }
 
    async listEntries(
-      projectId: string,
+      _projectId: string,
       packageName: string,
    ): Promise<ManifestEntry[]> {
       const rows = await this.db.all<Record<string, unknown>>(
-         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
-         [projectId, packageName],
+         `SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`,
+         [this.projectName, packageName],
       );
       return rows.map(this.mapToEntry);
    }
@@ -133,7 +142,7 @@ export class DuckLakeManifestStore implements ManifestStore {
    private mapToEntry(row: Record<string, unknown>): ManifestEntry {
       return {
          id: row.id as string,
-         projectId: row.project_id as string,
+         projectId: row.project_name as string,
          packageName: row.package_name as string,
          buildId: row.build_id as string,
          tableName: row.table_name as string,

package/tests/harness/mcp_test_setup.ts

@@ -5,6 +5,7 @@ import {
    Request,
    Result,
 } from "@modelcontextprotocol/sdk/types.js";
+import { Mutex } from "async-mutex";
 import http from "http";
 import { AddressInfo } from "net";
 import path from "path";
@@ -24,37 +25,24 @@ export interface McpE2ETestEnvironment {
    originalInitializeStorage: string | undefined;
 }
 
-// Counter for unique port assignment per test suite
+// Counter for unique port assignment per test suite (incremented only under e2eSetupMutex)
 let portCounter = 0;
 
-// Mutex to prevent concurrent initialization (since all tests share the same database)
-let initializationLock: Promise<void> | null = null;
+// True mutex: the previous Promise-based lock had a TOCTOU race where two beforeAll hooks
+// could both pass the `if (initializationLock)` check and run setup concurrently, sharing
+// one publisher.db / server singleton and causing flaky listResources and port collisions.
+const e2eSetupMutex = new Mutex();
 
 /**
  * Starts the real application server and connects a real MCP client.
  */
 export async function setupE2ETestEnvironment(): Promise<McpE2ETestEnvironment> {
-   // Wait for any ongoing initialization to complete (prevents concurrent DB initialization)
-   if (initializationLock) {
+   return e2eSetupMutex.runExclusive(async () => {
       console.log(
-         "[E2E Test Setup] Waiting for previous initialization to complete...",
+         "[E2E Test Setup] Acquired setup mutex; starting environment...",
       );
-      await initializationLock;
-   }
-
-   // Create a new lock for this initialization
-   let resolveLock: () => void;
-   initializationLock = new Promise((resolve) => {
-      resolveLock = resolve;
+      return setupE2ETestEnvironmentInternal();
    });
-
-   try {
-      return await setupE2ETestEnvironmentInternal();
-   } finally {
-      // Release the lock
-      resolveLock!();
-      initializationLock = null;
-   }
 }
 
 async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment> {
@@ -129,9 +117,9 @@ async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment>
 
    // --- Wait for server to be ready (packages downloaded) ---
    // Poll the readiness endpoint to ensure initialization completes before tests run
-   // Note: Reduced timeout to 90s to be under the 100s test timeout
+   // Allow >90s: cold CI can spend most of the budget on clone + package load before markReady().
    console.log("[E2E Test Setup] Waiting for server to be ready...");
-   const maxWaitTime = 90000; // 90 seconds max wait (under 100s test timeout)
+   const maxWaitTime = 150000; // 150 seconds (INITIALIZE_STORAGE + large samples can be slow)
    const pollInterval = 1000; // Check every second
    const startTime = Date.now();
    let isReady = false;

package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts

@@ -19,7 +19,7 @@ import {
 } from "../../harness/mcp_test_setup";
 
 // --- Test Suite ---
-describe("MCP Tool Handlers (E2E Integration)", () => {
+describe.serial("MCP Tool Handlers (E2E Integration)", () => {
    let env: McpE2ETestEnvironment | null = null;
    let mcpClient: Client;
 

package/tests/integration/mcp/mcp_resource.integration.spec.ts

@@ -20,7 +20,8 @@ interface PackageContentEntry {
 }
 
 // --- Test Suite ---
-describe("MCP Resource Handlers (E2E Integration)", () => {
+// Serial: one shared MCP client; concurrent it() blocks can interleave StreamableHTTP requests.
+describe.serial("MCP Resource Handlers (E2E Integration)", () => {
    let env: McpE2ETestEnvironment | null = null;
    let mcpClient: Client;
 
@@ -85,17 +86,17 @@ describe("MCP Resource Handlers (E2E Integration)", () => {
             expect(faaPackageEntry).toBeDefined();
             expect(faaPackageEntry?.name).toBe("faa");
 
-            const firstResource = result.resources[0];
-            expect(firstResource).toBeDefined();
-            expect(firstResource.uri).toBeDefined();
-            expect(typeof firstResource.uri).toBe("string");
-            expect(firstResource.uri).toMatch(/^malloy:\/\//);
-            expect(firstResource.name).toBeDefined();
-            expect(typeof firstResource.name).toBe("string");
-            expect(firstResource.description).toBeDefined();
-            expect(typeof firstResource.description).toBe("string");
-            if (firstResource.mimeType) {
-               expect(typeof firstResource.mimeType).toBe("string");
+            // Assert shape on a known row (order of listResources is not guaranteed).
+            const sample = faaPackageEntry!;
+            expect(sample.uri).toBeDefined();
+            expect(typeof sample.uri).toBe("string");
+            expect(sample.uri).toMatch(/^malloy:\/\//);
+            expect(sample.name).toBeDefined();
+            expect(typeof sample.name).toBe("string");
+            expect(sample.description).toBeDefined();
+            expect(typeof sample.description).toBe("string");
+            if (sample.mimeType) {
+               expect(typeof sample.mimeType).toBe("string");
             }
          },
          { timeout: 30000 },

package/tests/integration/mcp/mcp_transport.integration.spec.ts

@@ -18,7 +18,7 @@ import {
 // --- Test Suite ---
 // Note: These tests assume interaction via a standard MCP client.
 // Tests for raw HTTP edge cases (non-JSON, non-RPC) are omitted based on this assumption.
-describe("MCP Transport Tests (E2E Integration)", () => {
+describe.serial("MCP Transport Tests (E2E Integration)", () => {
    let env: McpE2ETestEnvironment | null = null;
    let mcpClient: Client<Request, Notification, Result>; // Convenience variable