@malloy-publisher/server 0.0.195 → 0.0.196

package/src/storage/duckdb/PackageRepository.ts

@@ -12,10 +12,10 @@ export class PackageRepository {
       return new Date();
    }
 
-   async listPackages(projectId: string): Promise<Package[]> {
+   async listPackages(environmentId: string): Promise<Package[]> {
       const rows = await this.db.all<Record<string, unknown>>(
-         "SELECT * FROM packages WHERE project_id = ? ORDER BY name",
-         [projectId],
+         "SELECT * FROM packages WHERE environment_id = ? ORDER BY name",
+         [environmentId],
       );
       return rows.map(this.mapToPackage);
    }
@@ -29,12 +29,12 @@ export class PackageRepository {
    }
 
    async getPackageByName(
-      projectId: string,
+      environmentId: string,
       name: string,
    ): Promise<Package | null> {
       const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM packages WHERE project_id = ? AND name = ?",
-         [projectId, name],
+         "SELECT * FROM packages WHERE environment_id = ? AND name = ?",
+         [environmentId, name],
       );
       return row ? this.mapToPackage(row) : null;
    }
@@ -46,11 +46,11 @@ export class PackageRepository {
       const now = this.now();
 
       await this.db.run(
-         `INSERT INTO packages (id, project_id, name, description, manifest_path, metadata, created_at, updated_at)
+         `INSERT INTO packages (id, environment_id, name, description, manifest_path, metadata, created_at, updated_at)
       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
          [
             id,
-            pkg.projectId,
+            pkg.environmentId,
             pkg.name,
             pkg.description || null,
             pkg.manifestPath,
@@ -114,14 +114,14 @@ export class PackageRepository {
       await this.db.run("DELETE FROM packages WHERE id = ?", [id]);
    }
 
-   async deletePackagesByProjectId(id: string): Promise<void> {
-      await this.db.run("DELETE FROM packages WHERE project_id = ?", [id]);
+   async deletePackagesByEnvironmentId(id: string): Promise<void> {
+      await this.db.run("DELETE FROM packages WHERE environment_id = ?", [id]);
    }
 
    private mapToPackage(row: Record<string, unknown>): Package {
       return {
          id: row.id as string,
-         projectId: row.project_id as string,
+         environmentId: row.environment_id as string,
          name: row.name as string,
          description: row.description as string | undefined,
          manifestPath: row.manifest_path as string,

package/src/storage/duckdb/manifest_store.spec.ts

@@ -6,7 +6,7 @@ import { DuckDBManifestStore } from "./DuckDBManifestStore";
 function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
    return {
       id: "entry-1",
-      projectId: "proj-1",
+      environmentId: "proj-1",
       packageName: "pkg",
       buildId: "build-abc",
       tableName: "my_table",
@@ -83,7 +83,7 @@ describe("DuckDBManifestStore", () => {
          expect(ctx.repository.upsertManifestEntry.calledOnce).toBe(true);
          const arg = ctx.repository.upsertManifestEntry.firstCall.args[0];
          expect(arg).toEqual({
-            projectId: "proj-1",
+            environmentId: "proj-1",
             packageName: "pkg",
             buildId: "build-abc",
             tableName: "tbl",

package/src/storage/duckdb/schema.ts

@@ -17,12 +17,21 @@ export async function initializeSchema(
       );
       await dropAllTables(db);
    } else {
+      // TODO: Remove this during projects cleanup
+      // If a pre-rename `projects` schema is on disk, the new
+      // CREATE TABLE IF NOT EXISTS pass below would silently leave child
+      // tables on the old `project_id` column and the first query against
+      // `environment_id` would crash. Drop the legacy tables (with a loud
+      // warning) so the fresh schema can be created cleanly. This is
+      // destructive — operators upgrading should re-create their environments
+      // and packages via the API after the upgrade.
+      await dropLegacyProjectSchema(db);
       logger.info("Creating database schema for the first time...");
    }
 
-   // Projects table
+   // Environments table
    await db.run(`
-    CREATE TABLE IF NOT EXISTS projects (
+    CREATE TABLE IF NOT EXISTS environments (
       id VARCHAR PRIMARY KEY,
       name VARCHAR NOT NULL UNIQUE,
       path VARCHAR NOT NULL,
@@ -37,15 +46,15 @@ export async function initializeSchema(
    await db.run(`
     CREATE TABLE IF NOT EXISTS packages (
       id VARCHAR PRIMARY KEY,
-      project_id VARCHAR NOT NULL,
+      environment_id VARCHAR NOT NULL,
       name VARCHAR NOT NULL,
       description VARCHAR,
       manifest_path VARCHAR NOT NULL,
       metadata JSON,
       created_at TIMESTAMP NOT NULL,
       updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (project_id) REFERENCES projects(id),
-      UNIQUE (project_id, name)
+      FOREIGN KEY (environment_id) REFERENCES environments(id),
+      UNIQUE (environment_id, name)
     )
   `);
 
@@ -53,22 +62,22 @@ export async function initializeSchema(
    await db.run(`
     CREATE TABLE IF NOT EXISTS connections (
       id VARCHAR PRIMARY KEY,
-      project_id VARCHAR NOT NULL,
+      environment_id VARCHAR NOT NULL,
       name VARCHAR NOT NULL,
       type VARCHAR NOT NULL,
       config JSON NOT NULL,
       created_at TIMESTAMP NOT NULL,
       updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (project_id) REFERENCES projects(id),
-      UNIQUE (project_id, name)
+      FOREIGN KEY (environment_id) REFERENCES environments(id),
+      UNIQUE (environment_id, name)
     )
   `);
 
    // Materializations table.
    //
    // `active_key` enforces at-most-one active (PENDING or RUNNING)
-   // materialization per (project, package) at the DB layer. It is set to
-   // `{project_id}|{package_name}` while the row is active and cleared
+   // materialization per (environment, package) at the DB layer. It is set to
+   // `{environment_id}|{package_name}` while the row is active and cleared
    // to NULL on transition to any terminal state. A unique index on
    // `active_key` (see below) makes the insert-then-check race impossible —
    // a second concurrent create fails with a constraint violation, which the
@@ -76,7 +85,7 @@ export async function initializeSchema(
    await db.run(`
     CREATE TABLE IF NOT EXISTS materializations (
       id VARCHAR PRIMARY KEY,
-      project_id VARCHAR NOT NULL,
+      environment_id VARCHAR NOT NULL,
       package_name VARCHAR NOT NULL,
       status VARCHAR NOT NULL,
       active_key VARCHAR,
@@ -86,7 +95,7 @@ export async function initializeSchema(
       metadata JSON,
       created_at TIMESTAMP NOT NULL,
       updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (project_id) REFERENCES projects(id)
+      FOREIGN KEY (environment_id) REFERENCES environments(id)
     )
   `);
 
@@ -94,7 +103,7 @@ export async function initializeSchema(
    await db.run(`
     CREATE TABLE IF NOT EXISTS build_manifests (
       id VARCHAR PRIMARY KEY,
-      project_id VARCHAR NOT NULL,
+      environment_id VARCHAR NOT NULL,
       package_name VARCHAR NOT NULL,
       build_id VARCHAR NOT NULL,
       table_name VARCHAR NOT NULL,
@@ -102,36 +111,68 @@ export async function initializeSchema(
       connection_name VARCHAR NOT NULL,
       created_at TIMESTAMP NOT NULL,
       updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (project_id) REFERENCES projects(id),
-      UNIQUE (project_id, package_name, build_id)
+      FOREIGN KEY (environment_id) REFERENCES environments(id),
+      UNIQUE (environment_id, package_name, build_id)
     )
   `);
 
    // Create indexes for better query performance
    await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_packages_project_id ON packages(project_id)",
+      "CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)",
    );
    await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_connections_project_id ON connections(project_id)",
+      "CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)",
    );
    await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_materializations_project_package ON materializations(project_id, package_name)",
+      "CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)",
    );
    await db.run(
       "CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
    );
    await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_build_manifests_project_package ON build_manifests(project_id, package_name)",
+      "CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)",
    );
 }
 
+// TODO: Remove this during projects cleanup
+// Tables in the pre-rename schema, listed children-first so DROP order
+// satisfies foreign-key dependencies on the legacy `projects` table.
+const LEGACY_TABLES_DROP_ORDER = [
+   "build_manifests",
+   "materializations",
+   "packages",
+   "connections",
+   "projects",
+] as const;
+
+async function dropLegacyProjectSchema(db: DuckDBConnection): Promise<void> {
+   const legacy = await db.all<{ name: string }>(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
+   );
+   if (!legacy || legacy.length === 0) {
+      return;
+   }
+
+   logger.warn(
+      "Detected legacy 'projects' schema. Dropping legacy tables; existing environments/packages/connections/materializations data will be lost. Re-create them via the API after upgrade.",
+   );
+
+   for (const table of LEGACY_TABLES_DROP_ORDER) {
+      try {
+         await db.run(`DROP TABLE IF EXISTS ${table}`);
+      } catch (err) {
+         logger.warn(`Failed to drop legacy table ${table}:`, err);
+      }
+   }
+}
+
 async function dropAllTables(db: DuckDBConnection): Promise<void> {
    const tables = [
       "build_manifests",
       "materializations",
       "packages",
       "connections",
-      "projects",
+      "environments",
    ];
 
    logger.info("Dropping tables:", tables.join(", "));

package/src/storage/ducklake/DuckLakeManifestStore.ts

@@ -31,23 +31,32 @@ import { DuckDBConnection } from "../duckdb/DuckDBConnection";
  */
 export class DuckLakeManifestStore implements ManifestStore {
    private readonly table: string;
+   private readonly environmentName: string;
 
    constructor(
       private db: DuckDBConnection,
       catalogName: string,
+      environmentName: string,
    ) {
       this.table = `${catalogName}.build_manifests`;
+      this.environmentName = environmentName;
    }
 
    /**
     * Idempotently creates the `build_manifests` table and indices in the
     * DuckLake catalog. Safe to call from every worker on startup.
+    *
+    * Note: this table uses `project_name` for the partition column, unlike
+    * the local DuckDB `build_manifests` table (in `storage/duckdb/schema.ts`)
+    * which uses `project_id` and FK-references `projects.id`. The two stores
+    * partition by different identifiers — local random id vs cross-pod-stable
+    * project name — so they intentionally diverge here.
     */
    async bootstrapSchema(): Promise<void> {
       await this.db.run(`
          CREATE TABLE IF NOT EXISTS ${this.table} (
             id VARCHAR,
-            project_id VARCHAR NOT NULL,
+            environment_name VARCHAR NOT NULL,
             package_name VARCHAR NOT NULL,
             build_id VARCHAR NOT NULL,
             table_name VARCHAR NOT NULL,
@@ -61,12 +70,12 @@ export class DuckLakeManifestStore implements ManifestStore {
    }
 
    async getManifest(
-      projectId: string,
+      _environmentId: string,
       packageName: string,
    ): Promise<BuildManifest> {
       const rows = await this.db.all<Record<string, unknown>>(
-         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
-         [projectId, packageName],
+         `SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`,
+         [this.environmentName, packageName],
       );
       const manifest: BuildManifest = { entries: {}, strict: false };
       for (const row of rows) {
@@ -88,7 +97,7 @@ export class DuckLakeManifestStore implements ManifestStore {
     * {@link getManifest} deduplicates by build_id keeping the newest row.
     */
    async writeEntry(
-      projectId: string,
+      _environmentId: string,
       packageName: string,
       buildId: string,
       tableName: string,
@@ -99,11 +108,11 @@ export class DuckLakeManifestStore implements ManifestStore {
       const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
 
       await this.db.run(
-         `INSERT INTO ${this.table} (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
+         `INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
          [
             id,
-            projectId,
+            this.environmentName,
             packageName,
             buildId,
             tableName,
@@ -120,12 +129,12 @@ export class DuckLakeManifestStore implements ManifestStore {
    }
 
    async listEntries(
-      projectId: string,
+      _environmentId: string,
       packageName: string,
    ): Promise<ManifestEntry[]> {
       const rows = await this.db.all<Record<string, unknown>>(
-         `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
-         [projectId, packageName],
+         `SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`,
+         [this.environmentName, packageName],
       );
       return rows.map(this.mapToEntry);
    }
@@ -133,7 +142,7 @@ export class DuckLakeManifestStore implements ManifestStore {
    private mapToEntry(row: Record<string, unknown>): ManifestEntry {
       return {
          id: row.id as string,
-         projectId: row.project_id as string,
+         environmentId: row.environment_name as string,
          packageName: row.package_name as string,
          buildId: row.build_id as string,
          tableName: row.table_name as string,

package/tests/fixtures/publisher.config.json

@@ -1,6 +1,6 @@
 {
   "frozenConfig": false,
-  "projects": [
+  "environments": [
     {
       "name": "test-project",
       "packages": [

package/tests/harness/e2e.ts

@@ -28,7 +28,7 @@ let originalServerRoot: string | undefined;
  */
 export async function startE2E(): Promise<E2EEnv & { stop(): Promise<void> }> {
    //--------------------------------------------------------------------------
-   // 1.  Set SERVER_ROOT so ProjectStore loader finds publisher.config.json
+   // 1.  Set SERVER_ROOT so EnvironmentStore loader finds publisher.config.json
    //--------------------------------------------------------------------------
    originalServerRoot = process.env.SERVER_ROOT;
    // Use import.meta.url for cross-platform compatibility (works on Windows)

package/tests/harness/mcp_test_setup.ts

@@ -5,6 +5,7 @@ import {
    Request,
    Result,
 } from "@modelcontextprotocol/sdk/types.js";
+import { Mutex } from "async-mutex";
 import http from "http";
 import { AddressInfo } from "net";
 import path from "path";
@@ -24,42 +25,29 @@ export interface McpE2ETestEnvironment {
    originalInitializeStorage: string | undefined;
 }
 
-// Counter for unique port assignment per test suite
+// Counter for unique port assignment per test suite (incremented only under e2eSetupMutex)
 let portCounter = 0;
 
-// Mutex to prevent concurrent initialization (since all tests share the same database)
-let initializationLock: Promise<void> | null = null;
+// True mutex: the previous Promise-based lock had a TOCTOU race where two beforeAll hooks
+// could both pass the `if (initializationLock)` check and run setup concurrently, sharing
+// one publisher.db / server singleton and causing flaky listResources and port collisions.
+const e2eSetupMutex = new Mutex();
 
 /**
  * Starts the real application server and connects a real MCP client.
  */
 export async function setupE2ETestEnvironment(): Promise<McpE2ETestEnvironment> {
-   // Wait for any ongoing initialization to complete (prevents concurrent DB initialization)
-   if (initializationLock) {
+   return e2eSetupMutex.runExclusive(async () => {
       console.log(
-         "[E2E Test Setup] Waiting for previous initialization to complete...",
+         "[E2E Test Setup] Acquired setup mutex; starting environment...",
       );
-      await initializationLock;
-   }
-
-   // Create a new lock for this initialization
-   let resolveLock: () => void;
-   initializationLock = new Promise((resolve) => {
-      resolveLock = resolve;
+      return setupE2ETestEnvironmentInternal();
    });
-
-   try {
-      return await setupE2ETestEnvironmentInternal();
-   } finally {
-      // Release the lock
-      resolveLock!();
-      initializationLock = null;
-   }
 }
 
 async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment> {
    // --- Store and Set SERVER_ROOT Env Var ---
-   // The ProjectStore relies on SERVER_ROOT to find publisher.config.json.
+   // The EnvironmentStore relies on SERVER_ROOT to find publisher.config.json.
    const originalServerRoot = process.env.SERVER_ROOT; // Store original value
    // Resolve the path to 'packages/server' based on the location of this file
    // Use import.meta.url for cross-platform compatibility (works on Windows)
@@ -129,9 +117,9 @@ async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment>
 
    // --- Wait for server to be ready (packages downloaded) ---
    // Poll the readiness endpoint to ensure initialization completes before tests run
-   // Note: Reduced timeout to 90s to be under the 100s test timeout
+   // Allow >90s: cold CI can spend most of the budget on clone + package load before markReady().
    console.log("[E2E Test Setup] Waiting for server to be ready...");
-   const maxWaitTime = 90000; // 90 seconds max wait (under 100s test timeout)
+   const maxWaitTime = 150000; // 150 seconds (INITIALIZE_STORAGE + large samples can be slow)
    const pollInterval = 1000; // Check every second
    const startTime = Date.now();
    let isReady = false;

package/tests/harness/mocks.ts

@@ -1,12 +1,14 @@
 import sinon from "sinon";
-import { ProjectStore } from "../../src/service/project_store";
+import { EnvironmentStore } from "../../src/service/environment_store";
 
-/** Return a stubbed ProjectStore where every lookup throws or returns minimal objects. */
-export function fakeProjectStore(): sinon.SinonStubbedInstance<ProjectStore> {
-   const ps = sinon.createStubInstance(ProjectStore);
-   // For now just have getProject reject; suites can stub more.
-   ps.getProject.rejects(new Error("fakeProjectStore: getProject not stubbed"));
-   return ps;
+/** Return a stubbed EnvironmentStore where every lookup throws or returns minimal objects. */
+export function fakeEnvironmentStore(): sinon.SinonStubbedInstance<EnvironmentStore> {
+   const es = sinon.createStubInstance(EnvironmentStore);
+   // For now just have getEnvironment reject; suites can stub more.
+   es.getEnvironment.rejects(
+      new Error("fakeEnvironmentStore: getEnvironment not stubbed"),
+   );
+   return es;
 }
 
 // The runtime implementation of `McpServer` lives in the MCP SDK package, which
@@ -27,7 +29,7 @@ export type McpServer = DummyMcpServer;
 /** Convenience helper mimicking the old mocks used in integration specs. */
 export function createMalloyServiceMocks() {
    return {
-      projectStore: fakeProjectStore(),
+      environmentStore: fakeEnvironmentStore(),
    } as const;
 }
 

package/tests/harness/rest_e2e.ts

@@ -12,8 +12,8 @@ export interface RestE2EEnv {
  * reuses the cached Express app and binds on an OS-assigned port
  * to avoid collisions.
  *
- * Callers are responsible for creating any test-specific projects
- * via the REST API (POST /api/v0/projects) and cleaning them up.
+ * Callers are responsible for creating any test-specific environments
+ * via the REST API (POST /api/v0/environments) and cleaning them up.
  */
 export async function startRestE2E(): Promise<
    RestE2EEnv & { stop(): Promise<void> }