@malloy-publisher/server 0.0.196-dev → 0.0.196

package/tests/unit/duckdb/attached_databases.test.ts

@@ -12,7 +12,7 @@ import fs from "fs/promises";
 import os from "os";
 import path from "path";
 import type { components } from "../../../src/api";
-import { createProjectConnections } from "../../../src/service/connection";
+import { createEnvironmentConnections } from "../../../src/service/connection";
 
 type ApiConnection = components["schemas"]["Connection"];
 
@@ -355,7 +355,7 @@ describe("DuckDB Attached Databases", () => {
    });
 });
 
-describe("createProjectConnections - DuckDB", () => {
+describe("createEnvironmentConnections - DuckDB", () => {
    const PROJECT_TEST_DIR = path.join(os.tmpdir(), "duckdb-project-tests");
    let createdConnections: Map<string, unknown> = new Map();
 
@@ -402,7 +402,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          const { malloyConnections, apiConnections } =
-            await createProjectConnections(connections, PROJECT_TEST_DIR);
+            await createEnvironmentConnections(connections, PROJECT_TEST_DIR);
 
          createdConnections = malloyConnections;
 
@@ -433,7 +433,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          const { malloyConnections, apiConnections } =
-            await createProjectConnections(connections, PROJECT_TEST_DIR);
+            await createEnvironmentConnections(connections, PROJECT_TEST_DIR);
 
          createdConnections = malloyConnections;
 
@@ -462,7 +462,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          const { malloyConnections, apiConnections } =
-            await createProjectConnections(connections, PROJECT_TEST_DIR);
+            await createEnvironmentConnections(connections, PROJECT_TEST_DIR);
 
          createdConnections = malloyConnections;
 
@@ -502,7 +502,7 @@ describe("createProjectConnections - DuckDB", () => {
             },
          ];
 
-         const { malloyConnections } = await createProjectConnections(
+         const { malloyConnections } = await createEnvironmentConnections(
             connections,
             PROJECT_TEST_DIR,
          );
@@ -533,7 +533,7 @@ describe("createProjectConnections - DuckDB", () => {
             },
          ];
 
-         const { malloyConnections } = await createProjectConnections(
+         const { malloyConnections } = await createEnvironmentConnections(
             connections,
             PROJECT_TEST_DIR,
          );
@@ -591,7 +591,7 @@ describe("createProjectConnections - DuckDB", () => {
             },
          ];
 
-         const { malloyConnections } = await createProjectConnections(
+         const { malloyConnections } = await createEnvironmentConnections(
             connections,
             PROJECT_TEST_DIR,
          );
@@ -624,7 +624,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow();
       });
 
@@ -637,7 +637,7 @@ describe("createProjectConnections - DuckDB", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("DuckDB connection configuration is missing");
       });
 
@@ -658,7 +658,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("Unsupported database type");
       });
 
@@ -682,7 +682,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("service account key required");
       });
 
@@ -704,7 +704,7 @@ describe("createProjectConnections - DuckDB", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("keyId and secret are required");
       });
 
@@ -726,7 +726,7 @@ describe("createProjectConnections - DuckDB", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("accessKeyId and secretAccessKey are required");
       });
 
@@ -748,7 +748,7 @@ describe("createProjectConnections - DuckDB", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("PostgreSQL connection configuration is required");
       });
 
@@ -761,7 +761,7 @@ describe("createProjectConnections - DuckDB", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("DuckLake connection configuration is missing");
       });
 
@@ -785,7 +785,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("username is required");
       });
 
@@ -806,7 +806,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("PostgreSQL connection configuration missing");
       });
 
@@ -836,7 +836,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow(
             "DuckDB attached database names cannot conflict with connection name",
          );
@@ -854,7 +854,7 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("DuckDB connection name cannot be 'duckdb'");
       });
 
@@ -884,10 +884,28 @@ describe("createProjectConnections - DuckDB", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("DuckDB connection name cannot be 'duckdb'");
       });
 
+      it("should throw when DuckDB connection has no attached databases", async () => {
+         const connections: ApiConnection[] = [
+            {
+               name: "no_attached_db",
+               type: "duckdb",
+               duckdbConnection: {
+                  attachedDatabases: [],
+               },
+            },
+         ];
+
+         await expect(
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
+         ).rejects.toThrow(
+            "DuckDB connection must have at least one attached database",
+         );
+      });
+
       it("should throw on unsupported connection type", async () => {
          const connections = [
             {
@@ -897,7 +915,7 @@ describe("createProjectConnections - DuckDB", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("Unsupported connection type");
       });
 
@@ -921,13 +939,13 @@ describe("createProjectConnections - DuckDB", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow();
       });
    });
 });
 
-describe("createProjectConnections - Other Connection Types", () => {
+describe("createEnvironmentConnections - Other Connection Types", () => {
    const PROJECT_TEST_DIR = path.join(
       os.tmpdir(),
       "connection-validation-tests",
@@ -970,7 +988,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("Snowflake connection configuration is missing");
       });
 
@@ -988,7 +1006,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("Snowflake account is required");
       });
 
@@ -1006,7 +1024,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("Snowflake username is required");
       });
 
@@ -1024,7 +1042,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow(
             "Snowflake password or private key or private key path is required",
          );
@@ -1044,7 +1062,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("Snowflake warehouse is required");
       });
    });
@@ -1059,7 +1077,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("Trino connection configuration is missing");
       });
 
@@ -1076,7 +1094,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow(
             'Invalid Trino connection: expected "http://server:port" or "https://server:port"',
          );
@@ -1093,7 +1111,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ] as ApiConnection[];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("MotherDuck connection configuration is missing");
       });
 
@@ -1107,7 +1125,7 @@ describe("createProjectConnections - Other Connection Types", () => {
          ];
 
          await expect(
-            createProjectConnections(connections, PROJECT_TEST_DIR),
+            createEnvironmentConnections(connections, PROJECT_TEST_DIR),
          ).rejects.toThrow("MotherDuck access token is required");
       });
    });

package/tests/unit/duckdb/legacy_schema_migration.test.ts

@@ -0,0 +1,194 @@
+/// <reference types="bun-types" />
+
+// TODO: Remove this during projects cleanup
+
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { DuckDBConnection } from "../../../src/storage/duckdb/DuckDBConnection";
+import { initializeSchema } from "../../../src/storage/duckdb/schema";
+
+const TEST_DB_DIR = path.join(os.tmpdir(), "duckdb-legacy-migration-tests");
+
+// Seed a pre-rename schema on an *already-open* connection. We deliberately
+// avoid opening, closing, and reopening the same DuckDB file within a test:
+// on Windows runners the second `duckdb.Database(path)` call sometimes fails
+// with `Invalid Error` because the OS hasn't released the file handle yet.
+// Sharing one connection between seed and assertion sidesteps that entirely.
+async function seedLegacySchema(db: DuckDBConnection): Promise<void> {
+   // Seed a pre-rename schema: parent table named `projects` and child
+   // tables with `project_id` foreign-key columns. Mirrors what an existing
+   // installation looked like before the projects→environments rename.
+   await db.run(`
+      CREATE TABLE projects (
+         id VARCHAR PRIMARY KEY,
+         name VARCHAR NOT NULL UNIQUE,
+         path VARCHAR NOT NULL,
+         description VARCHAR,
+         metadata JSON,
+         created_at TIMESTAMP NOT NULL,
+         updated_at TIMESTAMP NOT NULL
+      )
+   `);
+   await db.run(`
+      CREATE TABLE packages (
+         id VARCHAR PRIMARY KEY,
+         project_id VARCHAR NOT NULL,
+         name VARCHAR NOT NULL,
+         description VARCHAR,
+         manifest_path VARCHAR NOT NULL,
+         metadata JSON,
+         created_at TIMESTAMP NOT NULL,
+         updated_at TIMESTAMP NOT NULL,
+         FOREIGN KEY (project_id) REFERENCES projects(id)
+      )
+   `);
+   await db.run(`
+      CREATE TABLE connections (
+         id VARCHAR PRIMARY KEY,
+         project_id VARCHAR NOT NULL,
+         name VARCHAR NOT NULL,
+         type VARCHAR NOT NULL,
+         config JSON NOT NULL,
+         created_at TIMESTAMP NOT NULL,
+         updated_at TIMESTAMP NOT NULL,
+         FOREIGN KEY (project_id) REFERENCES projects(id)
+      )
+   `);
+   await db.run(`
+      CREATE TABLE materializations (
+         id VARCHAR PRIMARY KEY,
+         project_id VARCHAR NOT NULL,
+         package_name VARCHAR NOT NULL,
+         status VARCHAR NOT NULL,
+         active_key VARCHAR,
+         started_at TIMESTAMP,
+         completed_at TIMESTAMP,
+         error TEXT,
+         metadata JSON,
+         created_at TIMESTAMP NOT NULL,
+         updated_at TIMESTAMP NOT NULL,
+         FOREIGN KEY (project_id) REFERENCES projects(id)
+      )
+   `);
+   await db.run(`
+      CREATE TABLE build_manifests (
+         id VARCHAR PRIMARY KEY,
+         project_id VARCHAR NOT NULL,
+         package_name VARCHAR NOT NULL,
+         build_id VARCHAR NOT NULL,
+         table_name VARCHAR NOT NULL,
+         source_name VARCHAR NOT NULL,
+         connection_name VARCHAR NOT NULL,
+         created_at TIMESTAMP NOT NULL,
+         updated_at TIMESTAMP NOT NULL,
+         FOREIGN KEY (project_id) REFERENCES projects(id)
+      )
+   `);
+
+   await db.run(
+      `INSERT INTO projects VALUES ('p1', 'proj-one', '/p1', 'd1', NULL,
+         TIMESTAMP '2024-01-01 00:00:00', TIMESTAMP '2024-01-01 00:00:00')`,
+   );
+   await db.run(
+      `INSERT INTO packages VALUES ('pkg1', 'p1', 'pkg-one', NULL, '/m', NULL,
+         TIMESTAMP '2024-01-01 00:00:00', TIMESTAMP '2024-01-01 00:00:00')`,
+   );
+}
+
+describe("DuckDB legacy projects schema cleanup", () => {
+   beforeEach(async () => {
+      await fs.mkdir(TEST_DB_DIR, { recursive: true });
+   });
+
+   afterEach(async () => {
+      try {
+         await fs.rm(TEST_DB_DIR, { recursive: true, force: true });
+      } catch {
+         // ignore
+      }
+   });
+
+   it("drops legacy projects schema and creates the new environments schema cleanly", async () => {
+      const dbPath = path.join(TEST_DB_DIR, "legacy.duckdb");
+      const db = new DuckDBConnection(dbPath);
+      await db.initialize();
+
+      // Seed the legacy schema on the same connection, then run the
+      // production schema-init path. This mirrors a server upgrade
+      // (legacy data on disk, new code starting up) without forcing a
+      // close+reopen, which is unreliable on Windows runners.
+      await seedLegacySchema(db);
+      await initializeSchema(db);
+
+      // Legacy parent table is gone.
+      const legacyProjects = await db.all<{ name: string }>(
+         "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
+      );
+      expect(legacyProjects.length).toBe(0);
+
+      // New `environments` table exists and is empty (legacy data dropped).
+      const envs = await db.all<{ id: string }>("SELECT id FROM environments");
+      expect(envs.length).toBe(0);
+
+      // Child tables are queryable by `environment_id` (the new column),
+      // proving they were recreated with the new schema rather than left on
+      // the old `project_id` column.
+      const pkgs = await db.all<{ id: string }>(
+         "SELECT id FROM packages WHERE environment_id = ?",
+         ["p1"],
+      );
+      expect(pkgs.length).toBe(0);
+      const conns = await db.all<{ id: string }>(
+         "SELECT id FROM connections WHERE environment_id = ?",
+         ["p1"],
+      );
+      expect(conns.length).toBe(0);
+      const mats = await db.all<{ id: string }>(
+         "SELECT id FROM materializations WHERE environment_id = ?",
+         ["p1"],
+      );
+      expect(mats.length).toBe(0);
+      const manifests = await db.all<{ id: string }>(
+         "SELECT id FROM build_manifests WHERE environment_id = ?",
+         ["p1"],
+      );
+      expect(manifests.length).toBe(0);
+
+      await db.close();
+   });
+
+   it("is idempotent: running initializeSchema twice on a migrated DB is a no-op", async () => {
+      const dbPath = path.join(TEST_DB_DIR, "legacy_idempotent.duckdb");
+      const db = new DuckDBConnection(dbPath);
+      await db.initialize();
+
+      await seedLegacySchema(db);
+      await initializeSchema(db);
+      // Second call should hit the early-return path (isInitialized() === true).
+      await initializeSchema(db);
+
+      const envs = await db.all<{ id: string }>("SELECT id FROM environments");
+      expect(envs.length).toBe(0);
+
+      await db.close();
+   });
+
+   it("creates a fresh schema unchanged when no legacy projects table is present", async () => {
+      const dbPath = path.join(TEST_DB_DIR, "fresh.duckdb");
+      const db = new DuckDBConnection(dbPath);
+      await db.initialize();
+      await initializeSchema(db);
+
+      const envs = await db.all<{ id: string }>("SELECT id FROM environments");
+      expect(envs.length).toBe(0);
+
+      const legacy = await db.all<{ name: string }>(
+         "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
+      );
+      expect(legacy.length).toBe(0);
+
+      await db.close();
+   });
+});

package/tests/unit/ducklake/ducklake.test.ts

@@ -4,7 +4,7 @@ import fs from "fs/promises";
 import path from "path";
 import { components } from "../../../src/api";
 import {
-   createProjectConnections,
+   createEnvironmentConnections,
    deleteDuckLakeConnectionFile,
    testConnectionConfig,
 } from "../../../src/service/connection";
@@ -143,7 +143,7 @@ describe("DuckLake Connection Tests", () => {
             };
 
             const { malloyConnections, apiConnections } =
-               await createProjectConnections(
+               await createEnvironmentConnections(
                   [ducklakeConnection],
                   testProjectPath,
                );
@@ -212,7 +212,7 @@ describe("DuckLake Connection Tests", () => {
                },
             };
 
-            const { malloyConnections } = await createProjectConnections(
+            const { malloyConnections } = await createEnvironmentConnections(
                [ducklakeConnection],
                testProjectPath,
             );
@@ -268,7 +268,7 @@ describe("DuckLake Connection Tests", () => {
                },
             };
 
-            const { malloyConnections } = await createProjectConnections(
+            const { malloyConnections } = await createEnvironmentConnections(
                [ducklakeConnection],
                testProjectPath,
             );
@@ -333,7 +333,7 @@ describe("DuckLake Connection Tests", () => {
                },
             };
 
-            const { malloyConnections } = await createProjectConnections(
+            const { malloyConnections } = await createEnvironmentConnections(
                [ducklakeConnection],
                testProjectPath,
             );
@@ -400,7 +400,7 @@ describe("DuckLake Connection Tests", () => {
                },
             };
 
-            const { malloyConnections } = await createProjectConnections(
+            const { malloyConnections } = await createEnvironmentConnections(
                [ducklakeConnection],
                testProjectPath,
             );
@@ -470,7 +470,7 @@ describe("DuckLake Connection Tests", () => {
                },
             };
 
-            const { malloyConnections } = await createProjectConnections(
+            const { malloyConnections } = await createEnvironmentConnections(
                [ducklakeConnection],
                testProjectPath,
             );
@@ -529,7 +529,7 @@ describe("DuckLake Connection Tests", () => {
                },
             };
 
-            const { malloyConnections } = await createProjectConnections(
+            const { malloyConnections } = await createEnvironmentConnections(
                [ducklakeConnection],
                testProjectPath,
             );
@@ -553,7 +553,7 @@ describe("DuckLake Connection Tests", () => {
    describe("Error Handling", () => {
       it("should throw error if DuckLake catalog connection is missing", async () => {
          await expect(
-            createProjectConnections(
+            createEnvironmentConnections(
                [
                   {
                      name: "ducklake_no_catalog",
@@ -576,7 +576,7 @@ describe("DuckLake Connection Tests", () => {
 
       it("should throw error if DuckLake storage bucketUrl is missing", async () => {
          await expect(
-            createProjectConnections(
+            createEnvironmentConnections(
                [
                   {
                      name: "ducklake_no_bucket",
@@ -607,7 +607,7 @@ describe("DuckLake Connection Tests", () => {
 
       it("should throw error if DuckLake connection config is missing", async () => {
          await expect(
-            createProjectConnections(
+            createEnvironmentConnections(
                [
                   {
                      name: "ducklake_missing_config",
@@ -653,19 +653,21 @@ describe("DuckLake Connection Tests", () => {
          };
 
          // Create connection twice - second should handle already attached gracefully
-         const { malloyConnections: conn1 } = await createProjectConnections(
-            [ducklakeConnection],
-            testProjectPath,
-         );
+         const { malloyConnections: conn1 } =
+            await createEnvironmentConnections(
+               [ducklakeConnection],
+               testProjectPath,
+            );
          const connection1 = conn1.get(
             "ducklake_duplicate_test",
          ) as DuckDBConnection;
          createdConnections.push(connection1);
 
-         const { malloyConnections: conn2 } = await createProjectConnections(
-            [ducklakeConnection],
-            testProjectPath,
-         );
+         const { malloyConnections: conn2 } =
+            await createEnvironmentConnections(
+               [ducklakeConnection],
+               testProjectPath,
+            );
          const connection2 = conn2.get(
             "ducklake_duplicate_test",
          ) as DuckDBConnection;
@@ -714,7 +716,7 @@ describe("DuckLake Connection Tests", () => {
                },
             };
 
-            const { malloyConnections } = await createProjectConnections(
+            const { malloyConnections } = await createEnvironmentConnections(
                [ducklakeConnection],
                testProjectPath,
             );
@@ -771,7 +773,7 @@ describe("DuckLake Connection Tests", () => {
             },
          };
 
-         const { malloyConnections } = await createProjectConnections(
+         const { malloyConnections } = await createEnvironmentConnections(
             [ducklakeConnection],
             testProjectPath,
          );
@@ -903,7 +905,7 @@ describe("DuckLake Connection Tests", () => {
                return;
             }
 
-            const { apiConnections } = await createProjectConnections(
+            const { apiConnections } = await createEnvironmentConnections(
                [
                   {
                      name: "ducklake_attrs_test",