@malloy-publisher/server 0.0.234 → 0.0.236

package/tests/unit/duckdb/duckdb_connection.test.ts

@@ -1,181 +0,0 @@
-/// <reference types="bun-types" />
-
-// Direct unit tests for the storage-layer DuckDBConnection -- the embedded
-// metadata DAO (publisher.db) that was migrated from the legacy `duckdb`
-// callback binding to `@duckdb/node-api` (the "neo" promise API). The
-// repositories depend on this class's public surface, so these assert it
-// behaves identically across the migration: lifecycle, parameterized CRUD with
-// `?` placeholders, row-shape parity, and error wrapping.
-
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import fs from "fs/promises";
-import os from "os";
-import path from "path";
-import { DuckDBConnection } from "../../../src/storage/duckdb/DuckDBConnection";
-
-describe("DuckDBConnection (storage DAO over @duckdb/node-api)", () => {
-   let dbDir: string;
-   let conn: DuckDBConnection;
-
-   beforeEach(async () => {
-      // Unique dir per test so file handles never collide across tests
-      // (Windows runners are sensitive to reopening the same DuckDB file).
-      dbDir = await fs.mkdtemp(path.join(os.tmpdir(), "duckdb-conn-"));
-      conn = new DuckDBConnection(path.join(dbDir, "test.db"));
-      await conn.initialize();
-   });
-
-   afterEach(async () => {
-      try {
-         await conn.close();
-      } catch {
-         // already closed / never opened
-      }
-      await fs.rm(dbDir, { recursive: true, force: true });
-   });
-
-   describe("lifecycle", () => {
-      it("initializes and reports a usable connection", async () => {
-         const rows = await conn.all<{ answer: number }>("SELECT 42 AS answer");
-         expect(rows[0].answer).toBe(42);
-      });
-
-      it("isInitialized is false before the schema exists", async () => {
-         // Fresh DB with no `environments` table yet.
-         expect(await conn.isInitialized()).toBe(false);
-      });
-
-      it("isInitialized becomes true once the environments table exists", async () => {
-         await conn.run("CREATE TABLE environments (id VARCHAR PRIMARY KEY)");
-         expect(await conn.isInitialized()).toBe(true);
-      });
-
-      it("close() is idempotent and a second close does not throw", async () => {
-         await conn.close();
-         await expect(conn.close()).resolves.toBeUndefined();
-      });
-   });
-
-   describe("run / all / get with ? placeholders", () => {
-      beforeEach(async () => {
-         await conn.run(`
-            CREATE TABLE t (
-               id VARCHAR PRIMARY KEY,
-               name VARCHAR,
-               flag BOOLEAN,
-               metadata JSON,
-               created_at TIMESTAMP
-            )
-         `);
-      });
-
-      it("binds positional ? parameters on INSERT and reads them back", async () => {
-         const now = new Date().toISOString();
-         await conn.run(
-            "INSERT INTO t (id, name, flag, metadata, created_at) VALUES (?, ?, ?, ?, ?)",
-            ["a", "alpha", true, JSON.stringify({ k: 1 }), now],
-         );
-
-         const row = await conn.get<{
-            id: string;
-            name: string;
-            flag: boolean;
-            metadata: string;
-            created_at: unknown;
-         }>("SELECT * FROM t WHERE id = ?", ["a"]);
-
-         expect(row).not.toBeNull();
-         expect(row!.id).toBe("a");
-         expect(row!.name).toBe("alpha");
-         // BOOLEAN round-trips as a JS boolean.
-         expect(row!.flag).toBe(true);
-         // JSON is stored/returned as a string the repositories JSON.parse().
-         expect(typeof row!.metadata).toBe("string");
-         expect(JSON.parse(row!.metadata)).toEqual({ k: 1 });
-         // TIMESTAMP is consumable by `new Date(...)` (repositories do exactly this).
-         expect(new Date(row!.created_at as string).toISOString()).toBe(now);
-      });
-
-      it("all() returns every matching row", async () => {
-         await conn.run("INSERT INTO t (id, name) VALUES (?, ?)", ["a", "x"]);
-         await conn.run("INSERT INTO t (id, name) VALUES (?, ?)", ["b", "y"]);
-
-         const rows = await conn.all<{ id: string }>(
-            "SELECT id FROM t ORDER BY id",
-         );
-         expect(rows.map((r) => r.id)).toEqual(["a", "b"]);
-      });
-
-      it("get() returns null when no row matches", async () => {
-         const row = await conn.get("SELECT * FROM t WHERE id = ?", [
-            "missing",
-         ]);
-         expect(row).toBeNull();
-      });
-
-      it("all() returns an empty array when no row matches", async () => {
-         const rows = await conn.all("SELECT * FROM t WHERE id = ?", ["nope"]);
-         expect(rows).toEqual([]);
-      });
-
-      it("supports statements with no parameters", async () => {
-         await conn.run("INSERT INTO t (id) VALUES ('solo')");
-         const rows = await conn.all<{ n: number | bigint }>(
-            "SELECT count(*) AS n FROM t",
-         );
-         expect(Number(rows[0].n)).toBe(1);
-      });
-
-      it("UPDATE and DELETE bind ? parameters", async () => {
-         await conn.run("INSERT INTO t (id, name) VALUES (?, ?)", ["a", "old"]);
-         await conn.run("UPDATE t SET name = ? WHERE id = ?", ["new", "a"]);
-         expect(
-            (await conn.get<{ name: string }>(
-               "SELECT name FROM t WHERE id = ?",
-               ["a"],
-            ))!.name,
-         ).toBe("new");
-
-         await conn.run("DELETE FROM t WHERE id = ?", ["a"]);
-         expect(
-            await conn.get("SELECT * FROM t WHERE id = ?", ["a"]),
-         ).toBeNull();
-      });
-   });
-
-   describe("error handling", () => {
-      it("run() wraps a failing query with the query text", async () => {
-         await expect(
-            conn.run("INSERT INTO does_not_exist VALUES (1)"),
-         ).rejects.toThrow(/Query execution failed[\s\S]*does_not_exist/);
-      });
-
-      it("all() wraps a failing query with the query text", async () => {
-         await expect(conn.all("SELECT * FROM does_not_exist")).rejects.toThrow(
-            /Query execution failed[\s\S]*does_not_exist/,
-         );
-      });
-
-      it("run() throws when the connection is not initialized", async () => {
-         const fresh = new DuckDBConnection(path.join(dbDir, "uninit.db"));
-         await expect(fresh.run("SELECT 1")).rejects.toThrow(/not initialized/);
-      });
-   });
-
-   describe("durability", () => {
-      it("persists data to the file across reopen", async () => {
-         const dbPath = path.join(dbDir, "persist.db");
-         const first = new DuckDBConnection(dbPath);
-         await first.initialize();
-         await first.run("CREATE TABLE k (v VARCHAR)");
-         await first.run("INSERT INTO k VALUES (?)", ["kept"]);
-         await first.close();
-
-         const second = new DuckDBConnection(dbPath);
-         await second.initialize();
-         const rows = await second.all<{ v: string }>("SELECT v FROM k");
-         await second.close();
-         expect(rows.map((r) => r.v)).toEqual(["kept"]);
-      });
-   });
-});

package/tests/unit/duckdb/legacy_schema_migration.test.ts

@@ -1,197 +0,0 @@
-/// <reference types="bun-types" />
-
-// TODO: Remove this during projects cleanup
-
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import fs from "fs/promises";
-import os from "os";
-import path from "path";
-import { DuckDBConnection } from "../../../src/storage/duckdb/DuckDBConnection";
-import { initializeSchema } from "../../../src/storage/duckdb/schema";
-
-const TEST_DB_DIR = path.join(os.tmpdir(), "duckdb-legacy-migration-tests");
-
-// Seed a pre-rename schema on an *already-open* connection. We deliberately
-// avoid opening, closing, and reopening the same DuckDB file within a test:
-// on Windows runners the second `duckdb.Database(path)` call sometimes fails
-// with `Invalid Error` because the OS hasn't released the file handle yet.
-// Sharing one connection between seed and assertion sidesteps that entirely.
-async function seedLegacySchema(db: DuckDBConnection): Promise<void> {
-   // Seed a pre-rename schema: parent table named `projects` and child
-   // tables with `project_id` foreign-key columns. Mirrors what an existing
-   // installation looked like before the projects→environments rename.
-   await db.run(`
-      CREATE TABLE projects (
-         id VARCHAR PRIMARY KEY,
-         name VARCHAR NOT NULL UNIQUE,
-         path VARCHAR NOT NULL,
-         description VARCHAR,
-         metadata JSON,
-         created_at TIMESTAMP NOT NULL,
-         updated_at TIMESTAMP NOT NULL
-      )
-   `);
-   await db.run(`
-      CREATE TABLE packages (
-         id VARCHAR PRIMARY KEY,
-         project_id VARCHAR NOT NULL,
-         name VARCHAR NOT NULL,
-         description VARCHAR,
-         manifest_path VARCHAR NOT NULL,
-         metadata JSON,
-         created_at TIMESTAMP NOT NULL,
-         updated_at TIMESTAMP NOT NULL,
-         FOREIGN KEY (project_id) REFERENCES projects(id)
-      )
-   `);
-   await db.run(`
-      CREATE TABLE connections (
-         id VARCHAR PRIMARY KEY,
-         project_id VARCHAR NOT NULL,
-         name VARCHAR NOT NULL,
-         type VARCHAR NOT NULL,
-         config JSON NOT NULL,
-         created_at TIMESTAMP NOT NULL,
-         updated_at TIMESTAMP NOT NULL,
-         FOREIGN KEY (project_id) REFERENCES projects(id)
-      )
-   `);
-   await db.run(`
-      CREATE TABLE materializations (
-         id VARCHAR PRIMARY KEY,
-         project_id VARCHAR NOT NULL,
-         package_name VARCHAR NOT NULL,
-         status VARCHAR NOT NULL,
-         active_key VARCHAR,
-         started_at TIMESTAMP,
-         completed_at TIMESTAMP,
-         error TEXT,
-         metadata JSON,
-         created_at TIMESTAMP NOT NULL,
-         updated_at TIMESTAMP NOT NULL,
-         FOREIGN KEY (project_id) REFERENCES projects(id)
-      )
-   `);
-   await db.run(`
-      CREATE TABLE build_manifests (
-         id VARCHAR PRIMARY KEY,
-         project_id VARCHAR NOT NULL,
-         package_name VARCHAR NOT NULL,
-         build_id VARCHAR NOT NULL,
-         table_name VARCHAR NOT NULL,
-         source_name VARCHAR NOT NULL,
-         connection_name VARCHAR NOT NULL,
-         created_at TIMESTAMP NOT NULL,
-         updated_at TIMESTAMP NOT NULL,
-         FOREIGN KEY (project_id) REFERENCES projects(id)
-      )
-   `);
-
-   await db.run(
-      `INSERT INTO projects VALUES ('p1', 'proj-one', '/p1', 'd1', NULL,
-         TIMESTAMP '2024-01-01 00:00:00', TIMESTAMP '2024-01-01 00:00:00')`,
-   );
-   await db.run(
-      `INSERT INTO packages VALUES ('pkg1', 'p1', 'pkg-one', NULL, '/m', NULL,
-         TIMESTAMP '2024-01-01 00:00:00', TIMESTAMP '2024-01-01 00:00:00')`,
-   );
-}
-
-describe("DuckDB legacy projects schema cleanup", () => {
-   beforeEach(async () => {
-      await fs.mkdir(TEST_DB_DIR, { recursive: true });
-   });
-
-   afterEach(async () => {
-      try {
-         await fs.rm(TEST_DB_DIR, { recursive: true, force: true });
-      } catch {
-         // ignore
-      }
-   });
-
-   it("drops legacy projects schema and creates the new environments schema cleanly", async () => {
-      const dbPath = path.join(TEST_DB_DIR, "legacy.duckdb");
-      const db = new DuckDBConnection(dbPath);
-      await db.initialize();
-
-      // Seed the legacy schema on the same connection, then run the
-      // production schema-init path. This mirrors a server upgrade
-      // (legacy data on disk, new code starting up) without forcing a
-      // close+reopen, which is unreliable on Windows runners.
-      await seedLegacySchema(db);
-      await initializeSchema(db);
-
-      // Legacy parent table is gone.
-      const legacyProjects = await db.all<{ name: string }>(
-         "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
-      );
-      expect(legacyProjects.length).toBe(0);
-
-      // New `environments` table exists and is empty (legacy data dropped).
-      const envs = await db.all<{ id: string }>("SELECT id FROM environments");
-      expect(envs.length).toBe(0);
-
-      // Child tables are queryable by `environment_id` (the new column),
-      // proving they were recreated with the new schema rather than left on
-      // the old `project_id` column.
-      const pkgs = await db.all<{ id: string }>(
-         "SELECT id FROM packages WHERE environment_id = ?",
-         ["p1"],
-      );
-      expect(pkgs.length).toBe(0);
-      const conns = await db.all<{ id: string }>(
-         "SELECT id FROM connections WHERE environment_id = ?",
-         ["p1"],
-      );
-      expect(conns.length).toBe(0);
-      const mats = await db.all<{ id: string }>(
-         "SELECT id FROM materializations WHERE environment_id = ?",
-         ["p1"],
-      );
-      expect(mats.length).toBe(0);
-
-      // The legacy `build_manifests` table is dropped and not recreated: the
-      // two-round refactor folds the manifest into a JSON column on the
-      // `materializations` row, so no standalone manifest table remains.
-      const manifestTable = await db.all<{ name: string }>(
-         "SELECT name FROM sqlite_master WHERE type='table' AND name='build_manifests'",
-      );
-      expect(manifestTable.length).toBe(0);
-
-      await db.close();
-   });
-
-   it("is idempotent: running initializeSchema twice on a migrated DB is a no-op", async () => {
-      const dbPath = path.join(TEST_DB_DIR, "legacy_idempotent.duckdb");
-      const db = new DuckDBConnection(dbPath);
-      await db.initialize();
-
-      await seedLegacySchema(db);
-      await initializeSchema(db);
-      // Second call should hit the early-return path (isInitialized() === true).
-      await initializeSchema(db);
-
-      const envs = await db.all<{ id: string }>("SELECT id FROM environments");
-      expect(envs.length).toBe(0);
-
-      await db.close();
-   });
-
-   it("creates a fresh schema unchanged when no legacy projects table is present", async () => {
-      const dbPath = path.join(TEST_DB_DIR, "fresh.duckdb");
-      const db = new DuckDBConnection(dbPath);
-      await db.initialize();
-      await initializeSchema(db);
-
-      const envs = await db.all<{ id: string }>("SELECT id FROM environments");
-      expect(envs.length).toBe(0);
-
-      const legacy = await db.all<{ name: string }>(
-         "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
-      );
-      expect(legacy.length).toBe(0);
-
-      await db.close();
-   });
-});

package/tests/unit/duckdb/repositories.test.ts

@@ -1,208 +0,0 @@
-/// <reference types="bun-types" />
-
-// Round-trip tests for the DuckDB storage repositories against a REAL
-// DuckDBConnection (no mocks). The repositories were untested at the storage
-// layer -- existing service specs mock the repository methods -- so these run
-// their actual SQL through the connection migrated to @duckdb/node-api. They
-// guard the migration-sensitive behaviors: parameterized writes, JSON-column
-// round-trips (config/metadata via JSON.stringify -> JSON.parse), TIMESTAMP ->
-// Date mapping, and foreign-key relationships between the tables.
-
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import fs from "fs/promises";
-import os from "os";
-import path from "path";
-import { DuckDBConnection } from "../../../src/storage/duckdb/DuckDBConnection";
-import { initializeSchema } from "../../../src/storage/duckdb/schema";
-import { EnvironmentRepository } from "../../../src/storage/duckdb/EnvironmentRepository";
-import { PackageRepository } from "../../../src/storage/duckdb/PackageRepository";
-import { ConnectionRepository } from "../../../src/storage/duckdb/ConnectionRepository";
-
-describe("DuckDB storage repositories (real connection)", () => {
-   let dbDir: string;
-   let db: DuckDBConnection;
-   let environments: EnvironmentRepository;
-   let packages: PackageRepository;
-   let connections: ConnectionRepository;
-
-   beforeEach(async () => {
-      dbDir = await fs.mkdtemp(path.join(os.tmpdir(), "duckdb-repos-"));
-      db = new DuckDBConnection(path.join(dbDir, "test.db"));
-      await db.initialize();
-      await initializeSchema(db);
-      environments = new EnvironmentRepository(db);
-      packages = new PackageRepository(db);
-      connections = new ConnectionRepository(db);
-   });
-
-   afterEach(async () => {
-      try {
-         await db.close();
-      } catch {
-         // ignore
-      }
-      await fs.rm(dbDir, { recursive: true, force: true });
-   });
-
-   describe("EnvironmentRepository", () => {
-      it("creates and reads back an environment with metadata", async () => {
-         const created = await environments.createEnvironment({
-            name: "env-a",
-            path: "/tmp/env-a",
-            description: "first",
-            metadata: { team: "data", tier: 2 },
-         });
-
-         expect(created.id).toBeTruthy();
-         expect(created.createdAt).toBeInstanceOf(Date);
-
-         const byId = await environments.getEnvironmentById(created.id);
-         expect(byId).not.toBeNull();
-         expect(byId!.name).toBe("env-a");
-         expect(byId!.description).toBe("first");
-         // metadata survives the JSON round-trip (stringify on write, parse on read)
-         expect(byId!.metadata).toEqual({ team: "data", tier: 2 });
-         // TIMESTAMP columns come back as Date
-         expect(byId!.createdAt).toBeInstanceOf(Date);
-         expect(byId!.updatedAt).toBeInstanceOf(Date);
-      });
-
-      it("looks up an environment by name", async () => {
-         await environments.createEnvironment({
-            name: "env-b",
-            path: "/tmp/b",
-         });
-         const byName = await environments.getEnvironmentByName("env-b");
-         expect(byName?.name).toBe("env-b");
-         expect(await environments.getEnvironmentByName("nope")).toBeNull();
-      });
-
-      it("lists, updates, and deletes environments", async () => {
-         const e = await environments.createEnvironment({
-            name: "env-c",
-            path: "/tmp/c",
-         });
-
-         expect(
-            (await environments.listEnvironments()).map((x) => x.name),
-         ).toContain("env-c");
-
-         const updated = await environments.updateEnvironment(e.id, {
-            description: "updated",
-            metadata: { changed: true },
-         });
-         expect(updated.description).toBe("updated");
-         expect(updated.metadata).toEqual({ changed: true });
-
-         await environments.deleteEnvironment(e.id);
-         expect(await environments.getEnvironmentById(e.id)).toBeNull();
-      });
-   });
-
-   describe("PackageRepository", () => {
-      it("creates a package under an environment and reads it back", async () => {
-         const env = await environments.createEnvironment({
-            name: "env-pkg",
-            path: "/tmp/env-pkg",
-         });
-
-         const pkg = await packages.createPackage({
-            environmentId: env.id,
-            name: "pkg-1",
-            manifestPath: "/tmp/env-pkg/pkg-1/publisher.json",
-            metadata: { source: "test" },
-         });
-
-         const byId = await packages.getPackageById(pkg.id);
-         expect(byId?.name).toBe("pkg-1");
-         expect(byId?.manifestPath).toBe("/tmp/env-pkg/pkg-1/publisher.json");
-         expect(byId?.metadata).toEqual({ source: "test" });
-
-         const byName = await packages.getPackageByName(env.id, "pkg-1");
-         expect(byName?.id).toBe(pkg.id);
-
-         const list = await packages.listPackages(env.id);
-         expect(list.map((p) => p.name)).toEqual(["pkg-1"]);
-
-         await packages.deletePackage(pkg.id);
-         expect(await packages.getPackageById(pkg.id)).toBeNull();
-      });
-   });
-
-   describe("ConnectionRepository", () => {
-      it("round-trips a connection's JSON config", async () => {
-         const env = await environments.createEnvironment({
-            name: "env-conn",
-            path: "/tmp/env-conn",
-         });
-
-         const config = {
-            account: "acct",
-            warehouse: "wh",
-            nested: { a: 1, b: ["x", "y"] },
-         };
-         const created = await connections.createConnection({
-            environmentId: env.id,
-            name: "snow",
-            type: "snowflake",
-            config,
-         });
-
-         const byId = await connections.getConnectionById(created.id);
-         expect(byId).not.toBeNull();
-         expect(byId!.type).toBe("snowflake");
-         // The config column is stored as JSON and must deserialize identically.
-         expect(byId!.config).toEqual(config);
-
-         const byName = await connections.getConnectionByName(env.id, "snow");
-         expect(byName?.id).toBe(created.id);
-      });
-
-      it("lists and deletes connections, including by environment", async () => {
-         const env = await environments.createEnvironment({
-            name: "env-conn2",
-            path: "/tmp/env-conn2",
-         });
-         await connections.createConnection({
-            environmentId: env.id,
-            name: "c1",
-            type: "postgres",
-            config: { host: "h" },
-         });
-         await connections.createConnection({
-            environmentId: env.id,
-            name: "c2",
-            type: "duckdb",
-            config: {},
-         });
-
-         expect(
-            (await connections.listConnections(env.id)).map((c) => c.name),
-         ).toEqual(["c1", "c2"]);
-
-         await connections.deleteConnectionsByEnvironmentId(env.id);
-         expect(await connections.listConnections(env.id)).toEqual([]);
-      });
-
-      it("updates a connection's config", async () => {
-         const env = await environments.createEnvironment({
-            name: "env-conn3",
-            path: "/tmp/env-conn3",
-         });
-         const c = await connections.createConnection({
-            environmentId: env.id,
-            name: "c",
-            type: "postgres",
-            config: { host: "old" },
-         });
-
-         const updated = await connections.updateConnection(c.id, {
-            config: { host: "new", port: 5432 },
-         });
-         expect(updated.config).toEqual({ host: "new", port: 5432 });
-
-         const reread = await connections.getConnectionById(c.id);
-         expect(reread!.config).toEqual({ host: "new", port: 5432 });
-      });
-   });
-});