@malloy-publisher/server 0.0.204 → 0.0.205

package/src/storage/duckdb/DuckDBConnection.ts

@@ -1,11 +1,30 @@
 import { Mutex } from "async-mutex";
-import duckdb from "duckdb";
+import {
+   DuckDBConnection as NeoConnection,
+   DuckDBInstance,
+   type DuckDBValue,
+} from "@duckdb/node-api";
 import * as path from "path";
 import { DatabaseConnection } from "../DatabaseInterface";
 
+/**
+ * Embedded persistence layer for the publisher's own metadata (environments,
+ * packages, connections, materializations, build manifests) in `publisher.db`.
+ *
+ * This is a plain DAO over a durable, exclusively-owned DuckDB handle -- it is
+ * deliberately NOT Malloy's `@malloydata/db-duckdb` connection, which is an
+ * analytical query connection (no prepared-statement parameter binding, a
+ * `:memory:` primary with ATTACH/DETACH/idle lifecycle, pooled/shared
+ * instances, and a poison-pill close). Those semantics are wrong for a
+ * source-of-truth store that must hold one handle open for the server's
+ * lifetime and run parameterized CRUD.
+ *
+ * It wraps `@duckdb/node-api` (the same DuckDB engine Malloy pulls in), so the
+ * repo carries a single DuckDB engine rather than a second, redundant driver.
+ */
 export class DuckDBConnection implements DatabaseConnection {
-   private db: duckdb.Database | null = null;
-   private connection: duckdb.Connection | null = null;
+   private instance: DuckDBInstance | null = null;
+   private connection: NeoConnection | null = null;
    private dbPath: string;
    private mutex: Mutex = new Mutex();
 
@@ -15,94 +34,47 @@ export class DuckDBConnection implements DatabaseConnection {
    }
 
    async initialize(): Promise<void> {
-      return new Promise((resolve, reject) => {
-         this.db = new duckdb.Database(this.dbPath, {}, (err) => {
-            if (err) {
-               console.error("Failed to create DuckDB database:", err);
-               reject(new Error(`Failed to initialize DuckDB: ${err.message}`));
-               return;
-            }
-
-            // Connect synchronously
-            this.connection = (
-               this.db as duckdb.Database & { connect(): duckdb.Connection }
-            ).connect();
-
-            if (!this.connection) {
-               reject(new Error("Failed to create connection object"));
-               return;
-            }
-
-            // Verify connection works
-            this.connection.all("SELECT 42 as answer", (testErr, _rows) => {
-               if (testErr) {
-                  console.error("Connection test failed:", testErr);
-                  reject(
-                     new Error(
-                        `Failed to verify DuckDB connection: ${testErr.message}`,
-                     ),
-                  );
-                  return;
-               }
-
-               resolve();
-            });
-         });
-      });
+      try {
+         this.instance = await DuckDBInstance.create(this.dbPath);
+         this.connection = await this.instance.connect();
+         // Verify the connection works
+         await this.connection.run("SELECT 42 as answer");
+      } catch (err) {
+         const message = err instanceof Error ? err.message : String(err);
+         console.error("Failed to create DuckDB database:", err);
+         throw new Error(`Failed to initialize DuckDB: ${message}`);
+      }
    }
 
    async close(): Promise<void> {
-      return new Promise((resolve, reject) => {
+      try {
          if (this.connection) {
-            this.connection.close((err) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Failed to close DuckDB connection: ${err.message}`,
-                     ),
-                  );
-                  return;
-               }
-
-               if (this.db) {
-                  this.db.close((dbErr) => {
-                     if (dbErr) {
-                        reject(
-                           new Error(
-                              `Failed to close DuckDB: ${dbErr.message}`,
-                           ),
-                        );
-                        return;
-                     }
-                     console.log("DuckDB connection closed");
-                     resolve();
-                  });
-               } else {
-                  resolve();
-               }
-            });
-         } else {
-            resolve();
+            this.connection.closeSync();
+            this.connection = null;
          }
-      });
+         if (this.instance) {
+            this.instance.closeSync();
+            this.instance = null;
+         }
+         console.log("DuckDB connection closed");
+      } catch (err) {
+         const message = err instanceof Error ? err.message : String(err);
+         throw new Error(`Failed to close DuckDB connection: ${message}`);
+      }
    }
 
    async isInitialized(): Promise<boolean> {
       if (!this.connection) return false;
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<boolean>((resolve) => {
-            this.connection!.all(
+         try {
+            const reader = await this.connection!.runAndReadAll(
                "SELECT name FROM sqlite_master WHERE type='table' AND name='environments'",
-               (err, rows) => {
-                  if (err) {
-                     resolve(false);
-                     return;
-                  }
-                  resolve(rows && rows.length > 0);
-               },
             );
-         });
+            return reader.getRowObjectsJS().length > 0;
+         } catch {
+            return false;
+         }
       });
    }
 
@@ -112,26 +84,14 @@ export class DuckDBConnection implements DatabaseConnection {
       }
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<void>((resolve, reject) => {
-            const callback = (err: Error | null) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Query execution failed: ${err.message}\nQuery: ${query}`,
-                     ),
-                  );
-                  return;
-               }
-               resolve();
-            };
-
-            // Pass params directly without the params argument if empty
-            if (params && params.length > 0) {
-               this.connection!.run(query, ...params, callback);
-            } else {
-               this.connection!.run(query, callback);
-            }
-         });
+         try {
+            await this.connection!.run(query, params as DuckDBValue[]);
+         } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new Error(
+               `Query execution failed: ${message}\nQuery: ${query}`,
+            );
+         }
       });
    }
 
@@ -141,25 +101,18 @@ export class DuckDBConnection implements DatabaseConnection {
       }
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<T[]>((resolve, reject) => {
-            const callback = (err: Error | null, rows: unknown[]) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Query execution failed: ${err.message}\nQuery: ${query}`,
-                     ),
-                  );
-                  return;
-               }
-               resolve((rows || []) as T[]);
-            };
-
-            if (params && params.length > 0) {
-               this.connection!.all(query, ...params, callback);
-            } else {
-               this.connection!.all(query, callback);
-            }
-         });
+         try {
+            const reader = await this.connection!.runAndReadAll(
+               query,
+               params as DuckDBValue[],
+            );
+            return reader.getRowObjectsJS() as T[];
+         } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new Error(
+               `Query execution failed: ${message}\nQuery: ${query}`,
+            );
+         }
       });
    }
 
@@ -167,11 +120,4 @@ export class DuckDBConnection implements DatabaseConnection {
       const rows = await this.all<T>(query, params);
       return rows.length > 0 ? rows[0] : null;
    }
-
-   getConnection(): duckdb.Connection {
-      if (!this.connection) {
-         throw new Error("Database not initialized");
-      }
-      return this.connection;
-   }
 }

package/tests/fixtures/authorize-compile/model.malloy

@@ -0,0 +1,9 @@
+##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$ROLE = 'analyst'"
+source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
+
+source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }

package/tests/fixtures/authorize-compile/publisher.json

@@ -0,0 +1,4 @@
+{
+   "name": "authorize-compile",
+   "description": "Fixture: a gated source for the /compile authorize HTTP integration test."
+}

package/tests/fixtures/html-pages-nopublic/model.malloy

@@ -0,0 +1 @@
+source: nums is duckdb.sql("SELECT 1 as n")

package/tests/fixtures/html-pages-nopublic/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "html-pages-nopublic",
+  "version": "0.0.1",
+  "description": "Package with no public/ directory; static file requests return 404."
+}

package/tests/fixtures/html-pages-test/data.csv

@@ -0,0 +1,3 @@
+id,name
+1,alpha
+2,beta

package/tests/fixtures/html-pages-test/public/assets/app.css

@@ -0,0 +1,3 @@
+body {
+   font-family: system-ui, sans-serif;
+}

package/tests/fixtures/html-pages-test/public/data.json

@@ -0,0 +1 @@
+{ "ok": true }

package/tests/fixtures/html-pages-test/public/index.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+   <head>
+      <title>Carrier Dashboard</title>
+   </head>
+   <body>
+      <h1>Hello from the in-package data app</h1>
+   </body>
+</html>

package/tests/fixtures/html-pages-test/public/sub/page2.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+   <head>
+      <title>Second Page</title>
+   </head>
+   <body>
+      <p>A page in a subdirectory.</p>
+   </body>
+</html>

package/tests/fixtures/html-pages-test/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "html-pages-test",
+  "version": "1.0.0",
+  "description": "Fixture for in-package HTML static-file serving + /pages + /events"
+}

package/tests/fixtures/html-pages-test/report.malloy

@@ -0,0 +1 @@
+source: report is duckdb.sql("SELECT 1 as n")

package/tests/integration/authorize/compile_authorize_http.integration.spec.ts

@@ -0,0 +1,92 @@
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import path from "path";
+import { fileURLToPath } from "url";
+import { type RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+// HTTP end-to-end for the /compile authorize gate: proves AccessDeniedError
+// surfaces as a real 403 body through the express route + internalErrorToHttpError,
+// and that the 403 names only the source (never the gate expression).
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const PROJECT = "authorize-compile-test-project";
+const PKG = "authorize-compile";
+const MODEL = "model.malloy";
+
+describe("compile authorize gate (HTTP E2E)", () => {
+   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+
+   beforeAll(async () => {
+      env = await startRestE2E();
+      baseUrl = env.baseUrl;
+      const fixtureDir = path.resolve(
+         __dirname,
+         "../../fixtures/authorize-compile",
+      );
+      const createRes = await fetch(`${baseUrl}/api/v0/projects`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: PROJECT,
+            packages: [{ name: PKG, location: fixtureDir }],
+            connections: [],
+         }),
+      });
+      if (!createRes.ok) {
+         throw new Error(
+            `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
+         );
+      }
+      const deadline = Date.now() + 30_000;
+      while (Date.now() < deadline) {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT}/packages/${PKG}`,
+         );
+         if (res.ok) break;
+         await new Promise((r) => setTimeout(r, 250));
+      }
+   });
+
+   afterAll(async () => {
+      await env?.stop();
+   });
+
+   const compile = (body: unknown) =>
+      fetch(
+         `${baseUrl}/api/v0/environments/${PROJECT}/packages/${PKG}/models/${MODEL}/compile`,
+         {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(body),
+         },
+      );
+
+   it("returns 403 with a source-only message when the gate is not satisfied", async () => {
+      const res = await compile({
+         source: "run: gated -> { aggregate: c }",
+         givens: {},
+      });
+      expect(res.status).toBe(403);
+      const json = (await res.json()) as { code: number; message: string };
+      expect(json.code).toBe(403);
+      expect(json.message).toBe('Access denied for source "gated".');
+      // Redaction: the runtime 403 must NOT leak the authorize expression.
+      expect(json.message).not.toContain("ROLE");
+      expect(json.message).not.toContain("analyst");
+   });
+
+   it("returns 200 when a satisfying given is supplied", async () => {
+      const res = await compile({
+         source: "run: gated -> { aggregate: c }",
+         givens: { ROLE: "analyst" },
+      });
+      expect(res.status).toBe(200);
+   });
+
+   it("compiles an ungated source without any given", async () => {
+      const res = await compile({
+         source: "run: open_src -> { aggregate: c }",
+      });
+      expect(res.status).toBe(200);
+   });
+});

package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts

@@ -0,0 +1,138 @@
+/// <reference types="bun-types" />
+
+// Integration test for the DuckDB storage stack end-to-end:
+//   StorageManager -> ResourceRepository -> DuckDBConnection (@duckdb/node-api)
+//   -> a real on-disk publisher.db
+//
+// Unlike the repository unit tests (which construct a single repo against a bare
+// connection) this drives the same path the server uses -- StorageManager.initialize()
+// wires the schema + repositories over the migrated connection, and getRepository()
+// is what the service layer calls. It also reopens a second StorageManager against
+// the same file to prove data persists across a restart, which is the real
+// end-to-end guarantee for the migration.
+
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { StorageManager } from "../../../src/storage/StorageManager";
+
+describe("DuckDB storage stack (integration)", () => {
+   let dbDir: string;
+   let dbPath: string;
+   let sm: StorageManager;
+
+   beforeEach(async () => {
+      dbDir = await fs.mkdtemp(path.join(os.tmpdir(), "duckdb-storage-int-"));
+      dbPath = path.join(dbDir, "publisher.db");
+      sm = new StorageManager({ type: "duckdb", duckdb: { path: dbPath } });
+      await sm.initialize();
+   });
+
+   afterEach(async () => {
+      try {
+         await sm.close();
+      } catch {
+         // ignore
+      }
+      await fs.rm(dbDir, { recursive: true, force: true });
+   });
+
+   it("initializes the schema and reports ready", () => {
+      expect(sm.isInitialized()).toBe(true);
+   });
+
+   it("creates an environment with a package and connection through the repository", async () => {
+      const repo = sm.getRepository();
+
+      const env = await repo.createEnvironment({
+         name: "prod",
+         path: "/srv/prod",
+         metadata: { owner: "platform" },
+      });
+      expect(env.id).toBeTruthy();
+
+      const pkg = await repo.createPackage({
+         environmentId: env.id,
+         name: "sales",
+         manifestPath: "/srv/prod/sales/publisher.json",
+      });
+
+      const conn = await repo.createConnection({
+         environmentId: env.id,
+         name: "warehouse",
+         type: "snowflake",
+         config: { account: "acct", warehouse: "wh" },
+      });
+
+      // Read everything back through the same repository.
+      expect((await repo.getEnvironmentByName("prod"))?.id).toBe(env.id);
+      expect((await repo.listPackages(env.id)).map((p) => p.name)).toEqual([
+         "sales",
+      ]);
+      const conns = await repo.listConnections(env.id);
+      expect(conns.map((c) => c.name)).toEqual(["warehouse"]);
+      expect(conns[0].config).toEqual({ account: "acct", warehouse: "wh" });
+      expect(pkg.environmentId).toBe(env.id);
+   });
+
+   it("persists data across a StorageManager restart (reopen the same file)", async () => {
+      const repo = sm.getRepository();
+      const env = await repo.createEnvironment({
+         name: "persisted",
+         path: "/srv/persisted",
+         metadata: { keep: true },
+      });
+      await repo.createConnection({
+         environmentId: env.id,
+         name: "pg",
+         type: "postgres",
+         config: { host: "db.internal", port: 5432 },
+      });
+
+      // Close the manager (releases the file), then open a fresh one on the
+      // same path -- the data must still be there.
+      await sm.close();
+
+      const reopened = new StorageManager({
+         type: "duckdb",
+         duckdb: { path: dbPath },
+      });
+      await reopened.initialize();
+      try {
+         const repo2 = reopened.getRepository();
+         const env2 = await repo2.getEnvironmentByName("persisted");
+         expect(env2).not.toBeNull();
+         expect(env2!.metadata).toEqual({ keep: true });
+         expect(env2!.createdAt).toBeInstanceOf(Date);
+
+         const conns = await repo2.listConnections(env2!.id);
+         expect(conns.map((c) => c.name)).toEqual(["pg"]);
+         expect(conns[0].config).toEqual({ host: "db.internal", port: 5432 });
+      } finally {
+         await reopened.close();
+      }
+
+      // Point afterEach's close() at the already-closed original without error.
+      sm = reopened;
+   });
+
+   it("deleteEnvironment also removes the environment's connections", async () => {
+      const repo = sm.getRepository();
+      const env = await repo.createEnvironment({
+         name: "to-delete",
+         path: "/srv/del",
+      });
+      await repo.createConnection({
+         environmentId: env.id,
+         name: "c1",
+         type: "duckdb",
+         config: {},
+      });
+
+      await repo.deleteEnvironment(env.id);
+
+      expect(await repo.getEnvironmentById(env.id)).toBeNull();
+      expect(await repo.listConnections(env.id)).toEqual([]);
+   });
+});