@malloy-publisher/server 0.0.204 → 0.0.206

package/src/storage/duckdb/MaterializationRepository.ts

@@ -1,12 +1,25 @@
-import { Materialization, MaterializationStatus } from "../DatabaseInterface";
+import {
+   BuildManifestResult,
+   BuildPlan,
+   Materialization,
+   MaterializationStatus,
+   MaterializationUpdate,
+} from "../DatabaseInterface";
 import { DuckDBConnection } from "./DuckDBConnection";
 
 const TERMINAL_STATUSES: ReadonlySet<MaterializationStatus> = new Set([
-   "SUCCESS",
+   "MANIFEST_FILE_READY",
    "FAILED",
    "CANCELLED",
 ]);
 
+/** Non-terminal statuses count as "active" for the single-active guard. */
+const ACTIVE_STATUSES: readonly MaterializationStatus[] = [
+   "PENDING",
+   "BUILD_PLAN_READY",
+   "MANIFEST_ROWS_READY",
+];
+
 function activeKeyFor(environmentId: string, packageName: string): string {
    return `${environmentId}|${packageName}`;
 }
@@ -73,9 +86,10 @@ export class MaterializationRepository {
       environmentId: string,
       packageName: string,
    ): Promise<Materialization | null> {
+      const placeholders = ACTIVE_STATUSES.map(() => "?").join(", ");
       const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN ('PENDING', 'RUNNING')",
-         [environmentId, packageName],
+         `SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN (${placeholders})`,
+         [environmentId, packageName, ...ACTIVE_STATUSES],
       );
       return row ? this.mapRow(row) : null;
    }
@@ -100,8 +114,8 @@ export class MaterializationRepository {
 
       try {
          const rows = await this.db.all<Record<string, unknown>>(
-            `INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, created_at, updated_at)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            `INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
             RETURNING *`,
             [
                id,
@@ -128,13 +142,7 @@ export class MaterializationRepository {
 
    async update(
       id: string,
-      updates: {
-         status?: MaterializationStatus;
-         startedAt?: Date;
-         completedAt?: Date;
-         error?: string | null;
-         metadata?: Record<string, unknown> | null;
-      },
+      updates: MaterializationUpdate,
    ): Promise<Materialization> {
       const now = this.now();
       const setClauses: string[] = [];
@@ -173,6 +181,18 @@ export class MaterializationRepository {
             updates.metadata ? JSON.stringify(updates.metadata) : null,
          );
       }
+      if (updates.buildPlan !== undefined) {
+         setClauses.push(`build_plan = ?`);
+         params.push(
+            updates.buildPlan ? JSON.stringify(updates.buildPlan) : null,
+         );
+      }
+      if (updates.manifest !== undefined) {
+         setClauses.push(`manifest = ?`);
+         params.push(
+            updates.manifest ? JSON.stringify(updates.manifest) : null,
+         );
+      }
 
       setClauses.push(`updated_at = ?`);
       params.push(now.toISOString());
@@ -190,6 +210,10 @@ export class MaterializationRepository {
       return updated;
    }
 
+   async deleteById(id: string): Promise<void> {
+      await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
+   }
+
    async deleteByEnvironmentId(environmentId: string): Promise<void> {
       await this.db.run(
          "DELETE FROM materializations WHERE environment_id = ?",
@@ -197,10 +221,6 @@ export class MaterializationRepository {
       );
    }
 
-   async deleteById(id: string): Promise<void> {
-      await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
-   }
-
    async deleteByPackage(
       environmentId: string,
       packageName: string,
@@ -212,32 +232,37 @@ export class MaterializationRepository {
    }
 
    private mapRow(row: Record<string, unknown>): Materialization {
-      let metadata: Record<string, unknown> | null = null;
-      if (row.metadata) {
-         try {
-            metadata = JSON.parse(row.metadata as string);
-         } catch {
-            metadata = null;
-         }
-      }
-
+      const metadata = parseJsonColumn<Record<string, unknown>>(row.metadata);
       return {
          id: row.id as string,
          environmentId: row.environment_id as string,
          packageName: row.package_name as string,
+         pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
          status: row.status as MaterializationStatus,
+         buildPlan: parseJsonColumn<BuildPlan>(row.build_plan),
+         manifest: parseJsonColumn<BuildManifestResult>(row.manifest),
+         metadata,
          startedAt: row.started_at ? new Date(row.started_at as string) : null,
          completedAt: row.completed_at
             ? new Date(row.completed_at as string)
             : null,
          error: row.error != null ? (row.error as string) : null,
-         metadata,
          createdAt: new Date(row.created_at as string),
          updatedAt: new Date(row.updated_at as string),
       };
    }
 }
 
+/** Parse a nullable JSON text column, returning null on absence or parse error. */
+function parseJsonColumn<T>(value: unknown): T | null {
+   if (value == null) return null;
+   try {
+      return JSON.parse(value as string) as T;
+   } catch {
+      return null;
+   }
+}
+
 /**
  * DuckDB surfaces unique-constraint violations as plain Errors whose message
  * mentions the violated index. We match on the index name rather than a

package/src/storage/duckdb/schema.ts

@@ -82,6 +82,8 @@ export async function initializeSchema(
    // `active_key` (see below) makes the insert-then-check race impossible —
    // a second concurrent create fails with a constraint violation, which the
    // service layer translates to `MaterializationConflictError`.
+   // `build_plan` (Round 1) and `manifest` (Round 2) are JSON blobs holding
+   // the two-round protocol payloads returned inline on the resource.
    await db.run(`
     CREATE TABLE IF NOT EXISTS materializations (
       id VARCHAR PRIMARY KEY,
@@ -93,29 +95,14 @@ export async function initializeSchema(
       completed_at TIMESTAMP,
       error TEXT,
       metadata JSON,
+      build_plan JSON,
+      manifest JSON,
       created_at TIMESTAMP NOT NULL,
       updated_at TIMESTAMP NOT NULL,
       FOREIGN KEY (environment_id) REFERENCES environments(id)
     )
   `);
 
-   // Build manifests table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS build_manifests (
-      id VARCHAR PRIMARY KEY,
-      environment_id VARCHAR NOT NULL,
-      package_name VARCHAR NOT NULL,
-      build_id VARCHAR NOT NULL,
-      table_name VARCHAR NOT NULL,
-      source_name VARCHAR NOT NULL,
-      connection_name VARCHAR NOT NULL,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (environment_id) REFERENCES environments(id),
-      UNIQUE (environment_id, package_name, build_id)
-    )
-  `);
-
    // Create indexes for better query performance
    await db.run(
       "CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)",
@@ -129,9 +116,6 @@ export async function initializeSchema(
    await db.run(
       "CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
    );
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)",
-   );
 }
 
 // TODO: Remove this during projects cleanup

package/tests/fixtures/authorize-compile/model.malloy

@@ -0,0 +1,9 @@
+##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$ROLE = 'analyst'"
+source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
+
+source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }

package/tests/fixtures/authorize-compile/publisher.json

@@ -0,0 +1,4 @@
+{
+   "name": "authorize-compile",
+   "description": "Fixture: a gated source for the /compile authorize HTTP integration test."
+}

package/tests/fixtures/html-pages-nopublic/model.malloy

@@ -0,0 +1 @@
+source: nums is duckdb.sql("SELECT 1 as n")

package/tests/fixtures/html-pages-nopublic/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "html-pages-nopublic",
+  "version": "0.0.1",
+  "description": "Package with no public/ directory; static file requests return 404."
+}

package/tests/fixtures/html-pages-test/data.csv

@@ -0,0 +1,3 @@
+id,name
+1,alpha
+2,beta

package/tests/fixtures/html-pages-test/public/assets/app.css

@@ -0,0 +1,3 @@
+body {
+   font-family: system-ui, sans-serif;
+}

package/tests/fixtures/html-pages-test/public/data.json

@@ -0,0 +1 @@
+{ "ok": true }

package/tests/fixtures/html-pages-test/public/index.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+   <head>
+      <title>Carrier Dashboard</title>
+   </head>
+   <body>
+      <h1>Hello from the in-package data app</h1>
+   </body>
+</html>

package/tests/fixtures/html-pages-test/public/sub/page2.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+   <head>
+      <title>Second Page</title>
+   </head>
+   <body>
+      <p>A page in a subdirectory.</p>
+   </body>
+</html>

package/tests/fixtures/html-pages-test/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "html-pages-test",
+  "version": "1.0.0",
+  "description": "Fixture for in-package HTML static-file serving + /pages + /events"
+}

package/tests/fixtures/html-pages-test/report.malloy

@@ -0,0 +1 @@
+source: report is duckdb.sql("SELECT 1 as n")

package/tests/integration/authorize/compile_authorize_http.integration.spec.ts

@@ -0,0 +1,92 @@
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import path from "path";
+import { fileURLToPath } from "url";
+import { type RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+// HTTP end-to-end for the /compile authorize gate: proves AccessDeniedError
+// surfaces as a real 403 body through the express route + internalErrorToHttpError,
+// and that the 403 names only the source (never the gate expression).
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const PROJECT = "authorize-compile-test-project";
+const PKG = "authorize-compile";
+const MODEL = "model.malloy";
+
+describe("compile authorize gate (HTTP E2E)", () => {
+   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+
+   beforeAll(async () => {
+      env = await startRestE2E();
+      baseUrl = env.baseUrl;
+      const fixtureDir = path.resolve(
+         __dirname,
+         "../../fixtures/authorize-compile",
+      );
+      const createRes = await fetch(`${baseUrl}/api/v0/projects`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: PROJECT,
+            packages: [{ name: PKG, location: fixtureDir }],
+            connections: [],
+         }),
+      });
+      if (!createRes.ok) {
+         throw new Error(
+            `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
+         );
+      }
+      const deadline = Date.now() + 30_000;
+      while (Date.now() < deadline) {
+         const res = await fetch(
+            `${baseUrl}/api/v0/projects/${PROJECT}/packages/${PKG}`,
+         );
+         if (res.ok) break;
+         await new Promise((r) => setTimeout(r, 250));
+      }
+   });
+
+   afterAll(async () => {
+      await env?.stop();
+   });
+
+   const compile = (body: unknown) =>
+      fetch(
+         `${baseUrl}/api/v0/environments/${PROJECT}/packages/${PKG}/models/${MODEL}/compile`,
+         {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify(body),
+         },
+      );
+
+   it("returns 403 with a source-only message when the gate is not satisfied", async () => {
+      const res = await compile({
+         source: "run: gated -> { aggregate: c }",
+         givens: {},
+      });
+      expect(res.status).toBe(403);
+      const json = (await res.json()) as { code: number; message: string };
+      expect(json.code).toBe(403);
+      expect(json.message).toBe('Access denied for source "gated".');
+      // Redaction: the runtime 403 must NOT leak the authorize expression.
+      expect(json.message).not.toContain("ROLE");
+      expect(json.message).not.toContain("analyst");
+   });
+
+   it("returns 200 when a satisfying given is supplied", async () => {
+      const res = await compile({
+         source: "run: gated -> { aggregate: c }",
+         givens: { ROLE: "analyst" },
+      });
+      expect(res.status).toBe(200);
+   });
+
+   it("compiles an ungated source without any given", async () => {
+      const res = await compile({
+         source: "run: open_src -> { aggregate: c }",
+      });
+      expect(res.status).toBe(200);
+   });
+});

package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts

@@ -0,0 +1,138 @@
+/// <reference types="bun-types" />
+
+// Integration test for the DuckDB storage stack end-to-end:
+//   StorageManager -> ResourceRepository -> DuckDBConnection (@duckdb/node-api)
+//   -> a real on-disk publisher.db
+//
+// Unlike the repository unit tests (which construct a single repo against a bare
+// connection) this drives the same path the server uses -- StorageManager.initialize()
+// wires the schema + repositories over the migrated connection, and getRepository()
+// is what the service layer calls. It also reopens a second StorageManager against
+// the same file to prove data persists across a restart, which is the real
+// end-to-end guarantee for the migration.
+
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { StorageManager } from "../../../src/storage/StorageManager";
+
+describe("DuckDB storage stack (integration)", () => {
+   let dbDir: string;
+   let dbPath: string;
+   let sm: StorageManager;
+
+   beforeEach(async () => {
+      dbDir = await fs.mkdtemp(path.join(os.tmpdir(), "duckdb-storage-int-"));
+      dbPath = path.join(dbDir, "publisher.db");
+      sm = new StorageManager({ type: "duckdb", duckdb: { path: dbPath } });
+      await sm.initialize();
+   });
+
+   afterEach(async () => {
+      try {
+         await sm.close();
+      } catch {
+         // ignore
+      }
+      await fs.rm(dbDir, { recursive: true, force: true });
+   });
+
+   it("initializes the schema and reports ready", () => {
+      expect(sm.isInitialized()).toBe(true);
+   });
+
+   it("creates an environment with a package and connection through the repository", async () => {
+      const repo = sm.getRepository();
+
+      const env = await repo.createEnvironment({
+         name: "prod",
+         path: "/srv/prod",
+         metadata: { owner: "platform" },
+      });
+      expect(env.id).toBeTruthy();
+
+      const pkg = await repo.createPackage({
+         environmentId: env.id,
+         name: "sales",
+         manifestPath: "/srv/prod/sales/publisher.json",
+      });
+
+      const conn = await repo.createConnection({
+         environmentId: env.id,
+         name: "warehouse",
+         type: "snowflake",
+         config: { account: "acct", warehouse: "wh" },
+      });
+
+      // Read everything back through the same repository.
+      expect((await repo.getEnvironmentByName("prod"))?.id).toBe(env.id);
+      expect((await repo.listPackages(env.id)).map((p) => p.name)).toEqual([
+         "sales",
+      ]);
+      const conns = await repo.listConnections(env.id);
+      expect(conns.map((c) => c.name)).toEqual(["warehouse"]);
+      expect(conns[0].config).toEqual({ account: "acct", warehouse: "wh" });
+      expect(pkg.environmentId).toBe(env.id);
+   });
+
+   it("persists data across a StorageManager restart (reopen the same file)", async () => {
+      const repo = sm.getRepository();
+      const env = await repo.createEnvironment({
+         name: "persisted",
+         path: "/srv/persisted",
+         metadata: { keep: true },
+      });
+      await repo.createConnection({
+         environmentId: env.id,
+         name: "pg",
+         type: "postgres",
+         config: { host: "db.internal", port: 5432 },
+      });
+
+      // Close the manager (releases the file), then open a fresh one on the
+      // same path -- the data must still be there.
+      await sm.close();
+
+      const reopened = new StorageManager({
+         type: "duckdb",
+         duckdb: { path: dbPath },
+      });
+      await reopened.initialize();
+      try {
+         const repo2 = reopened.getRepository();
+         const env2 = await repo2.getEnvironmentByName("persisted");
+         expect(env2).not.toBeNull();
+         expect(env2!.metadata).toEqual({ keep: true });
+         expect(env2!.createdAt).toBeInstanceOf(Date);
+
+         const conns = await repo2.listConnections(env2!.id);
+         expect(conns.map((c) => c.name)).toEqual(["pg"]);
+         expect(conns[0].config).toEqual({ host: "db.internal", port: 5432 });
+      } finally {
+         await reopened.close();
+      }
+
+      // Point afterEach's close() at the already-closed original without error.
+      sm = reopened;
+   });
+
+   it("deleteEnvironment also removes the environment's connections", async () => {
+      const repo = sm.getRepository();
+      const env = await repo.createEnvironment({
+         name: "to-delete",
+         path: "/srv/del",
+      });
+      await repo.createConnection({
+         environmentId: env.id,
+         name: "c1",
+         type: "duckdb",
+         config: {},
+      });
+
+      await repo.deleteEnvironment(env.id);
+
+      expect(await repo.getEnvironmentById(env.id)).toBeNull();
+      expect(await repo.listConnections(env.id)).toEqual([]);
+   });
+});