@malloy-publisher/server 0.0.204 → 0.0.206

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.204",
+   "version": "0.0.206",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -17,8 +17,9 @@
       "test": "bun run test:unit && bun run test:integration",
       "test:unit": "bun test --timeout 100000 src",
       "test:integration": "bun test --timeout 200000 tests --max-workers=1",
-      "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
-      "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
+      "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts && bun run bake-duckdb-extensions",
+      "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts && bun run bake-duckdb-extensions",
+      "bake-duckdb-extensions": "bun scripts/bake-duckdb-extensions.js",
       "start": "NODE_ENV=production bun run ./dist/server.mjs",
       "start:init": "NODE_ENV=production bun run ./dist/server.mjs --init",
       "start:dev": "NODE_ENV=development bun --watch src/server.ts",
@@ -33,6 +34,7 @@
       "@aws-sdk/client-s3": "^3.958.0",
       "@azure/identity": "^4.13.0",
       "@azure/storage-blob": "^12.26.0",
+      "@duckdb/node-api": "1.5.3-r.2",
       "@google-cloud/storage": "^7.16.0",
       "@malloydata/db-bigquery": "^0.0.405",
       "@malloydata/db-databricks": "^0.0.405",
@@ -58,7 +60,6 @@
       "class-transformer": "^0.5.1",
       "class-validator": "^0.14.1",
       "cors": "^2.8.5",
-      "duckdb": "1.4.4",
       "express": "^4.21.0",
       "extract-zip": "^2.0.1",
       "globals": "^15.9.0",

package/scripts/bake-duckdb-extensions.js

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+
+/**
+ * Pre-download ("bake") the DuckDB extensions the server loads at runtime so
+ * they are on disk before any query runs.
+ *
+ * DuckDB fetches extensions on first use from extensions.duckdb.org and caches
+ * them under ~/.duckdb/extensions/v<version>/<platform>/. Some hosts cannot
+ * reach that CDN (notably the macOS GitHub Actions fleet), so an on-demand
+ * fetch at query time fails and takes the query (or a test) down. Baking
+ * populates the cache via the same @duckdb/node-api engine the runtime uses,
+ * so later INSTALL/LOAD calls are served from disk.
+ *
+ * Runs as the last step of this package's build (`build` / `build:server-only`),
+ * so local dev, CI, and the Docker builder all bake the same set. The Docker
+ * final stage copies the baked ~/.duckdb/extensions from the builder rather
+ * than re-baking, keeping a single bake mechanism.
+ *
+ * Each extension is baked independently: a failure (offline build, transient
+ * CDN error) is logged and skipped, never aborting the others or the build.
+ */
+
+import { DuckDBInstance } from "@duckdb/node-api";
+
+// Every extension the connection layer (packages/server/src/service/connection.ts)
+// and storage manager INSTALL/LOAD at runtime, for cloud attach, the per-package
+// sandbox, federated-database attach, and the materialization catalog. Keep this
+// in sync with the install sites in those files.
+//
+// `community: true` mirrors the runtime's `FORCE INSTALL '<name>' FROM community`
+// (bigquery, snowflake); the rest are core extensions installed by name.
+// `registered` is the name the extension reports in duckdb_extensions() when it
+// differs from the INSTALL name (only postgres -> postgres_scanner).
+const EXTENSIONS = [
+   { name: "httpfs", community: false }, // cloud storage (gcs/s3/azure) + per-package sandbox
+   { name: "aws", community: false }, // s3 credential chain
+   { name: "azure", community: false }, // azure blob storage
+   { name: "postgres", community: false, registered: "postgres_scanner" }, // postgres attach + ducklake postgres catalog
+   { name: "ducklake", community: false }, // materialization catalog
+   { name: "bigquery", community: true },
+   { name: "snowflake", community: true },
+];
+
+async function main() {
+   const instance = await DuckDBInstance.create(":memory:");
+   const connection = await instance.connect();
+
+   const results = [];
+   for (const { name, community, registered } of EXTENSIONS) {
+      try {
+         const install = community
+            ? `FORCE INSTALL '${name}' FROM community;`
+            : `INSTALL ${name};`;
+         await connection.run(`${install} LOAD ${name};`);
+
+         // Verify the extension actually reports as loaded, rather than trusting
+         // that INSTALL/LOAD returned without error.
+         const reader = await connection.runAndReadAll(
+            `SELECT loaded, installed FROM duckdb_extensions() WHERE extension_name = '${registered ?? name}';`,
+         );
+         const row = reader.getRowObjectsJS()[0];
+         const loaded = row?.loaded === true;
+         const installed = row?.installed === true;
+
+         results.push({ name, installed, loaded });
+         if (loaded) {
+            console.log(`baked DuckDB extension: ${name} (loaded)`);
+         } else {
+            console.warn(
+               `DuckDB extension "${name}" installed=${installed} but not loaded`,
+            );
+         }
+      } catch (err) {
+         const message = err instanceof Error ? err.message : String(err);
+         results.push({
+            name,
+            installed: false,
+            loaded: false,
+            error: message,
+         });
+         console.warn(`skipped DuckDB extension "${name}": ${message}`);
+      }
+   }
+
+   const ok = results.filter((r) => r.loaded).map((r) => r.name);
+   const missing = results.filter((r) => !r.loaded).map((r) => r.name);
+   console.log(
+      `DuckDB extensions baked: ${ok.length}/${results.length} loaded` +
+         (ok.length ? ` [${ok.join(", ")}]` : "") +
+         (missing.length ? `; not loaded [${missing.join(", ")}]` : ""),
+   );
+
+   connection.closeSync();
+   instance.closeSync();
+}
+
+main().catch((err) => {
+   // Never fail the build on a bake error -- the extensions are an optimization,
+   // not a correctness requirement (the runtime can still fetch on demand where
+   // the network allows).
+   console.warn(
+      `DuckDB extension bake skipped: ${err instanceof Error ? err.message : String(err)}`,
+   );
+});

package/src/constants.ts

@@ -5,6 +5,18 @@ export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
 export const PACKAGE_MANIFEST_NAME = "publisher.json";
 export const MODEL_FILE_SUFFIX = ".malloy";
 export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
+
+/**
+ * Normalize a package-relative model path so author-written `explores`
+ * entries compare equal to the paths produced by `listPackageFiles`
+ * (forward slashes, no leading "./"). Shared by every input channel — the
+ * worker's on-disk parse and the API publish/update path — so a manifest
+ * behaves the same whether it's loaded from disk or written via the API.
+ */
+export function normalizeModelPath(p: string): string {
+   return p.replace(/\\/g, "/").replace(/^\.\//, "");
+}
+
 /**
  * Default row cap applied to Malloy model queries (the `runnable.run`
  * path used by `getQueryResults` and notebook cell execution) when

package/src/controller/materialization.controller.ts

@@ -1,4 +1,5 @@
 import { BadRequestError } from "../errors";
+import { BuildInstruction } from "../storage/DatabaseInterface";
 import { MaterializationService } from "../service/materialization_service";
 
 export class MaterializationController {
@@ -9,46 +10,113 @@ export class MaterializationController {
       packageName: string,
       body: Record<string, unknown>,
    ) {
-      const options = this.validateCreateBody(body);
       return this.materializationService.createMaterialization(
          environmentName,
          packageName,
-         options,
+         this.validateCreateBody(body),
       );
    }
 
    private validateCreateBody(body: Record<string, unknown>): {
       forceRefresh?: boolean;
-      autoLoadManifest?: boolean;
+      sourceNames?: string[];
+      pauseBetweenPhases?: boolean;
    } {
-      const result: { forceRefresh?: boolean; autoLoadManifest?: boolean } = {};
+      const result: {
+         forceRefresh?: boolean;
+         sourceNames?: string[];
+         pauseBetweenPhases?: boolean;
+      } = {};
+      if (body.pauseBetweenPhases !== undefined) {
+         if (typeof body.pauseBetweenPhases !== "boolean") {
+            throw new BadRequestError("pauseBetweenPhases must be a boolean");
+         }
+         result.pauseBetweenPhases = body.pauseBetweenPhases;
+      }
       if (body.forceRefresh !== undefined) {
          if (typeof body.forceRefresh !== "boolean") {
             throw new BadRequestError("forceRefresh must be a boolean");
          }
          result.forceRefresh = body.forceRefresh;
       }
-      if (body.autoLoadManifest !== undefined) {
-         if (typeof body.autoLoadManifest !== "boolean") {
-            throw new BadRequestError("autoLoadManifest must be a boolean");
+      if (body.sourceNames !== undefined) {
+         if (
+            !Array.isArray(body.sourceNames) ||
+            body.sourceNames.some((n) => typeof n !== "string")
+         ) {
+            throw new BadRequestError(
+               "sourceNames must be an array of strings",
+            );
          }
-         result.autoLoadManifest = body.autoLoadManifest;
+         result.sourceNames = body.sourceNames as string[];
       }
       return result;
    }
 
-   async startMaterialization(
+   async buildMaterialization(
       environmentName: string,
       packageName: string,
       materializationId: string,
+      body: Record<string, unknown>,
    ) {
-      return this.materializationService.startMaterialization(
+      return this.materializationService.buildMaterialization(
          environmentName,
          packageName,
          materializationId,
+         this.validateBuildBody(body),
       );
    }
 
+   private validateBuildBody(
+      body: Record<string, unknown>,
+   ): BuildInstruction[] {
+      const sources = body.sources;
+      if (!Array.isArray(sources) || sources.length === 0) {
+         throw new BadRequestError(
+            "build requires a non-empty 'sources' array of BuildInstruction",
+         );
+      }
+      return sources.map((raw) => this.validateInstruction(raw));
+   }
+
+   private validateInstruction(raw: unknown): BuildInstruction {
+      if (typeof raw !== "object" || raw === null) {
+         throw new BadRequestError("Each build instruction must be an object");
+      }
+      const instruction = raw as Record<string, unknown>;
+      const required = [
+         "buildId",
+         "materializedTableId",
+         "physicalTableName",
+         "realization",
+      ] as const;
+      for (const field of required) {
+         if (typeof instruction[field] !== "string") {
+            throw new BadRequestError(
+               `Build instruction is missing required string field '${field}'`,
+            );
+         }
+      }
+      if (
+         instruction.realization !== "COPY" &&
+         instruction.realization !== "SNAPSHOT"
+      ) {
+         throw new BadRequestError(
+            "Build instruction 'realization' must be COPY or SNAPSHOT",
+         );
+      }
+      return {
+         buildId: instruction.buildId as string,
+         sourceID:
+            typeof instruction.sourceID === "string"
+               ? instruction.sourceID
+               : undefined,
+         materializedTableId: instruction.materializedTableId as string,
+         physicalTableName: instruction.physicalTableName as string,
+         realization: instruction.realization,
+      };
+   }
+
    async stopMaterialization(
       environmentName: string,
       packageName: string,
@@ -89,37 +157,13 @@ export class MaterializationController {
       environmentName: string,
       packageName: string,
       materializationId: string,
+      options: { dropTables?: boolean } = {},
    ) {
       return this.materializationService.deleteMaterialization(
          environmentName,
          packageName,
          materializationId,
-      );
-   }
-
-   async teardownPackage(
-      environmentName: string,
-      packageName: string,
-      body: Record<string, unknown>,
-   ) {
-      const options = this.validateTeardownBody(body);
-      return this.materializationService.teardownPackage(
-         environmentName,
-         packageName,
          options,
       );
    }
-
-   private validateTeardownBody(body: Record<string, unknown>): {
-      dryRun?: boolean;
-   } {
-      const options: { dryRun?: boolean } = {};
-      if (body.dryRun !== undefined) {
-         if (typeof body.dryRun !== "boolean") {
-            throw new BadRequestError("dryRun must be a boolean");
-         }
-         options.dryRun = body.dryRun;
-      }
-      return options;
-   }
 }

package/src/controller/package.controller.spec.ts

@@ -0,0 +1,179 @@
+import { afterEach, describe, expect, it } from "bun:test";
+import sinon from "sinon";
+
+import { BadRequestError } from "../errors";
+import type { EnvironmentStore } from "../service/environment_store";
+import { PackageController } from "./package.controller";
+
+describe("PackageController.addPackage explores validation", () => {
+   afterEach(() => {
+      sinon.restore();
+   });
+
+   it("no-location: rejects invalid explores and rolls back via unloadPackage (NOT deletePackage)", async () => {
+      // The no-location path registers a PRE-EXISTING user directory, so a bad
+      // manifest must unload it from memory — never deletePackage, which would
+      // delete the user's files.
+      const invalidMsg =
+         "Invalid explores entry 'missing.malloy' in publisher.json: file not found";
+      const mockPackage = {
+         formatInvalidExplores: () => invalidMsg,
+      };
+      const unloadPackage = sinon.stub().resolves(undefined);
+      const deletePackage = sinon.stub().resolves(undefined);
+      const addPackage = sinon.stub().resolves(mockPackage);
+      const environment = { addPackage, unloadPackage, deletePackage };
+      const getEnvironment = sinon.stub().resolves(environment);
+      const addPackageToDatabase = sinon.stub().resolves(undefined);
+      const environmentStore = {
+         publisherConfigIsFrozen: false,
+         getEnvironment,
+         addPackageToDatabase,
+      } as unknown as EnvironmentStore;
+
+      const controller = new PackageController(environmentStore);
+
+      await expect(
+         controller.addPackage("env", {
+            name: "pkg",
+            description: "test",
+            explores: ["missing.malloy"],
+         }),
+      ).rejects.toBeInstanceOf(BadRequestError);
+
+      expect(unloadPackage.calledOnceWith("pkg")).toBe(true);
+      expect(deletePackage.called).toBe(false);
+      expect(addPackageToDatabase.called).toBe(false);
+   });
+
+   it("location: validation runs inside installPackage's rollback window, not as a controller delete", async () => {
+      // For the location path the tree was freshly downloaded, so validation is
+      // delegated to installPackage (which rolls the swap back on failure). The
+      // controller passes a validator and does NOT call delete/unload itself.
+      const invalidMsg =
+         "Invalid explores entry 'missing.malloy' in publisher.json: file not found";
+      const mockPackage = { formatInvalidExplores: () => invalidMsg };
+      // installPackage mimics the real contract: invoke the validator and, if it
+      // returns a message, throw BadRequestError (after its internal rollback).
+      const installPackage = sinon
+         .stub()
+         .callsFake(
+            async (
+               _name: string,
+               _downloader: unknown,
+               validate?: (pkg: unknown) => string | undefined,
+            ) => {
+               const msg = validate?.(mockPackage);
+               if (msg) throw new BadRequestError(msg);
+               return mockPackage;
+            },
+         );
+      const unloadPackage = sinon.stub().resolves(undefined);
+      const deletePackage = sinon.stub().resolves(undefined);
+      const environment = { installPackage, unloadPackage, deletePackage };
+      const getEnvironment = sinon.stub().resolves(environment);
+      const addPackageToDatabase = sinon.stub().resolves(undefined);
+      const environmentStore = {
+         publisherConfigIsFrozen: false,
+         getEnvironment,
+         addPackageToDatabase,
+      } as unknown as EnvironmentStore;
+
+      const controller = new PackageController(environmentStore);
+
+      await expect(
+         controller.addPackage("env", {
+            name: "pkg",
+            description: "test",
+            location: "gs://bucket/pkg.zip",
+            explores: ["missing.malloy"],
+         }),
+      ).rejects.toBeInstanceOf(BadRequestError);
+
+      expect(installPackage.calledOnce).toBe(true);
+      expect(typeof installPackage.firstCall.args[2]).toBe("function");
+      expect(deletePackage.called).toBe(false);
+      expect(unloadPackage.called).toBe(false);
+      expect(addPackageToDatabase.called).toBe(false);
+   });
+
+   it("persists when explores are valid (no-location)", async () => {
+      const mockPackage = {
+         formatInvalidExplores: () => "",
+      };
+      const addPackage = sinon.stub().resolves(mockPackage);
+      const getEnvironment = sinon.stub().resolves({ addPackage });
+      const addPackageToDatabase = sinon.stub().resolves(undefined);
+      const environmentStore = {
+         publisherConfigIsFrozen: false,
+         getEnvironment,
+         addPackageToDatabase,
+      } as unknown as EnvironmentStore;
+
+      const controller = new PackageController(environmentStore);
+
+      await controller.addPackage("env", {
+         name: "pkg",
+         description: "test",
+         explores: ["index.malloy"],
+      });
+
+      expect(addPackageToDatabase.calledOnceWith("env", "pkg")).toBe(true);
+   });
+});
+
+describe("PackageController.updatePackage explores validation", () => {
+   afterEach(() => {
+      sinon.restore();
+   });
+
+   it("location update: validates the EFFECTIVE explores (body override) before the swap commits", async () => {
+      // body.location triggers a reinstall (atomic swap). The effective explores
+      // — the body override here — must be validated inside installPackage so a
+      // bad update rolls back to the previous tree instead of swapping in the
+      // rejected one and 400-ing after the fact.
+      const invalidMsg =
+         "Invalid explores entry 'nope.malloy' in publisher.json: file not found";
+      // The mock package validates whatever override it's handed.
+      const mockPackage = {
+         formatInvalidExplores: (override?: string[]) =>
+            override?.includes("nope.malloy") ? invalidMsg : "",
+      };
+      const installPackage = sinon
+         .stub()
+         .callsFake(
+            async (
+               _name: string,
+               _downloader: unknown,
+               validate?: (pkg: unknown) => string | undefined,
+            ) => {
+               const msg = validate?.(mockPackage);
+               if (msg) throw new BadRequestError(msg);
+               return mockPackage;
+            },
+         );
+      const updatePackage = sinon.stub().resolves(mockPackage);
+      const environment = { installPackage, updatePackage };
+      const getEnvironment = sinon.stub().resolves(environment);
+      const addPackageToDatabase = sinon.stub().resolves(undefined);
+      const environmentStore = {
+         publisherConfigIsFrozen: false,
+         getEnvironment,
+         addPackageToDatabase,
+      } as unknown as EnvironmentStore;
+
+      const controller = new PackageController(environmentStore);
+
+      await expect(
+         controller.updatePackage("env", "pkg", {
+            name: "pkg",
+            location: "gs://bucket/pkg.zip",
+            explores: ["nope.malloy"],
+         }),
+      ).rejects.toBeInstanceOf(BadRequestError);
+
+      // The rejected swap never reached the metadata-apply / persist steps.
+      expect(updatePackage.called).toBe(false);
+      expect(addPackageToDatabase.called).toBe(false);
+   });
+});