@malloy-publisher/server 0.0.204 → 0.0.205

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.204",
+   "version": "0.0.205",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -17,8 +17,9 @@
       "test": "bun run test:unit && bun run test:integration",
       "test:unit": "bun test --timeout 100000 src",
       "test:integration": "bun test --timeout 200000 tests --max-workers=1",
-      "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
-      "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
+      "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts && bun run bake-duckdb-extensions",
+      "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts && bun run bake-duckdb-extensions",
+      "bake-duckdb-extensions": "bun scripts/bake-duckdb-extensions.js",
       "start": "NODE_ENV=production bun run ./dist/server.mjs",
       "start:init": "NODE_ENV=production bun run ./dist/server.mjs --init",
       "start:dev": "NODE_ENV=development bun --watch src/server.ts",
@@ -33,6 +34,7 @@
       "@aws-sdk/client-s3": "^3.958.0",
       "@azure/identity": "^4.13.0",
       "@azure/storage-blob": "^12.26.0",
+      "@duckdb/node-api": "1.5.3-r.2",
       "@google-cloud/storage": "^7.16.0",
       "@malloydata/db-bigquery": "^0.0.405",
       "@malloydata/db-databricks": "^0.0.405",
@@ -58,7 +60,6 @@
       "class-transformer": "^0.5.1",
       "class-validator": "^0.14.1",
       "cors": "^2.8.5",
-      "duckdb": "1.4.4",
       "express": "^4.21.0",
       "extract-zip": "^2.0.1",
       "globals": "^15.9.0",

package/scripts/bake-duckdb-extensions.js

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+
+/**
+ * Pre-download ("bake") the DuckDB extensions the server loads at runtime so
+ * they are on disk before any query runs.
+ *
+ * DuckDB fetches extensions on first use from extensions.duckdb.org and caches
+ * them under ~/.duckdb/extensions/v<version>/<platform>/. Some hosts cannot
+ * reach that CDN (notably the macOS GitHub Actions fleet), so an on-demand
+ * fetch at query time fails and takes the query (or a test) down. Baking
+ * populates the cache via the same @duckdb/node-api engine the runtime uses,
+ * so later INSTALL/LOAD calls are served from disk.
+ *
+ * Runs as the last step of this package's build (`build` / `build:server-only`),
+ * so local dev, CI, and the Docker builder all bake the same set. The Docker
+ * final stage copies the baked ~/.duckdb/extensions from the builder rather
+ * than re-baking, keeping a single bake mechanism.
+ *
+ * Each extension is baked independently: a failure (offline build, transient
+ * CDN error) is logged and skipped, never aborting the others or the build.
+ */
+
+import { DuckDBInstance } from "@duckdb/node-api";
+
+// Every extension the connection layer (packages/server/src/service/connection.ts)
+// and storage manager INSTALL/LOAD at runtime, for cloud attach, the per-package
+// sandbox, federated-database attach, and the materialization catalog. Keep this
+// in sync with the install sites in those files.
+//
+// `community: true` mirrors the runtime's `FORCE INSTALL '<name>' FROM community`
+// (bigquery, snowflake); the rest are core extensions installed by name.
+// `registered` is the name the extension reports in duckdb_extensions() when it
+// differs from the INSTALL name (only postgres -> postgres_scanner).
+const EXTENSIONS = [
+   { name: "httpfs", community: false }, // cloud storage (gcs/s3/azure) + per-package sandbox
+   { name: "aws", community: false }, // s3 credential chain
+   { name: "azure", community: false }, // azure blob storage
+   { name: "postgres", community: false, registered: "postgres_scanner" }, // postgres attach + ducklake postgres catalog
+   { name: "ducklake", community: false }, // materialization catalog
+   { name: "bigquery", community: true },
+   { name: "snowflake", community: true },
+];
+
+async function main() {
+   const instance = await DuckDBInstance.create(":memory:");
+   const connection = await instance.connect();
+
+   const results = [];
+   for (const { name, community, registered } of EXTENSIONS) {
+      try {
+         const install = community
+            ? `FORCE INSTALL '${name}' FROM community;`
+            : `INSTALL ${name};`;
+         await connection.run(`${install} LOAD ${name};`);
+
+         // Verify the extension actually reports as loaded, rather than trusting
+         // that INSTALL/LOAD returned without error.
+         const reader = await connection.runAndReadAll(
+            `SELECT loaded, installed FROM duckdb_extensions() WHERE extension_name = '${registered ?? name}';`,
+         );
+         const row = reader.getRowObjectsJS()[0];
+         const loaded = row?.loaded === true;
+         const installed = row?.installed === true;
+
+         results.push({ name, installed, loaded });
+         if (loaded) {
+            console.log(`baked DuckDB extension: ${name} (loaded)`);
+         } else {
+            console.warn(
+               `DuckDB extension "${name}" installed=${installed} but not loaded`,
+            );
+         }
+      } catch (err) {
+         const message = err instanceof Error ? err.message : String(err);
+         results.push({
+            name,
+            installed: false,
+            loaded: false,
+            error: message,
+         });
+         console.warn(`skipped DuckDB extension "${name}": ${message}`);
+      }
+   }
+
+   const ok = results.filter((r) => r.loaded).map((r) => r.name);
+   const missing = results.filter((r) => !r.loaded).map((r) => r.name);
+   console.log(
+      `DuckDB extensions baked: ${ok.length}/${results.length} loaded` +
+         (ok.length ? ` [${ok.join(", ")}]` : "") +
+         (missing.length ? `; not loaded [${missing.join(", ")}]` : ""),
+   );
+
+   connection.closeSync();
+   instance.closeSync();
+}
+
+main().catch((err) => {
+   // Never fail the build on a bake error -- the extensions are an optimization,
+   // not a correctness requirement (the runtime can still fetch on demand where
+   // the network allows).
+   console.warn(
+      `DuckDB extension bake skipped: ${err instanceof Error ? err.message : String(err)}`,
+   );
+});

package/src/controller/watch-mode.controller.ts

@@ -1,23 +1,185 @@
 import chokidar, { FSWatcher } from "chokidar";
+import { EventEmitter } from "events";
 import { RequestHandler } from "express";
+import path from "path";
 import { components } from "../api";
 import { internalErrorToHttpError } from "../errors";
 import { logger } from "../logger";
-import { assertSafePackageName, safeJoinUnderRoot } from "../path_safety";
+import { assertSafePackageName } from "../path_safety";
 import { EnvironmentStore } from "../service/environment_store";
 
 type StartWatchReq = components["schemas"]["StartWatchRequest"];
 type WatchStatusRes = components["schemas"]["WatchStatus"];
 type Handler<Req = object, Res = void> = RequestHandler<object, Res, Req>;
 
+// File extensions that should trigger a live-reload event but NOT a Malloy
+// environment reload (these are package assets, not semantic-model sources).
+const ASSET_EXTS = new Set([
+   ".html",
+   ".htm",
+   ".css",
+   ".js",
+   ".mjs",
+   ".json",
+   ".png",
+   ".jpg",
+   ".jpeg",
+   ".gif",
+   ".svg",
+   ".webp",
+   ".ico",
+   ".woff",
+   ".woff2",
+]);
+const MODEL_EXTS = new Set([".malloy", ".malloynb", ".md"]);
+
 export class WatchModeController {
    watchingPath: string | null;
    watchingEnvironmentName: string | null;
-   watcher: FSWatcher;
+   watcher: FSWatcher | null = null;
+   // Serializes ensureWatching so concurrent callers can't both close and
+   // recreate the watcher and orphan one (each chokidar watcher holds OS
+   // handles). See ensureWatching.
+   private setupChain: Promise<void> | null = null;
+   /**
+    * Per-package change bus. Event name is `<environmentName>/<packageName>`.
+    * Used by the SSE live-reload endpoint to push refreshes to embedded HTML
+    * dashboards. Each emit carries `{ path, kind }` for diagnostics; the SSE
+    * handler currently ignores the payload and just sends "changed".
+    */
+   public events = new EventEmitter();
 
    constructor(private environmentStore: EnvironmentStore) {
       this.watchingPath = null;
       this.watchingEnvironmentName = null;
+      // Live-reload subscribers can be many (one per open browser tab);
+      // bump the default cap so Node doesn't warn under normal use.
+      this.events.setMaxListeners(100);
+   }
+
+   /**
+    * Idempotent: starts watching `environmentName` if not already.
+    *
+    * Throws if the environment can't be resolved — never falls back to a
+    * `<serverRoot>/<envName>` path that an attacker could control via
+    * URL-encoded path traversal. Callers (the SSE handler in particular)
+    * must validate the env name before calling this.
+    */
+   public async ensureWatching(environmentName: string): Promise<void> {
+      if (this.watchingEnvironmentName === environmentName && this.watcher) {
+         return;
+      }
+      // Serialize setup behind any in-flight one, then re-check the guard, so
+      // concurrent callers don't both close + recreate the watcher (which would
+      // orphan a chokidar instance and leak its OS file handles).
+      const run = (this.setupChain ?? Promise.resolve())
+         .catch(() => {})
+         .then(async () => {
+            if (
+               this.watchingEnvironmentName === environmentName &&
+               this.watcher
+            ) {
+               return;
+            }
+            const env = await this.environmentStore.getEnvironment(
+               environmentName,
+               false,
+            );
+            const watchPath = env.getEnvironmentPath();
+            if (this.watcher) {
+               await this.watcher.close();
+               this.watcher = null;
+            }
+            this.startWatcher(environmentName, watchPath);
+         });
+      this.setupChain = run;
+      await run;
+   }
+
+   /** Whether watch mode is currently running for the given env. */
+   public isWatching(environmentName: string): boolean {
+      return !!this.watcher && this.watchingEnvironmentName === environmentName;
+   }
+
+   private startWatcher(watchName: string, watchPath: string) {
+      this.watchingEnvironmentName = watchName;
+      this.watchingPath = watchPath;
+      this.watcher = chokidar.watch(this.watchingPath, {
+         ignored: (filePath, stats) => {
+            if (!stats?.isFile()) return false;
+            const ext = path.extname(filePath).toLowerCase();
+            return !MODEL_EXTS.has(ext) && !ASSET_EXTS.has(ext);
+         },
+         ignoreInitial: true,
+      });
+      // Reload just the package a changed file belongs to: this recompiles
+      // that package's models from disk (Package.create) and replaces the
+      // cached entry, so the next query sees the edit. Scoped to watch mode by
+      // construction — it only runs from this watcher, which is started only
+      // when watch mode is active.
+      //
+      // The previous env-level reload (getEnvironment(reload) + addEnvironment)
+      // was a no-op for compilation: addEnvironment on an existing env calls
+      // Environment.update(), which refreshes metadata/connections only and
+      // never touches this.packages, so edits never took effect.
+      // Returns true if the package recompiled cleanly. A transient compile
+      // error (e.g. a half-typed model saved mid-edit) returns false so we can
+      // avoid bouncing open browser tabs into a compile-error/404 — see onEvent.
+      const reloadPackage = async (pkgName: string): Promise<boolean> => {
+         try {
+            const environment = await this.environmentStore.getEnvironment(
+               watchName,
+               false,
+            );
+            await environment.getPackage(pkgName, true);
+            logger.info(
+               `Watch: recompiled package "${pkgName}" in environment "${watchName}"`,
+            );
+            return true;
+         } catch (error) {
+            logger.error(
+               `Watch: failed to recompile package "${pkgName}" in environment "${watchName}"`,
+               { error },
+            );
+            return false;
+         }
+      };
+      const onEvent =
+         (kind: "add" | "change" | "unlink") => async (filePath: string) => {
+            logger.info(`Watch ${kind}: ${filePath}; environment=${watchName}`);
+            // Resolve which package the file belongs to. Packages are
+            // subdirectories of the environment (env/<pkg>/...), so a file must
+            // be nested at least one level to belong to a package; files at the
+            // environment root belong to no package and are ignored.
+            const rel = path.relative(this.watchingPath ?? "", filePath);
+            const segments = rel.split(path.sep);
+            const pkgName =
+               segments.length > 1 &&
+               segments[0] &&
+               !segments[0].startsWith("..")
+                  ? segments[0]
+                  : null;
+            if (!pkgName) return;
+
+            // Recompile Malloy state only for model files. Asset edits
+            // (HTML/CSS/JS) skip recompile — they just need the live-reload
+            // fanout below. For model edits, only signal a reload once the
+            // recompile succeeds: a transient syntax error shouldn't bounce
+            // open pages into a compile error or 404.
+            const ext = path.extname(filePath).toLowerCase();
+            if (MODEL_EXTS.has(ext)) {
+               const recompiled = await reloadPackage(pkgName);
+               if (!recompiled) return;
+            }
+            // Fan out to SSE clients embedded in the affected package.
+            this.events.emit(`${watchName}/${pkgName}`, {
+               path: filePath,
+               kind,
+            });
+         };
+      this.watcher.on("add", onEvent("add"));
+      this.watcher.on("change", onEvent("change"));
+      this.watcher.on("unlink", onEvent("unlink"));
    }
 
    public getWatchStatus: Handler<void, WatchStatusRes> = async (_req, res) => {
@@ -45,9 +207,6 @@ export class WatchModeController {
          await EnvironmentStore.reloadEnvironmentManifest(
             this.environmentStore.serverRootPath,
          );
-      this.watchingEnvironmentName = watchName || null;
-
-      // Find the environment in the manifest
       const environment = environmentManifest.environments.find(
          (e) => e.name === watchName,
       );
@@ -61,51 +220,22 @@ export class WatchModeController {
          });
          return;
       }
-
-      this.watchingPath = safeJoinUnderRoot(
-         this.environmentStore.serverRootPath,
-         watchName,
-      );
-      this.watcher = chokidar.watch(this.watchingPath, {
-         ignored: (path, stats) =>
-            !!stats?.isFile() &&
-            !path.endsWith(".malloy") &&
-            !path.endsWith(".md"),
-         ignoreInitial: true,
-      });
-      const reloadEnvironment = async () => {
-         // Overwrite the environment with its existing metadata to trigger a re-read
-         const environment = await this.environmentStore.getEnvironment(
-            watchName,
-            true,
-         );
-         await this.environmentStore.addEnvironment(environment.metadata);
-         logger.info(`Reloaded environment ${watchName}`);
-      };
-
-      this.watcher.on("add", async (path) => {
-         logger.info(
-            `Detected new file ${path}, reloading environment ${watchName}`,
-         );
-         await reloadEnvironment();
-      });
-      this.watcher.on("unlink", async (path) => {
-         logger.info(
-            `Detected deletion of ${path}, reloading environment ${watchName}`,
-         );
-         await reloadEnvironment();
-      });
-      this.watcher.on("change", async (path) => {
-         logger.info(
-            `Detected change on ${path}, reloading environment ${watchName}`,
-         );
-         await reloadEnvironment();
-      });
+      try {
+         await this.ensureWatching(watchName);
+      } catch (error) {
+         logger.error(error);
+         const { status } = internalErrorToHttpError(error as Error);
+         res.status(status).json({ error: (error as Error).message });
+         return;
+      }
       res.json();
    };
 
    public stopWatchMode: Handler = async (_req, res) => {
-      this.watcher.close();
+      if (this.watcher) {
+         await this.watcher.close();
+         this.watcher = null;
+      }
       this.watchingPath = null;
       this.watchingEnvironmentName = null;
       res.json();

package/src/errors.spec.ts

@@ -1,9 +1,11 @@
 import { describe, expect, it } from "bun:test";
 import {
+   AccessDeniedError,
    BadRequestError,
    ConnectionAuthError,
    ConnectionError,
    internalErrorToHttpError,
+   ModelCompilationError,
    PayloadTooLargeError,
    QueryTimeoutError,
    ServiceUnavailableError,
@@ -29,6 +31,25 @@ describe("internalErrorToHttpError", () => {
       expect(json).toEqual({ code: 400, message: "bad input" });
    });
 
+   it("maps AccessDeniedError to 403 (authorize gate)", () => {
+      const { status, json } = internalErrorToHttpError(
+         new AccessDeniedError('Access denied for source "gated".'),
+      );
+      expect(status).toBe(403);
+      expect(json).toEqual({
+         code: 403,
+         message: 'Access denied for source "gated".',
+      });
+   });
+
+   it("maps ModelCompilationError to 424", () => {
+      const { status, json } = internalErrorToHttpError(
+         new ModelCompilationError({ message: "compile failed" }),
+      );
+      expect(status).toBe(424);
+      expect(json).toEqual({ code: 424, message: "compile failed" });
+   });
+
    it("maps ConnectionError to 502 (distinct from auth, still retryable)", () => {
       const { status, json } = internalErrorToHttpError(
          new ConnectionError("upstream broken"),

package/src/mcp/error_messages.spec.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "bun:test";
+import { AccessDeniedError } from "../errors";
+import { getMalloyErrorDetails } from "./error_messages";
+
+describe("getMalloyErrorDetails — access-denied branch", () => {
+   it("recognizes an authorize denial and gives access-relevant (not syntax) advice", () => {
+      const details = getMalloyErrorDetails(
+         "executeQuery",
+         "env/pkg/model.malloy",
+         new AccessDeniedError('Access denied for source "gated".'),
+      );
+
+      // Message carries the source name, never the gate expression.
+      expect(details.message).toContain('Access denied for source "gated".');
+
+      // The suggestion is about satisfying access (givens/role), and the
+      // generic Malloy-syntax suggestions are replaced, not appended.
+      expect(details.suggestions).toHaveLength(1);
+      expect(details.suggestions[0]).toMatch(/given|authorize|restricted/i);
+      // Not the generic "check the database connection / consult the language
+      // docs" advice that an unrecognized error would yield.
+      expect(details.suggestions.join(" ")).not.toMatch(
+         /database connection configuration|language documentation/i,
+      );
+   });
+
+   it("still falls back to generic suggestions for an unrecognized error", () => {
+      const details = getMalloyErrorDetails(
+         "executeQuery",
+         "env/pkg/model.malloy",
+         new Error("something unexpected"),
+      );
+      expect(details.suggestions.length).toBeGreaterThan(1);
+   });
+});

package/src/mcp/error_messages.ts

@@ -126,8 +126,21 @@ export function getMalloyErrorDetails(
       const invalidRequestMatch = error.message.match(
          /Invalid query request\\. Query OR queryName must be defined/i,
       );
+      // `#(authorize)` gate denial. The message names only the source (gate
+      // logic is never leaked), so the suggestion is about satisfying access,
+      // not fixing syntax — otherwise this falls through to generic Malloy
+      // syntax advice that misleads the caller.
+      const accessDeniedMatch = error.message.match(
+         /Access denied for source "([^"]+)"/i,
+      );
 
-      if (viewNotFoundMatch) {
+      if (accessDeniedMatch) {
+         refined = true;
+         const [, sourceName] = accessDeniedMatch;
+         suggestions = [
+            `Suggestion: Access to source '${sourceName}' is restricted by an #(authorize) gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`,
+         ];
+      } else if (viewNotFoundMatch) {
          refined = true;
          const [, viewName, sourceName] = viewNotFoundMatch;
          suggestions.unshift(

package/src/mcp/handler_utils.ts

@@ -3,6 +3,7 @@ import type { ResourceMetadata } from "@modelcontextprotocol/sdk/server/mcp";
 import type { ReadResourceResult } from "@modelcontextprotocol/sdk/types.js";
 import { EnvironmentStore } from "../service/environment_store";
 import {
+   AccessDeniedError,
    PackageNotFoundError,
    ModelNotFoundError,
    ModelCompilationError,
@@ -167,6 +168,17 @@ export async function getModelForQuery(
             `${environmentName}/${packageName}/${modelPath}`,
             error,
          );
+      } else if (error instanceof AccessDeniedError) {
+         // An #(authorize) denial during model setup. Funnel through
+         // getMalloyErrorDetails (which recognizes the access-denied message and
+         // gives supply-the-givens guidance) so a 403 never surfaces as an
+         // opaque internal error. Defensive: the gate fires during query
+         // execution today, not setup, but this keeps every error class homed.
+         errorDetails = getMalloyErrorDetails(
+            "executeQuery (load model)",
+            `${environmentName}/${packageName}/${modelPath}`,
+            error,
+         );
       } else if (error instanceof ServiceUnavailableError) {
          // Back-pressure: don't dress this up as a 404/500. Surface the
          // server's own message so the MCP caller knows to retry.