@malloy-publisher/server 0.0.204 → 0.0.206

package/src/controller/package.controller.ts

@@ -1,21 +1,15 @@
 import { components } from "../api";
+import { normalizeModelPath } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
-import { logger } from "../logger";
 import { EnvironmentStore } from "../service/environment_store";
-import { ManifestService } from "../service/manifest_service";
 
 type ApiPackage = components["schemas"]["Package"];
 
 export class PackageController {
    private environmentStore: EnvironmentStore;
-   private manifestService: ManifestService;
 
-   constructor(
-      environmentStore: EnvironmentStore,
-      manifestService: ManifestService,
-   ) {
+   constructor(environmentStore: EnvironmentStore) {
       this.environmentStore = environmentStore;
-      this.manifestService = manifestService;
    }
 
    public async listPackages(environmentName: string): Promise<ApiPackage[]> {
@@ -70,11 +64,7 @@ export class PackageController {
       return _package.getPackageMetadata();
    }
 
-   async addPackage(
-      environmentName: string,
-      body: ApiPackage,
-      options?: { autoLoadManifest?: boolean },
-   ) {
+   async addPackage(environmentName: string, body: ApiPackage) {
       if (this.environmentStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -86,73 +76,61 @@ export class PackageController {
          environmentName,
          false,
       );
+      // Strict at publish: the author is in the loop here, so reject a bad
+      // explores with an actionable 400 instead of silently serving a hidden
+      // surface. (At startup/reload we fail safe and only warn — see
+      // Package.loadViaWorker.) The rollback differs by path so a rejected
+      // publish never destroys user content:
+      //   - location: the tree was just downloaded into a fresh canonical, so
+      //     validation runs inside installPackage's swap window and a failure
+      //     wipes that download (the existing rollback) — nothing pre-existing
+      //     to lose.
+      //   - no-location: addPackage registered a *pre-existing* user directory,
+      //     so we validate after the fact and `unloadPackage` (evict from
+      //     memory, keep the files) rather than delete it.
       let result;
       if (body.location) {
          const bodyLocation = body.location;
-         result = await environment.installPackage(packageName, (stagingPath) =>
-            this.downloadInto(
-               environmentName,
-               packageName,
-               bodyLocation,
-               stagingPath,
-            ),
+         result = await environment.installPackage(
+            packageName,
+            (stagingPath) =>
+               this.downloadInto(
+                  environmentName,
+                  packageName,
+                  bodyLocation,
+                  stagingPath,
+               ),
+            (pkg) => pkg.formatInvalidExplores(),
          );
       } else {
          result = await environment.addPackage(packageName);
       }
+
+      // `addPackage`/`installPackage` are typed `Package | undefined`; a missing
+      // result here is a should-never-happen internal fault. Fail loudly rather
+      // than letting optional chaining silently skip the validation below.
+      if (!result) {
+         throw new Error(`Failed to create package ${packageName}`);
+      }
+
+      if (!body.location) {
+         const invalidMsg = result.formatInvalidExplores();
+         if (invalidMsg) {
+            await environment.unloadPackage(packageName).catch(() => {
+               /* best-effort; the package is not persisted below */
+            });
+            throw new BadRequestError(invalidMsg);
+         }
+      }
+
       await this.environmentStore.addPackageToDatabase(
          environmentName,
          packageName,
       );
 
-      if (options?.autoLoadManifest === true) {
-         await this.tryLoadExistingManifest(environmentName, packageName);
-      }
-
       return result;
    }
 
-   /**
-    * If there are already manifest entries for this package (e.g. from a
-    * previous materialization run), reload all models with the manifest so
-    * persist references resolve to the materialized tables immediately.
-    */
-   private async tryLoadExistingManifest(
-      environmentName: string,
-      packageName: string,
-   ): Promise<void> {
-      try {
-         const repository =
-            this.environmentStore.storageManager.getRepository();
-         const dbEnvironment =
-            await repository.getEnvironmentByName(environmentName);
-         if (!dbEnvironment) return;
-
-         const manifest = await this.manifestService.getManifest(
-            dbEnvironment.id,
-            packageName,
-         );
-         if (Object.keys(manifest.entries).length === 0) return;
-
-         await this.manifestService.reloadManifest(
-            dbEnvironment.id,
-            packageName,
-            environmentName,
-         );
-         logger.info("Auto-loaded existing manifest for added package", {
-            environmentName,
-            packageName,
-            entryCount: Object.keys(manifest.entries).length,
-         });
-      } catch (error) {
-         logger.warn("Failed to auto-load manifest for package", {
-            environmentName,
-            packageName,
-            error,
-         });
-      }
-   }
-
    public async deletePackage(environmentName: string, packageName: string) {
       if (this.environmentStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
@@ -184,15 +162,24 @@ export class PackageController {
       );
       if (body.location) {
          // Re-install: stream the new content into a staging dir (no lock)
-         // and atomically swap it in (under the lock).
+         // and atomically swap it in (under the lock). Validate the effective
+         // explores (the body override, else the new tree's own manifest)
+         // INSIDE the swap window, so a rejected update rolls back to the
+         // previous tree instead of swapping the bad one in and 400-ing after.
          const bodyLocation = body.location;
-         await environment.installPackage(packageName, (stagingPath) =>
-            this.downloadInto(
-               environmentName,
-               packageName,
-               bodyLocation,
-               stagingPath,
-            ),
+         await environment.installPackage(
+            packageName,
+            (stagingPath) =>
+               this.downloadInto(
+                  environmentName,
+                  packageName,
+                  bodyLocation,
+                  stagingPath,
+               ),
+            (pkg) =>
+               pkg.formatInvalidExplores(
+                  body.explores?.map(normalizeModelPath),
+               ),
          );
       }
       // Apply metadata changes (publisher.json) under the same per-package

package/src/controller/watch-mode.controller.ts

@@ -1,23 +1,185 @@
 import chokidar, { FSWatcher } from "chokidar";
+import { EventEmitter } from "events";
 import { RequestHandler } from "express";
+import path from "path";
 import { components } from "../api";
 import { internalErrorToHttpError } from "../errors";
 import { logger } from "../logger";
-import { assertSafePackageName, safeJoinUnderRoot } from "../path_safety";
+import { assertSafePackageName } from "../path_safety";
 import { EnvironmentStore } from "../service/environment_store";
 
 type StartWatchReq = components["schemas"]["StartWatchRequest"];
 type WatchStatusRes = components["schemas"]["WatchStatus"];
 type Handler<Req = object, Res = void> = RequestHandler<object, Res, Req>;
 
+// File extensions that should trigger a live-reload event but NOT a Malloy
+// environment reload (these are package assets, not semantic-model sources).
+const ASSET_EXTS = new Set([
+   ".html",
+   ".htm",
+   ".css",
+   ".js",
+   ".mjs",
+   ".json",
+   ".png",
+   ".jpg",
+   ".jpeg",
+   ".gif",
+   ".svg",
+   ".webp",
+   ".ico",
+   ".woff",
+   ".woff2",
+]);
+const MODEL_EXTS = new Set([".malloy", ".malloynb", ".md"]);
+
 export class WatchModeController {
    watchingPath: string | null;
    watchingEnvironmentName: string | null;
-   watcher: FSWatcher;
+   watcher: FSWatcher | null = null;
+   // Serializes ensureWatching so concurrent callers can't both close and
+   // recreate the watcher and orphan one (each chokidar watcher holds OS
+   // handles). See ensureWatching.
+   private setupChain: Promise<void> | null = null;
+   /**
+    * Per-package change bus. Event name is `<environmentName>/<packageName>`.
+    * Used by the SSE live-reload endpoint to push refreshes to embedded HTML
+    * dashboards. Each emit carries `{ path, kind }` for diagnostics; the SSE
+    * handler currently ignores the payload and just sends "changed".
+    */
+   public events = new EventEmitter();
 
    constructor(private environmentStore: EnvironmentStore) {
       this.watchingPath = null;
       this.watchingEnvironmentName = null;
+      // Live-reload subscribers can be many (one per open browser tab);
+      // bump the default cap so Node doesn't warn under normal use.
+      this.events.setMaxListeners(100);
+   }
+
+   /**
+    * Idempotent: starts watching `environmentName` if not already.
+    *
+    * Throws if the environment can't be resolved — never falls back to a
+    * `<serverRoot>/<envName>` path that an attacker could control via
+    * URL-encoded path traversal. Callers (the SSE handler in particular)
+    * must validate the env name before calling this.
+    */
+   public async ensureWatching(environmentName: string): Promise<void> {
+      if (this.watchingEnvironmentName === environmentName && this.watcher) {
+         return;
+      }
+      // Serialize setup behind any in-flight one, then re-check the guard, so
+      // concurrent callers don't both close + recreate the watcher (which would
+      // orphan a chokidar instance and leak its OS file handles).
+      const run = (this.setupChain ?? Promise.resolve())
+         .catch(() => {})
+         .then(async () => {
+            if (
+               this.watchingEnvironmentName === environmentName &&
+               this.watcher
+            ) {
+               return;
+            }
+            const env = await this.environmentStore.getEnvironment(
+               environmentName,
+               false,
+            );
+            const watchPath = env.getEnvironmentPath();
+            if (this.watcher) {
+               await this.watcher.close();
+               this.watcher = null;
+            }
+            this.startWatcher(environmentName, watchPath);
+         });
+      this.setupChain = run;
+      await run;
+   }
+
+   /** Whether watch mode is currently running for the given env. */
+   public isWatching(environmentName: string): boolean {
+      return !!this.watcher && this.watchingEnvironmentName === environmentName;
+   }
+
+   private startWatcher(watchName: string, watchPath: string) {
+      this.watchingEnvironmentName = watchName;
+      this.watchingPath = watchPath;
+      this.watcher = chokidar.watch(this.watchingPath, {
+         ignored: (filePath, stats) => {
+            if (!stats?.isFile()) return false;
+            const ext = path.extname(filePath).toLowerCase();
+            return !MODEL_EXTS.has(ext) && !ASSET_EXTS.has(ext);
+         },
+         ignoreInitial: true,
+      });
+      // Reload just the package a changed file belongs to: this recompiles
+      // that package's models from disk (Package.create) and replaces the
+      // cached entry, so the next query sees the edit. Scoped to watch mode by
+      // construction — it only runs from this watcher, which is started only
+      // when watch mode is active.
+      //
+      // The previous env-level reload (getEnvironment(reload) + addEnvironment)
+      // was a no-op for compilation: addEnvironment on an existing env calls
+      // Environment.update(), which refreshes metadata/connections only and
+      // never touches this.packages, so edits never took effect.
+      // Returns true if the package recompiled cleanly. A transient compile
+      // error (e.g. a half-typed model saved mid-edit) returns false so we can
+      // avoid bouncing open browser tabs into a compile-error/404 — see onEvent.
+      const reloadPackage = async (pkgName: string): Promise<boolean> => {
+         try {
+            const environment = await this.environmentStore.getEnvironment(
+               watchName,
+               false,
+            );
+            await environment.getPackage(pkgName, true);
+            logger.info(
+               `Watch: recompiled package "${pkgName}" in environment "${watchName}"`,
+            );
+            return true;
+         } catch (error) {
+            logger.error(
+               `Watch: failed to recompile package "${pkgName}" in environment "${watchName}"`,
+               { error },
+            );
+            return false;
+         }
+      };
+      const onEvent =
+         (kind: "add" | "change" | "unlink") => async (filePath: string) => {
+            logger.info(`Watch ${kind}: ${filePath}; environment=${watchName}`);
+            // Resolve which package the file belongs to. Packages are
+            // subdirectories of the environment (env/<pkg>/...), so a file must
+            // be nested at least one level to belong to a package; files at the
+            // environment root belong to no package and are ignored.
+            const rel = path.relative(this.watchingPath ?? "", filePath);
+            const segments = rel.split(path.sep);
+            const pkgName =
+               segments.length > 1 &&
+               segments[0] &&
+               !segments[0].startsWith("..")
+                  ? segments[0]
+                  : null;
+            if (!pkgName) return;
+
+            // Recompile Malloy state only for model files. Asset edits
+            // (HTML/CSS/JS) skip recompile — they just need the live-reload
+            // fanout below. For model edits, only signal a reload once the
+            // recompile succeeds: a transient syntax error shouldn't bounce
+            // open pages into a compile error or 404.
+            const ext = path.extname(filePath).toLowerCase();
+            if (MODEL_EXTS.has(ext)) {
+               const recompiled = await reloadPackage(pkgName);
+               if (!recompiled) return;
+            }
+            // Fan out to SSE clients embedded in the affected package.
+            this.events.emit(`${watchName}/${pkgName}`, {
+               path: filePath,
+               kind,
+            });
+         };
+      this.watcher.on("add", onEvent("add"));
+      this.watcher.on("change", onEvent("change"));
+      this.watcher.on("unlink", onEvent("unlink"));
    }
 
    public getWatchStatus: Handler<void, WatchStatusRes> = async (_req, res) => {
@@ -45,9 +207,6 @@ export class WatchModeController {
          await EnvironmentStore.reloadEnvironmentManifest(
             this.environmentStore.serverRootPath,
          );
-      this.watchingEnvironmentName = watchName || null;
-
-      // Find the environment in the manifest
       const environment = environmentManifest.environments.find(
          (e) => e.name === watchName,
       );
@@ -61,51 +220,22 @@ export class WatchModeController {
          });
          return;
       }
-
-      this.watchingPath = safeJoinUnderRoot(
-         this.environmentStore.serverRootPath,
-         watchName,
-      );
-      this.watcher = chokidar.watch(this.watchingPath, {
-         ignored: (path, stats) =>
-            !!stats?.isFile() &&
-            !path.endsWith(".malloy") &&
-            !path.endsWith(".md"),
-         ignoreInitial: true,
-      });
-      const reloadEnvironment = async () => {
-         // Overwrite the environment with its existing metadata to trigger a re-read
-         const environment = await this.environmentStore.getEnvironment(
-            watchName,
-            true,
-         );
-         await this.environmentStore.addEnvironment(environment.metadata);
-         logger.info(`Reloaded environment ${watchName}`);
-      };
-
-      this.watcher.on("add", async (path) => {
-         logger.info(
-            `Detected new file ${path}, reloading environment ${watchName}`,
-         );
-         await reloadEnvironment();
-      });
-      this.watcher.on("unlink", async (path) => {
-         logger.info(
-            `Detected deletion of ${path}, reloading environment ${watchName}`,
-         );
-         await reloadEnvironment();
-      });
-      this.watcher.on("change", async (path) => {
-         logger.info(
-            `Detected change on ${path}, reloading environment ${watchName}`,
-         );
-         await reloadEnvironment();
-      });
+      try {
+         await this.ensureWatching(watchName);
+      } catch (error) {
+         logger.error(error);
+         const { status } = internalErrorToHttpError(error as Error);
+         res.status(status).json({ error: (error as Error).message });
+         return;
+      }
       res.json();
    };
 
    public stopWatchMode: Handler = async (_req, res) => {
-      this.watcher.close();
+      if (this.watcher) {
+         await this.watcher.close();
+         this.watcher = null;
+      }
       this.watchingPath = null;
       this.watchingEnvironmentName = null;
       res.json();

package/src/dto/package.dto.ts

@@ -1,4 +1,10 @@
-import { IsString, IsNotEmpty } from "class-validator";
+import {
+   IsString,
+   IsNotEmpty,
+   IsOptional,
+   IsArray,
+   IsIn,
+} from "class-validator";
 import { ApiPackage } from "../service/package";
 
 export class PackageDto implements ApiPackage {
@@ -9,4 +15,13 @@ export class PackageDto implements ApiPackage {
    @IsString()
    @IsNotEmpty()
    description: string;
+
+   @IsOptional()
+   @IsArray()
+   @IsString({ each: true })
+   explores?: string[];
+
+   @IsOptional()
+   @IsIn(["declared", "all"])
+   queryableSources?: "declared" | "all";
 }

package/src/errors.spec.ts

@@ -1,9 +1,12 @@
 import { describe, expect, it } from "bun:test";
 import {
+   AccessDeniedError,
    BadRequestError,
    ConnectionAuthError,
    ConnectionError,
    internalErrorToHttpError,
+   ModelCompilationError,
+   NotQueryableError,
    PayloadTooLargeError,
    QueryTimeoutError,
    ServiceUnavailableError,
@@ -29,6 +32,36 @@ describe("internalErrorToHttpError", () => {
       expect(json).toEqual({ code: 400, message: "bad input" });
    });
 
+   it("maps AccessDeniedError to 403 (authorize gate)", () => {
+      const { status, json } = internalErrorToHttpError(
+         new AccessDeniedError('Access denied for source "gated".'),
+      );
+      expect(status).toBe(403);
+      expect(json).toEqual({
+         code: 403,
+         message: 'Access denied for source "gated".',
+      });
+   });
+
+   it("maps NotQueryableError to 404 (explore boundary)", () => {
+      const { status, json } = internalErrorToHttpError(
+         new NotQueryableError('No queryable source "hidden".'),
+      );
+      expect(status).toBe(404);
+      expect(json).toEqual({
+         code: 404,
+         message: 'No queryable source "hidden".',
+      });
+   });
+
+   it("maps ModelCompilationError to 424", () => {
+      const { status, json } = internalErrorToHttpError(
+         new ModelCompilationError({ message: "compile failed" }),
+      );
+      expect(status).toBe(424);
+      expect(json).toEqual({ code: 424, message: "compile failed" });
+   });
+
    it("maps ConnectionError to 502 (distinct from auth, still retryable)", () => {
       const { status, json } = internalErrorToHttpError(
          new ConnectionError("upstream broken"),

package/src/errors.ts

@@ -14,6 +14,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(404, error.message);
    } else if (error instanceof ModelNotFoundError) {
       return httpError(404, error.message);
+   } else if (error instanceof NotQueryableError) {
+      return httpError(404, error.message);
    } else if (error instanceof MalloyError) {
       return httpError(400, error.message);
    } else if (error instanceof ConnectionNotFoundError) {
@@ -128,6 +130,22 @@ export class AccessDeniedError extends Error {
    }
 }
 
+/**
+ * A query targeted a source/model that is not part of the package's queryable
+ * surface under `queryableSources: "declared"` (a non-`explores` model file, or
+ * a source not in a model's `export {}` closure). Mapped to HTTP **404**, not
+ * 403, and with a deliberately generic message: unlike `#(authorize)` (which is
+ * identity-scoped and answers "who"), the explore boundary is identity-free and
+ * answers "what is queryable" — so a hidden target should be indistinguishable
+ * from a non-existent one (no enumeration / existence oracle).
+ */
+export class NotQueryableError extends Error {
+   constructor(message: string) {
+      super(message);
+      this.name = "NotQueryableError";
+   }
+}
+
 export class MaterializationNotFoundError extends Error {
    constructor(message: string) {
       super(message);

package/src/health.spec.ts

@@ -1,7 +1,16 @@
-import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test";
+import {
+   afterAll,
+   afterEach,
+   beforeEach,
+   describe,
+   expect,
+   it,
+   spyOn,
+} from "bun:test";
 import { Server } from "http";
 import { performGracefulShutdownAfterDrain } from "./health";
 import { logger } from "./logger";
+import { __setPackageLoadPoolForTests } from "./package_load/package_load_pool";
 
 // Regression test for the graceful-shutdown ordering bug that caused
 //   [winston] Attempt to write logs with no transports: {"message":"Waiting 50 seconds..."}
@@ -16,6 +25,18 @@ describe("performGracefulShutdownAfterDrain: shutdown ordering", () => {
    const originalExit = process.exit;
    let callOrder: string[];
 
+   afterAll(async () => {
+      // performGracefulShutdownAfterDrain drains the package-load worker pool
+      // via getPackageLoadPool().shutdown() — which lazily CREATES the global
+      // singleton and leaves it installed in its shut-down state. In prod the
+      // process exits right after, so that's fine; in this shared test process
+      // it poisons every later spec that touches the worker path
+      // (Package.create → "PackageLoadPool is shutting down"), with failures
+      // that come and go with bun's platform-dependent file order. Reset the
+      // singleton so the next user lazily creates a fresh pool.
+      await __setPackageLoadPoolForTests(null);
+   });
+
    beforeEach(() => {
       callOrder = [];
 
@@ -41,6 +62,18 @@ describe("performGracefulShutdownAfterDrain: shutdown ordering", () => {
       process.exit = originalExit;
    });
 
+   // performGracefulShutdownAfterDrain lazily creates and shuts down the
+   // module-level PackageLoadPool singleton. If we leave that shut-down
+   // pool installed, sibling specs that run later in the same Bun process
+   // (e.g. package.spec.ts, package_race.spec.ts) inherit a dead pool and
+   // fail with "PackageLoadPool is shutting down". Reset the singleton so
+   // the next getPackageLoadPool() spins up a fresh one. The leaked-pool
+   // ordering is filesystem-readdir dependent, which is why it only
+   // surfaced on Linux CI.
+   afterAll(async () => {
+      await __setPackageLoadPoolForTests(null);
+   });
+
    const fakeServer = (): Server => ({ listening: false }) as unknown as Server;
 
    it("logs the 'Waiting ...' message before closing the logger", async () => {

package/src/instrumentation.ts

@@ -121,8 +121,20 @@ const httpRequestCount = meter.createCounter("http_server_requests_total", {
 // /health/liveness probe (a pure synchronous 200 handler) can fail under K8s,
 // so we surface p50/p99/max so an operator can correlate liveness restarts
 // with sustained event-loop pressure (large Malloy compiles, GC, etc.).
-const eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });
-eventLoopHistogram.enable();
+//
+// `monitorEventLoopDelay` is a Node-only API; some runtimes (e.g. Bun on
+// certain platforms, used by the integration test suite) don't implement it.
+// Degrade gracefully there rather than crashing module load: the metrics are
+// simply not registered.
+const eventLoopHistogram = (() => {
+   try {
+      const histogram = monitorEventLoopDelay({ resolution: 20 });
+      histogram.enable();
+      return histogram;
+   } catch {
+      return null;
+   }
+})();
 
 const eventLoopLagP50 = meter.createObservableGauge(
    "publisher_event_loop_lag_p50_ms",
@@ -150,21 +162,25 @@ const eventLoopLagMax = meter.createObservableGauge(
 );
 
 // Sample all three in one batch so the histogram reset can't race the reads.
-meter.addBatchObservableCallback(
-   (observableResult) => {
-      observableResult.observe(
-         eventLoopLagP50,
-         eventLoopHistogram.percentile(50) / 1e6,
-      );
-      observableResult.observe(
-         eventLoopLagP99,
-         eventLoopHistogram.percentile(99) / 1e6,
-      );
-      observableResult.observe(eventLoopLagMax, eventLoopHistogram.max / 1e6);
-      eventLoopHistogram.reset();
-   },
-   [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax],
-);
+// Skipped entirely when the runtime lacks event-loop-delay monitoring.
+if (eventLoopHistogram) {
+   const histogram = eventLoopHistogram;
+   meter.addBatchObservableCallback(
+      (observableResult) => {
+         observableResult.observe(
+            eventLoopLagP50,
+            histogram.percentile(50) / 1e6,
+         );
+         observableResult.observe(
+            eventLoopLagP99,
+            histogram.percentile(99) / 1e6,
+         );
+         observableResult.observe(eventLoopLagMax, histogram.max / 1e6);
+         histogram.reset();
+      },
+      [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax],
+   );
+}
 
 const IGNORED_PATHS = new Set([
    "/health",