@malloy-publisher/server 0.0.197 → 0.0.198-dev1

package/src/controller/package.controller.ts

@@ -1,6 +1,4 @@
-import * as path from "path";
 import { components } from "../api";
-import { PUBLISHER_DATA_DIR } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
 import { logger } from "../logger";
 import { EnvironmentStore } from "../service/environment_store";
@@ -37,15 +35,38 @@ export class PackageController {
          environmentName,
          false,
       );
-      const _package = await environment.getPackage(packageName, reload);
-      const packageLocation = _package.getPackageMetadata().location;
-      if (reload && packageLocation) {
-         await this.downloadPackage(
-            environmentName,
-            packageName,
-            packageLocation,
-         );
+
+      if (reload) {
+         // Resolve the package's source location from the currently-cached
+         // metadata WITHOUT triggering a stale-state reload. If a `location`
+         // is set, route the reload through `installPackage` so that
+         // download-then-load happens atomically; otherwise fall back to an
+         // in-place reload of the existing on-disk content.
+         let location: string | undefined;
+         try {
+            const cached = await environment.getPackage(packageName, false);
+            location = cached.getPackageMetadata().location;
+         } catch {
+            // Not previously loaded — nothing to reinstall from.
+         }
+         if (location) {
+            const reinstalled = await environment.installPackage(
+               packageName,
+               (stagingPath) =>
+                  this.downloadInto(
+                     environmentName,
+                     packageName,
+                     location,
+                     stagingPath,
+                  ),
+            );
+            return reinstalled.getPackageMetadata();
+         }
+         const _package = await environment.getPackage(packageName, true);
+         return _package.getPackageMetadata();
       }
+
+      const _package = await environment.getPackage(packageName, false);
       return _package.getPackageMetadata();
    }
 
@@ -60,21 +81,32 @@ export class PackageController {
       if (!body.name) {
          throw new BadRequestError("Package name is required");
       }
+      const packageName = body.name;
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
          false,
       );
+      let result;
       if (body.location) {
-         await this.downloadPackage(environmentName, body.name, body.location);
+         const bodyLocation = body.location;
+         result = await environment.installPackage(packageName, (stagingPath) =>
+            this.downloadInto(
+               environmentName,
+               packageName,
+               bodyLocation,
+               stagingPath,
+            ),
+         );
+      } else {
+         result = await environment.addPackage(packageName);
       }
-      const result = await environment.addPackage(body.name);
       await this.environmentStore.addPackageToDatabase(
          environmentName,
-         body.name,
+         packageName,
       );
 
       if (options?.autoLoadManifest === true) {
-         await this.tryLoadExistingManifest(environmentName, body.name);
+         await this.tryLoadExistingManifest(environmentName, packageName);
       }
 
       return result;
@@ -151,12 +183,20 @@ export class PackageController {
          false,
       );
       if (body.location) {
-         await this.downloadPackage(
-            environmentName,
-            packageName,
-            body.location,
+         // Re-install: stream the new content into a staging dir (no lock)
+         // and atomically swap it in (under the lock).
+         const bodyLocation = body.location;
+         await environment.installPackage(packageName, (stagingPath) =>
+            this.downloadInto(
+               environmentName,
+               packageName,
+               bodyLocation,
+               stagingPath,
+            ),
          );
       }
+      // Apply metadata changes (publisher.json) under the same per-package
+      // mutex via `Environment.updatePackage`.
       const result = await environment.updatePackage(packageName, body);
       await this.environmentStore.addPackageToDatabase(
          environmentName,
@@ -166,17 +206,18 @@ export class PackageController {
       return result;
    }
 
-   private async downloadPackage(
+   /**
+    * Run the right downloader for the given location into `targetPath`.
+    * This used to point at the canonical package directory, but the
+    * install pipeline now passes a sibling staging dir so the long-running
+    * download doesn't hold the per-package mutex.
+    */
+   private async downloadInto(
       environmentName: string,
       packageName: string,
       packageLocation: string,
+      targetPath: string,
    ) {
-      const absoluteTargetPath = path.join(
-         this.environmentStore.serverRootPath,
-         PUBLISHER_DATA_DIR,
-         environmentName,
-         packageName,
-      );
       const isCompressedFile = packageLocation.endsWith(".zip");
       if (
          packageLocation.startsWith("https://") ||
@@ -184,20 +225,20 @@ export class PackageController {
       ) {
          await this.environmentStore.downloadGitHubDirectory(
             packageLocation,
-            absoluteTargetPath,
+            targetPath,
          );
       } else if (packageLocation.startsWith("gs://")) {
          await this.environmentStore.downloadGcsDirectory(
             packageLocation,
             environmentName,
-            absoluteTargetPath,
+            targetPath,
             isCompressedFile,
          );
       } else if (packageLocation.startsWith("s3://")) {
          await this.environmentStore.downloadS3Directory(
             packageLocation,
             environmentName,
-            absoluteTargetPath,
+            targetPath,
             isCompressedFile,
          );
       }
@@ -207,7 +248,7 @@ export class PackageController {
          // so we need to mount them on the right place.
          await this.environmentStore.mountLocalDirectory(
             packageLocation,
-            absoluteTargetPath,
+            targetPath,
             environmentName,
             packageName,
          );

package/src/controller/query.controller.ts

@@ -4,6 +4,7 @@ import { API_PREFIX } from "../constants";
 import { ModelNotFoundError } from "../errors";
 import { EnvironmentStore } from "../service/environment_store";
 import type { FilterParams } from "../service/filter";
+import type { GivenValue } from "@malloydata/malloy";
 
 type ApiQuery = components["schemas"]["QueryResult"];
 
@@ -32,6 +33,7 @@ export class QueryController {
       compactJson: boolean = false,
       filterParams?: FilterParams,
       bypassFilters?: boolean,
+      givens?: Record<string, GivenValue>,
    ): Promise<ApiQuery> {
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
@@ -49,6 +51,7 @@ export class QueryController {
             query,
             filterParams,
             bypassFilters,
+            givens,
          );
          const renderLogs = validateRenderTags(result);
          return {

package/src/errors.ts

@@ -28,6 +28,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(409, error.message);
    } else if (error instanceof InvalidStateTransitionError) {
       return httpError(409, error.message);
+   } else if (error instanceof ServiceUnavailableError) {
+      return httpError(503, error.message);
    } else {
       return httpError(500, error.message);
    }
@@ -122,3 +124,14 @@ export class InvalidStateTransitionError extends Error {
       super(message);
    }
 }
+
+/**
+ * Thrown when the publisher is temporarily refusing a request to keep
+ * RSS under the configured `PUBLISHER_MAX_MEMORY_BYTES` cap. Mapped to
+ * HTTP 503 so an upstream proxy / client can retry with back-off.
+ */
+export class ServiceUnavailableError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}

package/src/health.spec.ts

@@ -0,0 +1,90 @@
+import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test";
+import { Server } from "http";
+import { performGracefulShutdownAfterDrain } from "./health";
+import { logger } from "./logger";
+
+// Regression test for the graceful-shutdown ordering bug that caused
+//   [winston] Attempt to write logs with no transports: {"message":"Waiting 50 seconds..."}
+// to appear in production logs. logger.close() must run after every
+// logger.* call, including the "Waiting ... seconds after server close
+// before exit..." message.
+//
+// Tests call performGracefulShutdownAfterDrain directly rather than
+// emitting SIGTERM, so module-level operationalState is not mutated
+// and the spec stays isolated from sibling tests in the same process.
+describe("performGracefulShutdownAfterDrain: shutdown ordering", () => {
+   const originalExit = process.exit;
+   let callOrder: string[];
+
+   beforeEach(() => {
+      callOrder = [];
+
+      spyOn(logger, "info").mockImplementation(((msg: string) => {
+         callOrder.push(`info:${msg}`);
+         return logger;
+      }) as never);
+      spyOn(logger, "close").mockImplementation((() => {
+         callOrder.push("close");
+         return logger;
+      }) as never);
+      // Silence warn/error calls so spec output stays clean. They are
+      // not load-bearing for these assertions.
+      spyOn(logger, "warn").mockImplementation((() => logger) as never);
+      spyOn(logger, "error").mockImplementation((() => logger) as never);
+
+      process.exit = ((_code?: number) => {
+         callOrder.push("exit");
+      }) as never;
+   });
+
+   afterEach(() => {
+      process.exit = originalExit;
+   });
+
+   const fakeServer = (): Server => ({ listening: false }) as unknown as Server;
+
+   it("logs the 'Waiting ...' message before closing the logger", async () => {
+      await performGracefulShutdownAfterDrain(fakeServer(), fakeServer(), 0.05);
+
+      const waitingIdx = callOrder.findIndex((entry) =>
+         entry.startsWith("info:Waiting"),
+      );
+      const closeIdx = callOrder.indexOf("close");
+      const exitIdx = callOrder.indexOf("exit");
+
+      expect(waitingIdx).toBeGreaterThanOrEqual(0);
+      expect(closeIdx).toBeGreaterThanOrEqual(0);
+      expect(exitIdx).toBeGreaterThanOrEqual(0);
+      expect(waitingIdx).toBeLessThan(closeIdx);
+      expect(closeIdx).toBeLessThan(exitIdx);
+   });
+
+   it("emits no logger.info calls after logger.close", async () => {
+      await performGracefulShutdownAfterDrain(fakeServer(), fakeServer(), 0.05);
+
+      const closeIdx = callOrder.indexOf("close");
+      const lateInfoIdx = callOrder.findIndex(
+         (entry, idx) => idx > closeIdx && entry.startsWith("info:"),
+      );
+      expect(closeIdx).toBeGreaterThanOrEqual(0);
+      expect(lateInfoIdx).toBe(-1);
+   });
+
+   it("closes the logger exactly once", async () => {
+      await performGracefulShutdownAfterDrain(fakeServer(), fakeServer(), 0.05);
+
+      const closes = callOrder.filter((entry) => entry === "close").length;
+      expect(closes).toBe(1);
+   });
+
+   it("skips the 'Waiting ...' message when gracefulCloseTimeoutSeconds is 0", async () => {
+      await performGracefulShutdownAfterDrain(fakeServer(), fakeServer(), 0);
+
+      const waitingCalls = callOrder.filter((entry) =>
+         entry.startsWith("info:Waiting"),
+      );
+      expect(waitingCalls.length).toBe(0);
+      expect(callOrder.indexOf("close")).toBeGreaterThanOrEqual(0);
+      expect(callOrder.indexOf("exit")).toBeGreaterThanOrEqual(0);
+   });
+});

package/src/health.ts

@@ -41,32 +41,6 @@ export function markReady(): void {
    }
 }
 
-/**
- * Marks the service as degraded: one or more environments failed to
- * initialize. The surviving environments are still queryable, and
- * callers polling /api/v0/status see operationalState="degraded" plus
- * a failedEnvironments list.
- *
- * Readiness probe (/health/readiness) returns 503 — degraded pods are
- * pulled out of K8s load-balancer rotation so traffic does not get
- * routed to a replica that can only serve a fraction of the configured
- * environments. Operators should fix the failing config and restart
- * the pod; if you want degraded traffic to be served anyway (e.g. for
- * a single-replica local dev instance), poll /api/v0/status directly
- * instead of /health/readiness.
- */
-export function markDegraded(): void {
-   if (operationalState !== "draining") {
-      operationalState = "degraded";
-      ready = false;
-      logger.warn(
-         "Service marked as degraded; one or more environments failed to initialize. Readiness probe will fail until the config is fixed and the process restarts.",
-      );
-   } else {
-      logger.error("Service is already draining - cannot mark as degraded");
-   }
-}
-
 /**
  * Marks the service as not ready (readiness probe will return 503).
  */
@@ -83,8 +57,8 @@ export function markNotReady(): void {
  * 2. Waits shutdownDrainDurationSeconds to allow in-flight requests to complete
  * 3. Sets preGracefulShutdownCompleted flag (enables drainingGuard middleware to reject new requests)
  * 4. Closes main server and MCP server (stops accepting new connections)
- * 5. Closes logger
- * 6. Waits shutdownGracefulCloseTimeoutSeconds (if > 0) for final cleanup
+ * 5. Waits shutdownGracefulCloseTimeoutSeconds (if > 0) for final cleanup
+ * 6. Closes logger (last, so any logs emitted during cleanup are flushed)
  * 7. Exits process
  *
  * Note: drainingGuard only rejects requests after step 3 completes. During step 2,
@@ -118,51 +92,92 @@ export function registerSignalHandlers(
          }, shutdownDrainDurationSeconds * 1000),
       );
 
-      const closeServer = (server: Server, name: string) =>
-         new Promise<void>((resolve) => {
-            if (server && server.listening) {
-               server.close((err) => {
-                  if (err) {
-                     logger.error(`${name} close error:`, err);
-                  } else {
-                     logger.info(`${name} closed`);
-                  }
-                  resolve();
-               });
-            } else {
-               resolve();
-            }
-         });
-
-      await Promise.all([
-         closeServer(server, "Main server"),
-         closeServer(mcpServer, "MCP server"),
-      ]);
-
-      try {
-         await shutdownSDK();
-         logger.info("OpenTelemetry SDK shut down");
-      } catch (_error) {
-         /* do nothing */
-      }
-
-      try {
-         logger.close();
-      } catch (_error) {
-         /* do nothing */
-      }
-
-      if (shutdownGracefulCloseTimeoutSeconds > 0) {
-         logger.info(
-            `Waiting ${shutdownGracefulCloseTimeoutSeconds} seconds after server close before exit...`,
-         );
-         await new Promise((resolve) =>
-            setTimeout(resolve, shutdownGracefulCloseTimeoutSeconds * 1000),
-         );
-      }
-      process.exit(0);
+      await performGracefulShutdownAfterDrain(
+         server,
+         mcpServer,
+         shutdownGracefulCloseTimeoutSeconds,
+      );
    });
 }
+
+/**
+ * Performs the post-drain shutdown work: closes both HTTP servers,
+ * shuts down the OpenTelemetry SDK, waits the optional graceful-close
+ * window so any in-flight cleanup can finish logging, closes the
+ * winston logger, and exits the process.
+ *
+ * Exported so unit tests can exercise the close + log + exit ordering
+ * without emitting SIGTERM (which would leave module-level
+ * operationalState stuck in "draining" and leak into sibling specs).
+ */
+export async function performGracefulShutdownAfterDrain(
+   server: Server,
+   mcpServer: Server,
+   shutdownGracefulCloseTimeoutSeconds: number,
+): Promise<void> {
+   const closeServer = (server: Server, name: string) =>
+      new Promise<void>((resolve) => {
+         if (server && server.listening) {
+            server.close((err) => {
+               if (err) {
+                  logger.error(`${name} close error:`, err);
+               } else {
+                  logger.info(`${name} closed`);
+               }
+               resolve();
+            });
+         } else {
+            resolve();
+         }
+      });
+
+   await Promise.all([
+      closeServer(server, "Main server"),
+      closeServer(mcpServer, "MCP server"),
+   ]);
+
+   try {
+      await shutdownSDK();
+      logger.info("OpenTelemetry SDK shut down");
+   } catch (_error) {
+      /* do nothing */
+   }
+
+   try {
+      // Drain in-flight compiles and terminate worker_threads before
+      // we exit so a slow compile doesn't leave orphan worker
+      // processes. Lazy-imported to avoid pulling the pool module
+      // into the health.ts dep graph for tests that don't exercise
+      // the compile path.
+      const { getCompilePool } = await import("./compile/compile_pool");
+      await getCompilePool().shutdown();
+      logger.info("Malloy compile worker pool shut down");
+   } catch (_error) {
+      /* do nothing */
+   }
+
+   if (shutdownGracefulCloseTimeoutSeconds > 0) {
+      logger.info(
+         `Waiting ${shutdownGracefulCloseTimeoutSeconds} seconds after server close before exit...`,
+      );
+      await new Promise((resolve) =>
+         setTimeout(resolve, shutdownGracefulCloseTimeoutSeconds * 1000),
+      );
+   }
+
+   // Close the logger last so anything emitted during the wait window
+   // above (or by other shutdown paths still running) reaches its
+   // transports. Closing earlier triggers winston's
+   // "Attempt to write logs with no transports" warning on any
+   // subsequent logger call.
+   try {
+      logger.close();
+   } catch (_error) {
+      /* do nothing */
+   }
+
+   process.exit(0);
+}
 /**
  * Middleware that returns 503 for non-health and metrics requests when service is draining.
  * Must be registered before application routes.

package/src/mcp/tools/discovery_tools.ts

@@ -222,8 +222,12 @@ export function registerTools(
                throw new Error(`Model not found: ${modelPath}`);
             }
 
-            // Use the new getModelFileText method
-            const fileText = await pkg.getModelFileText(modelPath);
+            // Route through the Environment so the disk read is serialized
+            // against installPackage / deletePackage.
+            const fileText = await environment.getModelFileText(
+               packageName,
+               modelPath,
+            );
 
             console.log(
                `[MCP LOG] Successfully retrieved model text for ${modelPath}`,

package/src/mcp/tools/execute_query_tool.ts

@@ -1,6 +1,7 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { ErrorCode, McpError } from "@modelcontextprotocol/sdk/types.js";
 import { z } from "zod";
+import type { GivenValue } from "@malloydata/malloy";
 import { logger } from "../../logger";
 import { EnvironmentStore } from "../../service/environment_store";
 import { getMalloyErrorDetails, type ErrorDetails } from "../error_messages";
@@ -30,6 +31,12 @@ const executeQueryShape = {
       .describe(
          "Filter parameter values keyed by filter name. Used with sources that declare #(filter) annotations.",
       ),
+   givens: z
+      .record(z.unknown())
+      .optional()
+      .describe(
+         "Per-query given values that override model defaults. Keys are given names declared in the model's given: block.",
+      ),
 };
 
 // Type inference is handled automatically by the MCP server based on the executeQueryShape
@@ -56,6 +63,7 @@ export function registerExecuteQueryTool(
             sourceName,
             queryName,
             filterParams,
+            givens,
          } = params;
 
          logger.info("[MCP Tool executeQuery] Received params:", { params });
@@ -128,6 +136,8 @@ export function registerExecuteQueryTool(
                   undefined,
                   query,
                   filterParams,
+                  undefined,
+                  givens as Record<string, GivenValue> | undefined,
                );
                const { validateRenderTags } = await import(
                   "@malloydata/render-validator"
@@ -174,6 +184,8 @@ export function registerExecuteQueryTool(
                   queryName,
                   undefined,
                   filterParams,
+                  undefined,
+                  givens as Record<string, GivenValue> | undefined,
                );
                const { validateRenderTags } = await import(
                   "@malloydata/render-validator"