@malloy-publisher/server 0.0.197 → 0.0.198

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.197",
+   "version": "0.0.198",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/config.spec.ts

@@ -1081,3 +1081,84 @@ describe("Config path resolution (--config and bundled default)", () => {
       expect(result.environments).toEqual([]);
    });
 });
+
+describe("getMemoryGovernorConfig", () => {
+   const GOVERNOR_ENV_VARS = [
+      "PUBLISHER_MAX_MEMORY_BYTES",
+      "PUBLISHER_MEMORY_HIGH_WATER_FRACTION",
+      "PUBLISHER_MEMORY_LOW_WATER_FRACTION",
+      "PUBLISHER_MEMORY_CHECK_INTERVAL_MS",
+      "PUBLISHER_MEMORY_BACKPRESSURE",
+   ];
+
+   beforeEach(() => {
+      for (const v of GOVERNOR_ENV_VARS) delete process.env[v];
+   });
+   afterEach(() => {
+      for (const v of GOVERNOR_ENV_VARS) delete process.env[v];
+   });
+
+   it("returns null when PUBLISHER_MAX_MEMORY_BYTES is unset", async () => {
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(getMemoryGovernorConfig()).toBeNull();
+   });
+
+   it("parses defaults when only PUBLISHER_MAX_MEMORY_BYTES is set", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = String(2 * 1024 * 1024 * 1024);
+      const { getMemoryGovernorConfig } = await import("./config");
+      const cfg = getMemoryGovernorConfig();
+      expect(cfg).not.toBeNull();
+      expect(cfg!.maxMemoryBytes).toBe(2 * 1024 * 1024 * 1024);
+      expect(cfg!.backpressureEnabled).toBe(true);
+      expect(cfg!.highWaterFraction).toBeGreaterThan(cfg!.lowWaterFraction);
+   });
+
+   it("honours fraction and interval overrides", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "1000000000";
+      process.env.PUBLISHER_MEMORY_HIGH_WATER_FRACTION = "0.85";
+      process.env.PUBLISHER_MEMORY_LOW_WATER_FRACTION = "0.7";
+      process.env.PUBLISHER_MEMORY_CHECK_INTERVAL_MS = "10000";
+      process.env.PUBLISHER_MEMORY_BACKPRESSURE = "false";
+      const { getMemoryGovernorConfig } = await import("./config");
+      const cfg = getMemoryGovernorConfig();
+      expect(cfg).not.toBeNull();
+      expect(cfg!.highWaterFraction).toBe(0.85);
+      expect(cfg!.lowWaterFraction).toBe(0.7);
+      expect(cfg!.checkIntervalMs).toBe(10000);
+      expect(cfg!.backpressureEnabled).toBe(false);
+   });
+
+   it("treats PUBLISHER_MAX_MEMORY_BYTES=0 as disabled (returns null)", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "0";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(getMemoryGovernorConfig()).toBeNull();
+   });
+
+   it("rejects a negative PUBLISHER_MAX_MEMORY_BYTES", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "-1";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(() => getMemoryGovernorConfig()).toThrow();
+   });
+
+   it("rejects low >= high", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "1000000000";
+      process.env.PUBLISHER_MEMORY_HIGH_WATER_FRACTION = "0.7";
+      process.env.PUBLISHER_MEMORY_LOW_WATER_FRACTION = "0.8";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(() => getMemoryGovernorConfig()).toThrow();
+   });
+
+   it("rejects an out-of-range fraction", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "1000000000";
+      process.env.PUBLISHER_MEMORY_HIGH_WATER_FRACTION = "1.5";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(() => getMemoryGovernorConfig()).toThrow();
+   });
+
+   it("rejects a check interval below the safety floor", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "1000000000";
+      process.env.PUBLISHER_MEMORY_CHECK_INTERVAL_MS = "10";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(() => getMemoryGovernorConfig()).toThrow();
+   });
+});

package/src/config.ts

@@ -99,6 +99,132 @@ export type ProcessedPublisherConfig = {
    environments: ProcessedEnvironment[];
 };
 
+/**
+ * Tunables for {@link PackageMemoryGovernor}. All values are sourced
+ * from environment variables at startup; see {@link getMemoryGovernorConfig}
+ * for parsing and defaults.
+ *
+ * The governor is admission control only: it polls process RSS on
+ * `checkIntervalMs` and toggles a single `isBackpressured` flag using
+ * a low/high-water hysteresis band. It does NOT evict, unload, or
+ * interrupt already-loaded packages — recovery is left to the kernel
+ * reclaiming pages as in-flight traffic completes.
+ */
+export interface MemoryGovernorConfig {
+   /** Hard ceiling for process RSS in bytes (the OOM-relevant figure). */
+   maxMemoryBytes: number;
+   /** Fraction of `maxMemoryBytes` at which the governor activates back-pressure (new package loads start returning HTTP 503). Must be in (0, 1) and strictly greater than `lowWaterFraction`. */
+   highWaterFraction: number;
+   /** Fraction of `maxMemoryBytes` at which the governor clears back-pressure (new package loads admitted again). Must be in (0, 1) and strictly less than `highWaterFraction`; the gap is the hysteresis band that prevents flap. */
+   lowWaterFraction: number;
+   /** Polling cadence for the RSS sampler, in milliseconds. */
+   checkIntervalMs: number;
+   /** When true, RSS crossings flip the back-pressure flag. When false, the governor still samples and emits metrics but never rejects requests — useful for a monitoring-only rollout before enabling the 503 behaviour. */
+   backpressureEnabled: boolean;
+}
+
+const DEFAULT_HIGH_WATER_FRACTION = 0.8;
+const DEFAULT_LOW_WATER_FRACTION = 0.7;
+const DEFAULT_CHECK_INTERVAL_MS = 5_000;
+const MIN_CHECK_INTERVAL_MS = 100;
+
+function parseIntEnv(name: string): number | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const value = Number.parseInt(raw, 10);
+   if (!Number.isFinite(value) || String(value) !== raw.trim()) {
+      throw new Error(
+         `Invalid value for ${name}: expected a base-10 integer, got "${raw}"`,
+      );
+   }
+   return value;
+}
+
+function parseFloatEnv(name: string): number | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const value = Number.parseFloat(raw);
+   if (!Number.isFinite(value)) {
+      throw new Error(
+         `Invalid value for ${name}: expected a finite number, got "${raw}"`,
+      );
+   }
+   return value;
+}
+
+function parseBoolEnv(name: string): boolean | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const normalised = raw.trim().toLowerCase();
+   if (["1", "true", "yes", "on"].includes(normalised)) return true;
+   if (["0", "false", "no", "off"].includes(normalised)) return false;
+   throw new Error(
+      `Invalid value for ${name}: expected a boolean (true/false), got "${raw}"`,
+   );
+}
+
+/**
+ * Parse memory-governor settings from environment variables and return
+ * either a fully-validated config or `null` when the feature is
+ * disabled. The feature is disabled iff `PUBLISHER_MAX_MEMORY_BYTES`
+ * is unset or set to `0`.
+ *
+ * Throws at startup on malformed input so a typo in a k8s manifest
+ * surfaces as a loud failure rather than silently disabling the cap.
+ */
+export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
+   const maxMemoryBytes = parseIntEnv("PUBLISHER_MAX_MEMORY_BYTES");
+   if (maxMemoryBytes === undefined || maxMemoryBytes === 0) {
+      return null;
+   }
+   if (maxMemoryBytes < 0) {
+      throw new Error(
+         `PUBLISHER_MAX_MEMORY_BYTES must be a positive integer (got ${maxMemoryBytes})`,
+      );
+   }
+
+   const highWaterFraction =
+      parseFloatEnv("PUBLISHER_MEMORY_HIGH_WATER_FRACTION") ??
+      DEFAULT_HIGH_WATER_FRACTION;
+   const lowWaterFraction =
+      parseFloatEnv("PUBLISHER_MEMORY_LOW_WATER_FRACTION") ??
+      DEFAULT_LOW_WATER_FRACTION;
+   const checkIntervalMs =
+      parseIntEnv("PUBLISHER_MEMORY_CHECK_INTERVAL_MS") ??
+      DEFAULT_CHECK_INTERVAL_MS;
+   const backpressureEnabled =
+      parseBoolEnv("PUBLISHER_MEMORY_BACKPRESSURE") ?? true;
+
+   if (highWaterFraction <= 0 || highWaterFraction >= 1) {
+      throw new Error(
+         `PUBLISHER_MEMORY_HIGH_WATER_FRACTION must be in (0, 1) (got ${highWaterFraction})`,
+      );
+   }
+   if (lowWaterFraction <= 0 || lowWaterFraction >= 1) {
+      throw new Error(
+         `PUBLISHER_MEMORY_LOW_WATER_FRACTION must be in (0, 1) (got ${lowWaterFraction})`,
+      );
+   }
+   if (lowWaterFraction >= highWaterFraction) {
+      throw new Error(
+         `PUBLISHER_MEMORY_LOW_WATER_FRACTION (${lowWaterFraction}) must be strictly less than PUBLISHER_MEMORY_HIGH_WATER_FRACTION (${highWaterFraction})`,
+      );
+   }
+   if (checkIntervalMs < MIN_CHECK_INTERVAL_MS) {
+      throw new Error(
+         `PUBLISHER_MEMORY_CHECK_INTERVAL_MS must be >= ${MIN_CHECK_INTERVAL_MS} (got ${checkIntervalMs})`,
+      );
+   }
+
+   return {
+      maxMemoryBytes,
+      highWaterFraction,
+      lowWaterFraction,
+      checkIntervalMs,
+      backpressureEnabled,
+   };
+};
+
 function substituteEnvVars(value: string): string {
    const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
 

package/src/controller/package.controller.ts

@@ -1,6 +1,4 @@
-import * as path from "path";
 import { components } from "../api";
-import { PUBLISHER_DATA_DIR } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
 import { logger } from "../logger";
 import { EnvironmentStore } from "../service/environment_store";
@@ -37,15 +35,38 @@ export class PackageController {
          environmentName,
          false,
       );
-      const _package = await environment.getPackage(packageName, reload);
-      const packageLocation = _package.getPackageMetadata().location;
-      if (reload && packageLocation) {
-         await this.downloadPackage(
-            environmentName,
-            packageName,
-            packageLocation,
-         );
+
+      if (reload) {
+         // Resolve the package's source location from the currently-cached
+         // metadata WITHOUT triggering a stale-state reload. If a `location`
+         // is set, route the reload through `installPackage` so that
+         // download-then-load happens atomically; otherwise fall back to an
+         // in-place reload of the existing on-disk content.
+         let location: string | undefined;
+         try {
+            const cached = await environment.getPackage(packageName, false);
+            location = cached.getPackageMetadata().location;
+         } catch {
+            // Not previously loaded — nothing to reinstall from.
+         }
+         if (location) {
+            const reinstalled = await environment.installPackage(
+               packageName,
+               (stagingPath) =>
+                  this.downloadInto(
+                     environmentName,
+                     packageName,
+                     location,
+                     stagingPath,
+                  ),
+            );
+            return reinstalled.getPackageMetadata();
+         }
+         const _package = await environment.getPackage(packageName, true);
+         return _package.getPackageMetadata();
       }
+
+      const _package = await environment.getPackage(packageName, false);
       return _package.getPackageMetadata();
    }
 
@@ -60,21 +81,32 @@ export class PackageController {
       if (!body.name) {
          throw new BadRequestError("Package name is required");
       }
+      const packageName = body.name;
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
          false,
       );
+      let result;
       if (body.location) {
-         await this.downloadPackage(environmentName, body.name, body.location);
+         const bodyLocation = body.location;
+         result = await environment.installPackage(packageName, (stagingPath) =>
+            this.downloadInto(
+               environmentName,
+               packageName,
+               bodyLocation,
+               stagingPath,
+            ),
+         );
+      } else {
+         result = await environment.addPackage(packageName);
       }
-      const result = await environment.addPackage(body.name);
       await this.environmentStore.addPackageToDatabase(
          environmentName,
-         body.name,
+         packageName,
       );
 
       if (options?.autoLoadManifest === true) {
-         await this.tryLoadExistingManifest(environmentName, body.name);
+         await this.tryLoadExistingManifest(environmentName, packageName);
       }
 
       return result;
@@ -151,12 +183,20 @@ export class PackageController {
          false,
       );
       if (body.location) {
-         await this.downloadPackage(
-            environmentName,
-            packageName,
-            body.location,
+         // Re-install: stream the new content into a staging dir (no lock)
+         // and atomically swap it in (under the lock).
+         const bodyLocation = body.location;
+         await environment.installPackage(packageName, (stagingPath) =>
+            this.downloadInto(
+               environmentName,
+               packageName,
+               bodyLocation,
+               stagingPath,
+            ),
          );
       }
+      // Apply metadata changes (publisher.json) under the same per-package
+      // mutex via `Environment.updatePackage`.
       const result = await environment.updatePackage(packageName, body);
       await this.environmentStore.addPackageToDatabase(
          environmentName,
@@ -166,17 +206,18 @@ export class PackageController {
       return result;
    }
 
-   private async downloadPackage(
+   /**
+    * Run the right downloader for the given location into `targetPath`.
+    * This used to point at the canonical package directory, but the
+    * install pipeline now passes a sibling staging dir so the long-running
+    * download doesn't hold the per-package mutex.
+    */
+   private async downloadInto(
       environmentName: string,
       packageName: string,
       packageLocation: string,
+      targetPath: string,
    ) {
-      const absoluteTargetPath = path.join(
-         this.environmentStore.serverRootPath,
-         PUBLISHER_DATA_DIR,
-         environmentName,
-         packageName,
-      );
       const isCompressedFile = packageLocation.endsWith(".zip");
       if (
          packageLocation.startsWith("https://") ||
@@ -184,20 +225,20 @@ export class PackageController {
       ) {
          await this.environmentStore.downloadGitHubDirectory(
             packageLocation,
-            absoluteTargetPath,
+            targetPath,
          );
       } else if (packageLocation.startsWith("gs://")) {
          await this.environmentStore.downloadGcsDirectory(
             packageLocation,
             environmentName,
-            absoluteTargetPath,
+            targetPath,
             isCompressedFile,
          );
       } else if (packageLocation.startsWith("s3://")) {
          await this.environmentStore.downloadS3Directory(
             packageLocation,
             environmentName,
-            absoluteTargetPath,
+            targetPath,
             isCompressedFile,
          );
       }
@@ -207,7 +248,7 @@ export class PackageController {
          // so we need to mount them on the right place.
          await this.environmentStore.mountLocalDirectory(
             packageLocation,
-            absoluteTargetPath,
+            targetPath,
             environmentName,
             packageName,
          );

package/src/errors.ts

@@ -28,6 +28,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(409, error.message);
    } else if (error instanceof InvalidStateTransitionError) {
       return httpError(409, error.message);
+   } else if (error instanceof ServiceUnavailableError) {
+      return httpError(503, error.message);
    } else {
       return httpError(500, error.message);
    }
@@ -122,3 +124,14 @@ export class InvalidStateTransitionError extends Error {
       super(message);
    }
 }
+
+/**
+ * Thrown when the publisher is temporarily refusing a request to keep
+ * RSS under the configured `PUBLISHER_MAX_MEMORY_BYTES` cap. Mapped to
+ * HTTP 503 so an upstream proxy / client can retry with back-off.
+ */
+export class ServiceUnavailableError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}

package/src/health.ts

@@ -41,32 +41,6 @@ export function markReady(): void {
    }
 }
 
-/**
- * Marks the service as degraded: one or more environments failed to
- * initialize. The surviving environments are still queryable, and
- * callers polling /api/v0/status see operationalState="degraded" plus
- * a failedEnvironments list.
- *
- * Readiness probe (/health/readiness) returns 503 — degraded pods are
- * pulled out of K8s load-balancer rotation so traffic does not get
- * routed to a replica that can only serve a fraction of the configured
- * environments. Operators should fix the failing config and restart
- * the pod; if you want degraded traffic to be served anyway (e.g. for
- * a single-replica local dev instance), poll /api/v0/status directly
- * instead of /health/readiness.
- */
-export function markDegraded(): void {
-   if (operationalState !== "draining") {
-      operationalState = "degraded";
-      ready = false;
-      logger.warn(
-         "Service marked as degraded; one or more environments failed to initialize. Readiness probe will fail until the config is fixed and the process restarts.",
-      );
-   } else {
-      logger.error("Service is already draining - cannot mark as degraded");
-   }
-}
-
 /**
  * Marks the service as not ready (readiness probe will return 503).
  */

package/src/mcp/tools/discovery_tools.ts

@@ -222,8 +222,12 @@ export function registerTools(
                throw new Error(`Model not found: ${modelPath}`);
             }
 
-            // Use the new getModelFileText method
-            const fileText = await pkg.getModelFileText(modelPath);
+            // Route through the Environment so the disk read is serialized
+            // against installPackage / deletePackage.
+            const fileText = await environment.getModelFileText(
+               packageName,
+               modelPath,
+            );
 
             console.log(
                `[MCP LOG] Successfully retrieved model text for ${modelPath}`,

package/src/path_safety.spec.ts

@@ -0,0 +1,158 @@
+import { describe, expect, it } from "bun:test";
+import * as path from "path";
+
+import { BadRequestError } from "./errors";
+import {
+   assertSafeEnvironmentPath,
+   assertSafePackageName,
+   assertSafeRelativeModelPath,
+   safeJoinUnderRoot,
+} from "./path_safety";
+
+describe("assertSafePackageName", () => {
+   it.each([
+      "pkg",
+      "test_package",
+      "test-package",
+      "TestPackage1",
+      "test.package.name",
+      "a",
+      "x".repeat(255),
+   ])("accepts %p", (name) => {
+      expect(() => assertSafePackageName(name)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["dot", "."],
+      ["dot-dot", ".."],
+      ["leading dot", ".staging"],
+      ["forward slash", "foo/bar"],
+      ["backslash", "foo\\bar"],
+      ["null byte", "foo\0bar"],
+      ["traversal", "../etc/passwd"],
+      ["abs", "/etc/passwd"],
+      ["space", "my pkg"],
+      ["unicode", "pkg\u202E"],
+      ["too long", "x".repeat(256)],
+   ])("rejects %s (%p)", (_label, name) => {
+      expect(() => assertSafePackageName(name)).toThrow(BadRequestError);
+   });
+
+   it.each([
+      ["number", 42],
+      ["null", null],
+      ["undefined", undefined],
+      ["object", { name: "pkg" }],
+   ])("rejects non-string %s (%p)", (_label, value) => {
+      expect(() => assertSafePackageName(value)).toThrow(BadRequestError);
+   });
+});
+
+describe("assertSafeRelativeModelPath", () => {
+   it.each([
+      "model.malloy",
+      "models/foo.malloy",
+      "a/b/c/d.malloynb",
+      "deep/nested/file_name-1.malloy",
+   ])("accepts %p", (modelPath) => {
+      expect(() => assertSafeRelativeModelPath(modelPath)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["leading slash (absolute)", "/etc/passwd"],
+      ["traversal", "../etc/passwd"],
+      ["embedded traversal", "models/../../../etc/passwd"],
+      ["embedded dot segment", "models/./foo.malloy"],
+      ["double slash", "models//foo.malloy"],
+      ["trailing slash", "models/foo/"],
+      ["backslash", "models\\foo.malloy"],
+      ["null byte", "models/foo\0.malloy"],
+      ["dotfile segment", ".staging/foo.malloy"],
+      ["dotfile leaf", "models/.hidden.malloy"],
+   ])("rejects %s (%p)", (_label, modelPath) => {
+      expect(() => assertSafeRelativeModelPath(modelPath)).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("rejects non-string inputs", () => {
+      expect(() => assertSafeRelativeModelPath(undefined)).toThrow(
+         BadRequestError,
+      );
+      expect(() => assertSafeRelativeModelPath(123)).toThrow(BadRequestError);
+   });
+});
+
+describe("assertSafeEnvironmentPath", () => {
+   it.each([
+      "/etc/publisher",
+      "/var/lib/publisher/env1",
+      "/Users/me/data",
+      "/a",
+      "C:\\Users\\me\\publisher",
+      "C:/Users/me/publisher",
+   ])("accepts %p", (p) => {
+      expect(() => assertSafeEnvironmentPath(p)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["relative", "publisher/data"],
+      ["traversal in middle", "/var/lib/../../etc/passwd"],
+      ["traversal at end", "/var/lib/publisher/.."],
+      ["null byte", "/var/lib/publisher\0"],
+      ["bare dot-dot", ".."],
+      ["bare dot", "."],
+      ["too long", "/" + "a".repeat(5000)],
+   ])("rejects %s (%p)", (_label, p) => {
+      expect(() => assertSafeEnvironmentPath(p)).toThrow(BadRequestError);
+   });
+
+   it("rejects non-string inputs", () => {
+      expect(() => assertSafeEnvironmentPath(undefined)).toThrow(
+         BadRequestError,
+      );
+      expect(() => assertSafeEnvironmentPath(null)).toThrow(BadRequestError);
+      expect(() => assertSafeEnvironmentPath(42)).toThrow(BadRequestError);
+   });
+});
+
+describe("safeJoinUnderRoot", () => {
+   const root = "/tmp/test-root";
+
+   it("returns the resolved root when joined with no segments", () => {
+      expect(safeJoinUnderRoot(root)).toBe(path.resolve(root));
+   });
+
+   it("joins safe segments into a path under root", () => {
+      expect(safeJoinUnderRoot(root, "pkg", "model.malloy")).toBe(
+         path.resolve(root, "pkg", "model.malloy"),
+      );
+   });
+
+   it("throws when traversal escapes the root", () => {
+      expect(() => safeJoinUnderRoot(root, "..")).toThrow(BadRequestError);
+      expect(() => safeJoinUnderRoot(root, "..", "etc", "passwd")).toThrow(
+         BadRequestError,
+      );
+      expect(() => safeJoinUnderRoot(root, "pkg", "..", "..", "etc")).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("throws when an absolute segment overrides the root", () => {
+      expect(() => safeJoinUnderRoot(root, "/etc/passwd")).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("does NOT match a sibling directory with the same prefix", () => {
+      // path.resolve("/tmp/test-root", "../test-root-bad")  ->  "/tmp/test-root-bad"
+      // which starts with "/tmp/test-root" textually but is NOT a child.
+      expect(() => safeJoinUnderRoot(root, "..", "test-root-bad")).toThrow(
+         BadRequestError,
+      );
+   });
+});