@malloy-publisher/server 0.0.197 → 0.0.198-dev1

package/src/path_safety.spec.ts

@@ -0,0 +1,158 @@
+import { describe, expect, it } from "bun:test";
+import * as path from "path";
+
+import { BadRequestError } from "./errors";
+import {
+   assertSafeEnvironmentPath,
+   assertSafePackageName,
+   assertSafeRelativeModelPath,
+   safeJoinUnderRoot,
+} from "./path_safety";
+
+describe("assertSafePackageName", () => {
+   it.each([
+      "pkg",
+      "test_package",
+      "test-package",
+      "TestPackage1",
+      "test.package.name",
+      "a",
+      "x".repeat(255),
+   ])("accepts %p", (name) => {
+      expect(() => assertSafePackageName(name)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["dot", "."],
+      ["dot-dot", ".."],
+      ["leading dot", ".staging"],
+      ["forward slash", "foo/bar"],
+      ["backslash", "foo\\bar"],
+      ["null byte", "foo\0bar"],
+      ["traversal", "../etc/passwd"],
+      ["abs", "/etc/passwd"],
+      ["space", "my pkg"],
+      ["unicode", "pkg\u202E"],
+      ["too long", "x".repeat(256)],
+   ])("rejects %s (%p)", (_label, name) => {
+      expect(() => assertSafePackageName(name)).toThrow(BadRequestError);
+   });
+
+   it.each([
+      ["number", 42],
+      ["null", null],
+      ["undefined", undefined],
+      ["object", { name: "pkg" }],
+   ])("rejects non-string %s (%p)", (_label, value) => {
+      expect(() => assertSafePackageName(value)).toThrow(BadRequestError);
+   });
+});
+
+describe("assertSafeRelativeModelPath", () => {
+   it.each([
+      "model.malloy",
+      "models/foo.malloy",
+      "a/b/c/d.malloynb",
+      "deep/nested/file_name-1.malloy",
+   ])("accepts %p", (modelPath) => {
+      expect(() => assertSafeRelativeModelPath(modelPath)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["leading slash (absolute)", "/etc/passwd"],
+      ["traversal", "../etc/passwd"],
+      ["embedded traversal", "models/../../../etc/passwd"],
+      ["embedded dot segment", "models/./foo.malloy"],
+      ["double slash", "models//foo.malloy"],
+      ["trailing slash", "models/foo/"],
+      ["backslash", "models\\foo.malloy"],
+      ["null byte", "models/foo\0.malloy"],
+      ["dotfile segment", ".staging/foo.malloy"],
+      ["dotfile leaf", "models/.hidden.malloy"],
+   ])("rejects %s (%p)", (_label, modelPath) => {
+      expect(() => assertSafeRelativeModelPath(modelPath)).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("rejects non-string inputs", () => {
+      expect(() => assertSafeRelativeModelPath(undefined)).toThrow(
+         BadRequestError,
+      );
+      expect(() => assertSafeRelativeModelPath(123)).toThrow(BadRequestError);
+   });
+});
+
+describe("assertSafeEnvironmentPath", () => {
+   it.each([
+      "/etc/publisher",
+      "/var/lib/publisher/env1",
+      "/Users/me/data",
+      "/a",
+      "C:\\Users\\me\\publisher",
+      "C:/Users/me/publisher",
+   ])("accepts %p", (p) => {
+      expect(() => assertSafeEnvironmentPath(p)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["relative", "publisher/data"],
+      ["traversal in middle", "/var/lib/../../etc/passwd"],
+      ["traversal at end", "/var/lib/publisher/.."],
+      ["null byte", "/var/lib/publisher\0"],
+      ["bare dot-dot", ".."],
+      ["bare dot", "."],
+      ["too long", "/" + "a".repeat(5000)],
+   ])("rejects %s (%p)", (_label, p) => {
+      expect(() => assertSafeEnvironmentPath(p)).toThrow(BadRequestError);
+   });
+
+   it("rejects non-string inputs", () => {
+      expect(() => assertSafeEnvironmentPath(undefined)).toThrow(
+         BadRequestError,
+      );
+      expect(() => assertSafeEnvironmentPath(null)).toThrow(BadRequestError);
+      expect(() => assertSafeEnvironmentPath(42)).toThrow(BadRequestError);
+   });
+});
+
+describe("safeJoinUnderRoot", () => {
+   const root = "/tmp/test-root";
+
+   it("returns the resolved root when joined with no segments", () => {
+      expect(safeJoinUnderRoot(root)).toBe(path.resolve(root));
+   });
+
+   it("joins safe segments into a path under root", () => {
+      expect(safeJoinUnderRoot(root, "pkg", "model.malloy")).toBe(
+         path.resolve(root, "pkg", "model.malloy"),
+      );
+   });
+
+   it("throws when traversal escapes the root", () => {
+      expect(() => safeJoinUnderRoot(root, "..")).toThrow(BadRequestError);
+      expect(() => safeJoinUnderRoot(root, "..", "etc", "passwd")).toThrow(
+         BadRequestError,
+      );
+      expect(() => safeJoinUnderRoot(root, "pkg", "..", "..", "etc")).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("throws when an absolute segment overrides the root", () => {
+      expect(() => safeJoinUnderRoot(root, "/etc/passwd")).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("does NOT match a sibling directory with the same prefix", () => {
+      // path.resolve("/tmp/test-root", "../test-root-bad")  ->  "/tmp/test-root-bad"
+      // which starts with "/tmp/test-root" textually but is NOT a child.
+      expect(() => safeJoinUnderRoot(root, "..", "test-root-bad")).toThrow(
+         BadRequestError,
+      );
+   });
+});

package/src/path_safety.ts

@@ -0,0 +1,140 @@
+import * as path from "path";
+
+import { BadRequestError } from "./errors";
+
+/**
+ * Path-safety helpers used by `Environment` (and any other service that
+ * builds an on-disk path from request data) to defend against directory
+ * traversal. The intent is two-fold:
+ *
+ *  1. **Source-side allowlist**: `assertSafePackageName` /
+ *     `assertSafeRelativeModelPath` reject hostile inputs (`..`, leading
+ *     `/`, `\`, NUL, dotfiles) at the entry of every public service
+ *     method before any path-construction happens. These throw
+ *     `BadRequestError` so the controller layer's error mapper returns
+ *     HTTP 400.
+ *
+ *  2. **Sink-side containment**: `safeJoinUnderRoot` joins, resolves,
+ *     and verifies the result is strictly within the supplied root.
+ *     Even if a future caller forgets the source-side check, the sink
+ *     refuses to hand back an escaping path. This is the standard
+ *     "resolve-and-contain" pattern that CodeQL's `js/path-injection`
+ *     query recognises as a sanitizer.
+ */
+
+// Single path segment: ASCII letters, digits, `-`, `_`, `.`. No leading
+// `.` so internal sibling dirs (`.staging`, `.retired`) and editor /
+// VCS dirs can't be addressed by name from outside.
+const SAFE_NAME_RE = /^(?!\.\.?$)(?!\.)[A-Za-z0-9._-]{1,255}$/;
+
+const MAX_MODEL_PATH_LEN = 1024;
+
+// An environment path is server-controlled (config / disk-derived), but
+// CodeQL conservatively treats it as tainted because Express handlers on
+// the same class touch user input. The combined regex test +
+// `..` / NUL / length check at the constructor gate is the sanitizer
+// barrier the `js/path-injection` query recognises. Printable ASCII
+// only; absolute POSIX-or-Windows path; no `..`, no NUL.
+const SAFE_ENVIRONMENT_PATH_RE = /^(?:\/|[A-Za-z]:[\\/])[\x20-\x7E]*$/;
+const MAX_ENVIRONMENT_PATH_LEN = 4096;
+
+/**
+ * Reject anything that isn't a plausible single-segment package name.
+ * The allowlist is deliberately conservative — every existing test and
+ * production package name we've seen fits within it, and tightening
+ * here costs nothing.
+ */
+export function assertSafePackageName(packageName: unknown): void {
+   if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
+      throw new BadRequestError(
+         `Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`,
+      );
+   }
+}
+
+/**
+ * Reject anything that isn't a plausible *relative* path to a model
+ * file inside a package directory. Forward slashes are allowed (models
+ * live in subdirectories like `models/foo.malloy`); backslashes,
+ * absolute paths, NUL bytes, and `..` / `.` segments are not.
+ */
+export function assertSafeRelativeModelPath(modelPath: unknown): void {
+   if (
+      typeof modelPath !== "string" ||
+      modelPath.length === 0 ||
+      modelPath.length > MAX_MODEL_PATH_LEN ||
+      modelPath.includes("\0") ||
+      modelPath.includes("\\") ||
+      path.isAbsolute(modelPath) ||
+      modelPath.startsWith("/")
+   ) {
+      throw new BadRequestError(`Invalid model path`);
+   }
+
+   const segments = modelPath.split("/");
+   for (const segment of segments) {
+      if (segment === "" || segment === "." || segment === "..") {
+         throw new BadRequestError(`Invalid model path`);
+      }
+      if (segment.startsWith(".")) {
+         throw new BadRequestError(`Invalid model path`);
+      }
+   }
+}
+
+/**
+ * Reject anything that doesn't look like a server-controlled absolute
+ * filesystem path. Applied to `environmentPath` at the constructor
+ * gate so all downstream `path.join(this.environmentPath, …)` sites
+ * see a value that has cleared an allowlist check — the canonical
+ * sanitizer-barrier pattern CodeQL's `js/path-injection` query
+ * recognises.
+ */
+export function assertSafeEnvironmentPath(environmentPath: unknown): void {
+   if (typeof environmentPath !== "string") {
+      throw new BadRequestError(`Invalid environment path: must be a string`);
+   }
+   if (
+      environmentPath.length === 0 ||
+      environmentPath.length > MAX_ENVIRONMENT_PATH_LEN
+   ) {
+      throw new BadRequestError(`Invalid environment path: bad length`);
+   }
+   if (environmentPath.indexOf("\0") !== -1) {
+      throw new BadRequestError(`Invalid environment path: contains NUL byte`);
+   }
+   // Sanitizer barrier in the shape `x.indexOf("..") !== -1` that the
+   // CodeQL `js/path-injection` query recognises as a traversal guard.
+   if (environmentPath.indexOf("..") !== -1) {
+      throw new BadRequestError(
+         `Invalid environment path: contains ".." traversal segment`,
+      );
+   }
+   if (!SAFE_ENVIRONMENT_PATH_RE.test(environmentPath)) {
+      throw new BadRequestError(
+         `Invalid environment path: must be an absolute path of printable ASCII characters`,
+      );
+   }
+}
+
+/**
+ * Resolve `path.join(root, ...segments)` and verify the result lives
+ * strictly inside `root` (or is `root` itself). Throws
+ * `BadRequestError` if the resolved path escapes the root via `..`,
+ * absolute segments, or symlink-style trickery in the input.
+ *
+ * Callers should still run `assertSafePackageName` / similar on
+ * user-controlled segments first — this helper is the second line of
+ * defense, not the first.
+ */
+export function safeJoinUnderRoot(root: string, ...segments: string[]): string {
+   const resolvedRoot = path.resolve(root);
+   const joined = path.resolve(resolvedRoot, ...segments);
+   const rootWithSep = resolvedRoot.endsWith(path.sep)
+      ? resolvedRoot
+      : resolvedRoot + path.sep;
+   if (joined !== resolvedRoot && !joined.startsWith(rootWithSep)) {
+      throw new BadRequestError(`Resolved path is outside of root`);
+   }
+   return joined;
+}

package/src/server.ts

@@ -1,4 +1,5 @@
 // Pre-load the instrumentation module; the instrumentation module must be loaded before the other imports.
+import type { GivenValue } from "@malloydata/malloy";
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
 import bodyParser from "body-parser";
 import cors from "cors";
@@ -33,6 +34,7 @@ import {
 } from "./instrumentation";
 import { logger, loggerMiddleware } from "./logger";
 
+import { getMemoryGovernorConfig } from "./config";
 import { ManifestController } from "./controller/manifest.controller";
 import { MaterializationController } from "./controller/materialization.controller";
 import { initializeMcpServer } from "./mcp/server";
@@ -40,6 +42,7 @@ import { registerLegacyRoutes } from "./server-old";
 import { EnvironmentStore } from "./service/environment_store";
 import { ManifestService } from "./service/manifest_service";
 import { MaterializationService } from "./service/materialization_service";
+import { PackageMemoryGovernor } from "./service/package_memory_governor";
 
 /** Normalize an Express query param into a string[] or undefined. */
 export function normalizeQueryArray(value: unknown): string[] | undefined {
@@ -158,6 +161,17 @@ const manifestService = new ManifestService(environmentStore);
 const watchModeController = new WatchModeController(environmentStore);
 const connectionController = new ConnectionController(environmentStore);
 const modelController = new ModelController(environmentStore);
+// PackageMemoryGovernor is opt-in via PUBLISHER_MAX_MEMORY_BYTES.
+// When set, it polls process RSS and flips an `isBackpressured` flag
+// that Environment.getPackage / addPackage consult before allocating
+// any new package — the server responds with HTTP 503 instead of
+// OOM-killing the pod.
+const memoryGovernorConfig = getMemoryGovernorConfig();
+const memoryGovernor = memoryGovernorConfig
+   ? new PackageMemoryGovernor(memoryGovernorConfig)
+   : null;
+memoryGovernor?.start();
+environmentStore.setMemoryGovernor(memoryGovernor);
 const packageController = new PackageController(
    environmentStore,
    manifestService,
@@ -1097,6 +1111,18 @@ app.get(
          const bypassFilters =
             req.query.bypass_filters === "true" ? true : undefined;
 
+         let givens: Record<string, GivenValue> | undefined;
+         if (typeof req.query.givens === "string") {
+            try {
+               givens = JSON.parse(req.query.givens);
+            } catch {
+               res.status(400).json({
+                  error: "Invalid givens: must be valid JSON",
+               });
+               return;
+            }
+         }
+
          res.status(200).json(
             await modelController.executeNotebookCell(
                req.params.environmentName,
@@ -1105,6 +1131,7 @@ app.get(
                cellIndex,
                filterParams,
                bypassFilters,
+               givens,
             ),
          );
       } catch (error) {
@@ -1165,6 +1192,7 @@ app.post(
                   | Record<string, string | string[]>
                   | undefined,
                req.body.bypassFilters === true ? true : undefined,
+               req.body.givens as Record<string, GivenValue> | undefined,
             ),
          );
       } catch (error) {
@@ -1208,6 +1236,7 @@ app.post(
             req.params.modelName,
             req.body.source,
             req.body.includeSql === true,
+            req.body.givens as Record<string, GivenValue> | undefined,
          );
          res.status(200).json(result);
       } catch (error) {