@malloy-publisher/server 0.0.198-dev → 0.0.198

package/src/controller/connection.controller.ts

@@ -91,7 +91,7 @@ function validateAdminAuthoredConnection(
 ): void {
    if (connectionName === "duckdb" || connectionConfig.name === "duckdb") {
       throw new BadRequestError(
-         "DuckDB connection name cannot be 'duckdb'; it is reserved for Publisher package sandboxes.",
+         "Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').",
       );
    }
 

package/src/controller/package.controller.ts

@@ -1,6 +1,4 @@
-import * as path from "path";
 import { components } from "../api";
-import { PUBLISHER_DATA_DIR } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
 import { logger } from "../logger";
 import { EnvironmentStore } from "../service/environment_store";
@@ -37,15 +35,38 @@ export class PackageController {
          environmentName,
          false,
       );
-      const _package = await environment.getPackage(packageName, reload);
-      const packageLocation = _package.getPackageMetadata().location;
-      if (reload && packageLocation) {
-         await this.downloadPackage(
-            environmentName,
-            packageName,
-            packageLocation,
-         );
+
+      if (reload) {
+         // Resolve the package's source location from the currently-cached
+         // metadata WITHOUT triggering a stale-state reload. If a `location`
+         // is set, route the reload through `installPackage` so that
+         // download-then-load happens atomically; otherwise fall back to an
+         // in-place reload of the existing on-disk content.
+         let location: string | undefined;
+         try {
+            const cached = await environment.getPackage(packageName, false);
+            location = cached.getPackageMetadata().location;
+         } catch {
+            // Not previously loaded — nothing to reinstall from.
+         }
+         if (location) {
+            const reinstalled = await environment.installPackage(
+               packageName,
+               (stagingPath) =>
+                  this.downloadInto(
+                     environmentName,
+                     packageName,
+                     location,
+                     stagingPath,
+                  ),
+            );
+            return reinstalled.getPackageMetadata();
+         }
+         const _package = await environment.getPackage(packageName, true);
+         return _package.getPackageMetadata();
       }
+
+      const _package = await environment.getPackage(packageName, false);
       return _package.getPackageMetadata();
    }
 
@@ -60,21 +81,32 @@ export class PackageController {
       if (!body.name) {
          throw new BadRequestError("Package name is required");
       }
+      const packageName = body.name;
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
          false,
       );
+      let result;
       if (body.location) {
-         await this.downloadPackage(environmentName, body.name, body.location);
+         const bodyLocation = body.location;
+         result = await environment.installPackage(packageName, (stagingPath) =>
+            this.downloadInto(
+               environmentName,
+               packageName,
+               bodyLocation,
+               stagingPath,
+            ),
+         );
+      } else {
+         result = await environment.addPackage(packageName);
       }
-      const result = await environment.addPackage(body.name);
       await this.environmentStore.addPackageToDatabase(
          environmentName,
-         body.name,
+         packageName,
       );
 
       if (options?.autoLoadManifest === true) {
-         await this.tryLoadExistingManifest(environmentName, body.name);
+         await this.tryLoadExistingManifest(environmentName, packageName);
       }
 
       return result;
@@ -151,12 +183,20 @@ export class PackageController {
          false,
       );
       if (body.location) {
-         await this.downloadPackage(
-            environmentName,
-            packageName,
-            body.location,
+         // Re-install: stream the new content into a staging dir (no lock)
+         // and atomically swap it in (under the lock).
+         const bodyLocation = body.location;
+         await environment.installPackage(packageName, (stagingPath) =>
+            this.downloadInto(
+               environmentName,
+               packageName,
+               bodyLocation,
+               stagingPath,
+            ),
          );
       }
+      // Apply metadata changes (publisher.json) under the same per-package
+      // mutex via `Environment.updatePackage`.
       const result = await environment.updatePackage(packageName, body);
       await this.environmentStore.addPackageToDatabase(
          environmentName,
@@ -166,17 +206,18 @@ export class PackageController {
       return result;
    }
 
-   private async downloadPackage(
+   /**
+    * Run the right downloader for the given location into `targetPath`.
+    * This used to point at the canonical package directory, but the
+    * install pipeline now passes a sibling staging dir so the long-running
+    * download doesn't hold the per-package mutex.
+    */
+   private async downloadInto(
       environmentName: string,
       packageName: string,
       packageLocation: string,
+      targetPath: string,
    ) {
-      const absoluteTargetPath = path.join(
-         this.environmentStore.serverRootPath,
-         PUBLISHER_DATA_DIR,
-         environmentName,
-         packageName,
-      );
       const isCompressedFile = packageLocation.endsWith(".zip");
       if (
          packageLocation.startsWith("https://") ||
@@ -184,20 +225,20 @@ export class PackageController {
       ) {
          await this.environmentStore.downloadGitHubDirectory(
             packageLocation,
-            absoluteTargetPath,
+            targetPath,
          );
       } else if (packageLocation.startsWith("gs://")) {
          await this.environmentStore.downloadGcsDirectory(
             packageLocation,
             environmentName,
-            absoluteTargetPath,
+            targetPath,
             isCompressedFile,
          );
       } else if (packageLocation.startsWith("s3://")) {
          await this.environmentStore.downloadS3Directory(
             packageLocation,
             environmentName,
-            absoluteTargetPath,
+            targetPath,
             isCompressedFile,
          );
       }
@@ -207,7 +248,7 @@ export class PackageController {
          // so we need to mount them on the right place.
          await this.environmentStore.mountLocalDirectory(
             packageLocation,
-            absoluteTargetPath,
+            targetPath,
             environmentName,
             packageName,
          );

package/src/default-publisher.config.json

@@ -0,0 +1,23 @@
+{
+  "frozenConfig": false,
+  "environments": [
+    {
+      "name": "malloy-samples",
+      "packages": [
+        {
+          "name": "ecommerce",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/ecommerce"
+        },
+        {
+          "name": "imdb",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/imdb"
+        },
+        {
+          "name": "faa",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa"
+        }
+      ],
+      "connections": []
+    }
+  ]
+}

package/src/errors.spec.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "bun:test";
+import {
+   BadRequestError,
+   ConnectionAuthError,
+   ConnectionError,
+   internalErrorToHttpError,
+} from "./errors";
+
+describe("internalErrorToHttpError", () => {
+   it("maps ConnectionAuthError to 422", () => {
+      const { status, json } = internalErrorToHttpError(
+         new ConnectionAuthError("creds rejected for db_x"),
+      );
+      expect(status).toBe(422);
+      expect(json).toEqual({
+         code: 422,
+         message: "creds rejected for db_x",
+      });
+   });
+
+   it("maps BadRequestError to 400", () => {
+      const { status, json } = internalErrorToHttpError(
+         new BadRequestError("bad input"),
+      );
+      expect(status).toBe(400);
+      expect(json).toEqual({ code: 400, message: "bad input" });
+   });
+
+   it("maps ConnectionError to 502 (distinct from auth, still retryable)", () => {
+      const { status, json } = internalErrorToHttpError(
+         new ConnectionError("upstream broken"),
+      );
+      expect(status).toBe(502);
+      expect(json).toEqual({ code: 502, message: "upstream broken" });
+   });
+
+   it("falls through to 500 for unrecognized errors", () => {
+      const { status, json } = internalErrorToHttpError(new Error("boom"));
+      expect(status).toBe(500);
+      expect(json.message).toBe("boom");
+   });
+});

package/src/errors.ts

@@ -16,6 +16,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(400, error.message);
    } else if (error instanceof ConnectionNotFoundError) {
       return httpError(404, error.message);
+   } else if (error instanceof ConnectionAuthError) {
+      return httpError(422, error.message);
    } else if (error instanceof ModelCompilationError) {
       return httpError(424, error.message);
    } else if (error instanceof ConnectionError) {
@@ -26,6 +28,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(409, error.message);
    } else if (error instanceof InvalidStateTransitionError) {
       return httpError(409, error.message);
+   } else if (error instanceof ServiceUnavailableError) {
+      return httpError(503, error.message);
    } else {
       return httpError(500, error.message);
    }
@@ -83,6 +87,12 @@ export class ConnectionError extends Error {
    }
 }
 
+export class ConnectionAuthError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
 export class ModelCompilationError extends Error {
    constructor(error: MalloyError) {
       super(error.message);
@@ -114,3 +124,14 @@ export class InvalidStateTransitionError extends Error {
       super(message);
    }
 }
+
+/**
+ * Thrown when the publisher is temporarily refusing a request to keep
+ * RSS under the configured `PUBLISHER_MAX_MEMORY_BYTES` cap. Mapped to
+ * HTTP 503 so an upstream proxy / client can retry with back-off.
+ */
+export class ServiceUnavailableError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}

package/src/logger.ts

@@ -28,9 +28,7 @@ export const logger = winston.createLogger({
       ? winston.format.combine(
            winston.format.uncolorize(),
            winston.format.timestamp(),
-           winston.format.metadata({
-              fillExcept: ["message", "level", "timestamp"],
-           }),
+           winston.format.errors({ stack: true }),
            winston.format.json(),
         )
       : winston.format.combine(

package/src/mcp/tools/discovery_tools.ts

@@ -222,8 +222,12 @@ export function registerTools(
                throw new Error(`Model not found: ${modelPath}`);
             }
 
-            // Use the new getModelFileText method
-            const fileText = await pkg.getModelFileText(modelPath);
+            // Route through the Environment so the disk read is serialized
+            // against installPackage / deletePackage.
+            const fileText = await environment.getModelFileText(
+               packageName,
+               modelPath,
+            );
 
             console.log(
                `[MCP LOG] Successfully retrieved model text for ${modelPath}`,

package/src/path_safety.spec.ts

@@ -0,0 +1,158 @@
+import { describe, expect, it } from "bun:test";
+import * as path from "path";
+
+import { BadRequestError } from "./errors";
+import {
+   assertSafeEnvironmentPath,
+   assertSafePackageName,
+   assertSafeRelativeModelPath,
+   safeJoinUnderRoot,
+} from "./path_safety";
+
+describe("assertSafePackageName", () => {
+   it.each([
+      "pkg",
+      "test_package",
+      "test-package",
+      "TestPackage1",
+      "test.package.name",
+      "a",
+      "x".repeat(255),
+   ])("accepts %p", (name) => {
+      expect(() => assertSafePackageName(name)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["dot", "."],
+      ["dot-dot", ".."],
+      ["leading dot", ".staging"],
+      ["forward slash", "foo/bar"],
+      ["backslash", "foo\\bar"],
+      ["null byte", "foo\0bar"],
+      ["traversal", "../etc/passwd"],
+      ["abs", "/etc/passwd"],
+      ["space", "my pkg"],
+      ["unicode", "pkg\u202E"],
+      ["too long", "x".repeat(256)],
+   ])("rejects %s (%p)", (_label, name) => {
+      expect(() => assertSafePackageName(name)).toThrow(BadRequestError);
+   });
+
+   it.each([
+      ["number", 42],
+      ["null", null],
+      ["undefined", undefined],
+      ["object", { name: "pkg" }],
+   ])("rejects non-string %s (%p)", (_label, value) => {
+      expect(() => assertSafePackageName(value)).toThrow(BadRequestError);
+   });
+});
+
+describe("assertSafeRelativeModelPath", () => {
+   it.each([
+      "model.malloy",
+      "models/foo.malloy",
+      "a/b/c/d.malloynb",
+      "deep/nested/file_name-1.malloy",
+   ])("accepts %p", (modelPath) => {
+      expect(() => assertSafeRelativeModelPath(modelPath)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["leading slash (absolute)", "/etc/passwd"],
+      ["traversal", "../etc/passwd"],
+      ["embedded traversal", "models/../../../etc/passwd"],
+      ["embedded dot segment", "models/./foo.malloy"],
+      ["double slash", "models//foo.malloy"],
+      ["trailing slash", "models/foo/"],
+      ["backslash", "models\\foo.malloy"],
+      ["null byte", "models/foo\0.malloy"],
+      ["dotfile segment", ".staging/foo.malloy"],
+      ["dotfile leaf", "models/.hidden.malloy"],
+   ])("rejects %s (%p)", (_label, modelPath) => {
+      expect(() => assertSafeRelativeModelPath(modelPath)).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("rejects non-string inputs", () => {
+      expect(() => assertSafeRelativeModelPath(undefined)).toThrow(
+         BadRequestError,
+      );
+      expect(() => assertSafeRelativeModelPath(123)).toThrow(BadRequestError);
+   });
+});
+
+describe("assertSafeEnvironmentPath", () => {
+   it.each([
+      "/etc/publisher",
+      "/var/lib/publisher/env1",
+      "/Users/me/data",
+      "/a",
+      "C:\\Users\\me\\publisher",
+      "C:/Users/me/publisher",
+   ])("accepts %p", (p) => {
+      expect(() => assertSafeEnvironmentPath(p)).not.toThrow();
+   });
+
+   it.each([
+      ["empty", ""],
+      ["relative", "publisher/data"],
+      ["traversal in middle", "/var/lib/../../etc/passwd"],
+      ["traversal at end", "/var/lib/publisher/.."],
+      ["null byte", "/var/lib/publisher\0"],
+      ["bare dot-dot", ".."],
+      ["bare dot", "."],
+      ["too long", "/" + "a".repeat(5000)],
+   ])("rejects %s (%p)", (_label, p) => {
+      expect(() => assertSafeEnvironmentPath(p)).toThrow(BadRequestError);
+   });
+
+   it("rejects non-string inputs", () => {
+      expect(() => assertSafeEnvironmentPath(undefined)).toThrow(
+         BadRequestError,
+      );
+      expect(() => assertSafeEnvironmentPath(null)).toThrow(BadRequestError);
+      expect(() => assertSafeEnvironmentPath(42)).toThrow(BadRequestError);
+   });
+});
+
+describe("safeJoinUnderRoot", () => {
+   const root = "/tmp/test-root";
+
+   it("returns the resolved root when joined with no segments", () => {
+      expect(safeJoinUnderRoot(root)).toBe(path.resolve(root));
+   });
+
+   it("joins safe segments into a path under root", () => {
+      expect(safeJoinUnderRoot(root, "pkg", "model.malloy")).toBe(
+         path.resolve(root, "pkg", "model.malloy"),
+      );
+   });
+
+   it("throws when traversal escapes the root", () => {
+      expect(() => safeJoinUnderRoot(root, "..")).toThrow(BadRequestError);
+      expect(() => safeJoinUnderRoot(root, "..", "etc", "passwd")).toThrow(
+         BadRequestError,
+      );
+      expect(() => safeJoinUnderRoot(root, "pkg", "..", "..", "etc")).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("throws when an absolute segment overrides the root", () => {
+      expect(() => safeJoinUnderRoot(root, "/etc/passwd")).toThrow(
+         BadRequestError,
+      );
+   });
+
+   it("does NOT match a sibling directory with the same prefix", () => {
+      // path.resolve("/tmp/test-root", "../test-root-bad")  ->  "/tmp/test-root-bad"
+      // which starts with "/tmp/test-root" textually but is NOT a child.
+      expect(() => safeJoinUnderRoot(root, "..", "test-root-bad")).toThrow(
+         BadRequestError,
+      );
+   });
+});

package/src/path_safety.ts

@@ -0,0 +1,140 @@
+import * as path from "path";
+
+import { BadRequestError } from "./errors";
+
+/**
+ * Path-safety helpers used by `Environment` (and any other service that
+ * builds an on-disk path from request data) to defend against directory
+ * traversal. The intent is two-fold:
+ *
+ *  1. **Source-side allowlist**: `assertSafePackageName` /
+ *     `assertSafeRelativeModelPath` reject hostile inputs (`..`, leading
+ *     `/`, `\`, NUL, dotfiles) at the entry of every public service
+ *     method before any path-construction happens. These throw
+ *     `BadRequestError` so the controller layer's error mapper returns
+ *     HTTP 400.
+ *
+ *  2. **Sink-side containment**: `safeJoinUnderRoot` joins, resolves,
+ *     and verifies the result is strictly within the supplied root.
+ *     Even if a future caller forgets the source-side check, the sink
+ *     refuses to hand back an escaping path. This is the standard
+ *     "resolve-and-contain" pattern that CodeQL's `js/path-injection`
+ *     query recognises as a sanitizer.
+ */
+
+// Single path segment: ASCII letters, digits, `-`, `_`, `.`. No leading
+// `.` so internal sibling dirs (`.staging`, `.retired`) and editor /
+// VCS dirs can't be addressed by name from outside.
+const SAFE_NAME_RE = /^(?!\.\.?$)(?!\.)[A-Za-z0-9._-]{1,255}$/;
+
+const MAX_MODEL_PATH_LEN = 1024;
+
+// An environment path is server-controlled (config / disk-derived), but
+// CodeQL conservatively treats it as tainted because Express handlers on
+// the same class touch user input. The combined regex test +
+// `..` / NUL / length check at the constructor gate is the sanitizer
+// barrier the `js/path-injection` query recognises. Printable ASCII
+// only; absolute POSIX-or-Windows path; no `..`, no NUL.
+const SAFE_ENVIRONMENT_PATH_RE = /^(?:\/|[A-Za-z]:[\\/])[\x20-\x7E]*$/;
+const MAX_ENVIRONMENT_PATH_LEN = 4096;
+
+/**
+ * Reject anything that isn't a plausible single-segment package name.
+ * The allowlist is deliberately conservative — every existing test and
+ * production package name we've seen fits within it, and tightening
+ * here costs nothing.
+ */
+export function assertSafePackageName(packageName: unknown): void {
+   if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
+      throw new BadRequestError(
+         `Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`,
+      );
+   }
+}
+
+/**
+ * Reject anything that isn't a plausible *relative* path to a model
+ * file inside a package directory. Forward slashes are allowed (models
+ * live in subdirectories like `models/foo.malloy`); backslashes,
+ * absolute paths, NUL bytes, and `..` / `.` segments are not.
+ */
+export function assertSafeRelativeModelPath(modelPath: unknown): void {
+   if (
+      typeof modelPath !== "string" ||
+      modelPath.length === 0 ||
+      modelPath.length > MAX_MODEL_PATH_LEN ||
+      modelPath.includes("\0") ||
+      modelPath.includes("\\") ||
+      path.isAbsolute(modelPath) ||
+      modelPath.startsWith("/")
+   ) {
+      throw new BadRequestError(`Invalid model path`);
+   }
+
+   const segments = modelPath.split("/");
+   for (const segment of segments) {
+      if (segment === "" || segment === "." || segment === "..") {
+         throw new BadRequestError(`Invalid model path`);
+      }
+      if (segment.startsWith(".")) {
+         throw new BadRequestError(`Invalid model path`);
+      }
+   }
+}
+
+/**
+ * Reject anything that doesn't look like a server-controlled absolute
+ * filesystem path. Applied to `environmentPath` at the constructor
+ * gate so all downstream `path.join(this.environmentPath, …)` sites
+ * see a value that has cleared an allowlist check — the canonical
+ * sanitizer-barrier pattern CodeQL's `js/path-injection` query
+ * recognises.
+ */
+export function assertSafeEnvironmentPath(environmentPath: unknown): void {
+   if (typeof environmentPath !== "string") {
+      throw new BadRequestError(`Invalid environment path: must be a string`);
+   }
+   if (
+      environmentPath.length === 0 ||
+      environmentPath.length > MAX_ENVIRONMENT_PATH_LEN
+   ) {
+      throw new BadRequestError(`Invalid environment path: bad length`);
+   }
+   if (environmentPath.indexOf("\0") !== -1) {
+      throw new BadRequestError(`Invalid environment path: contains NUL byte`);
+   }
+   // Sanitizer barrier in the shape `x.indexOf("..") !== -1` that the
+   // CodeQL `js/path-injection` query recognises as a traversal guard.
+   if (environmentPath.indexOf("..") !== -1) {
+      throw new BadRequestError(
+         `Invalid environment path: contains ".." traversal segment`,
+      );
+   }
+   if (!SAFE_ENVIRONMENT_PATH_RE.test(environmentPath)) {
+      throw new BadRequestError(
+         `Invalid environment path: must be an absolute path of printable ASCII characters`,
+      );
+   }
+}
+
+/**
+ * Resolve `path.join(root, ...segments)` and verify the result lives
+ * strictly inside `root` (or is `root` itself). Throws
+ * `BadRequestError` if the resolved path escapes the root via `..`,
+ * absolute segments, or symlink-style trickery in the input.
+ *
+ * Callers should still run `assertSafePackageName` / similar on
+ * user-controlled segments first — this helper is the second line of
+ * defense, not the first.
+ */
+export function safeJoinUnderRoot(root: string, ...segments: string[]): string {
+   const resolvedRoot = path.resolve(root);
+   const joined = path.resolve(resolvedRoot, ...segments);
+   const rootWithSep = resolvedRoot.endsWith(path.sep)
+      ? resolvedRoot
+      : resolvedRoot + path.sep;
+   if (joined !== resolvedRoot && !joined.startsWith(rootWithSep)) {
+      throw new BadRequestError(`Resolved path is outside of root`);
+   }
+   return joined;
+}