@malloy-publisher/server 0.0.198-dev → 0.0.198-dev2

package/src/config.ts

@@ -1,9 +1,68 @@
 import fs from "fs";
 import path from "path";
+import { fileURLToPath } from "url";
 import { components } from "./api";
 import { API_PREFIX, PUBLISHER_CONFIG_NAME } from "./constants";
 import { logger } from "./logger";
 
+/**
+ * Path to the publisher.config.json file shipped inside the published
+ * package. Used as a last-resort fallback so `npx @malloy-publisher/server`
+ * with no args still boots with the DuckDB-only sample packages.
+ *
+ * The file is copied next to the running module by `build.ts` at production
+ * build time. In a source/dev checkout it lives alongside this file.
+ */
+const BUNDLED_DEFAULT_CONFIG_PATH = path.join(
+   path.dirname(fileURLToPath(import.meta.url)),
+   "default-publisher.config.json",
+);
+
+/**
+ * Decide which `publisher.config.json` to read.
+ *
+ * Precedence:
+ *   1. `--config <path>` (surfaced via `process.env.PUBLISHER_CONFIG_PATH`)
+ *   2. `<serverRoot>/publisher.config.json`
+ *   3. The bundled default shipped inside the package — ONLY when
+ *      `process.env.PUBLISHER_USE_BUNDLED_DEFAULT === "true"`. server.ts
+ *      sets that flag when the user passed neither `--server_root` nor
+ *      `--config`, so `npx @malloy-publisher/server` with zero args
+ *      boots into something usable. Callers that construct an
+ *      EnvironmentStore programmatically (tests, embeds) don't get
+ *      surprise filesystem fallbacks they didn't ask for.
+ *
+ * Returns `null` if step 1 was requested but the file doesn't exist —
+ * that's an explicit user mistake and the caller should surface it as
+ * an error rather than silently falling back.
+ */
+function resolvePublisherConfigPath(serverRoot: string): {
+   path: string;
+   isBundledDefault: boolean;
+} | null {
+   const explicitPath = process.env.PUBLISHER_CONFIG_PATH;
+   if (explicitPath && explicitPath.length > 0) {
+      if (!fs.existsSync(explicitPath)) {
+         return null;
+      }
+      return { path: explicitPath, isBundledDefault: false };
+   }
+
+   const serverRootPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
+   if (fs.existsSync(serverRootPath)) {
+      return { path: serverRootPath, isBundledDefault: false };
+   }
+
+   if (
+      process.env.PUBLISHER_USE_BUNDLED_DEFAULT === "true" &&
+      fs.existsSync(BUNDLED_DEFAULT_CONFIG_PATH)
+   ) {
+      return { path: BUNDLED_DEFAULT_CONFIG_PATH, isBundledDefault: true };
+   }
+
+   return null;
+}
+
 type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
 type GcsPath = `gs://${string}`;
 type ApiConnection = components["schemas"]["Connection"];
@@ -40,6 +99,132 @@ export type ProcessedPublisherConfig = {
    environments: ProcessedEnvironment[];
 };
 
+/**
+ * Tunables for {@link PackageMemoryGovernor}. All values are sourced
+ * from environment variables at startup; see {@link getMemoryGovernorConfig}
+ * for parsing and defaults.
+ *
+ * The governor is admission control only: it polls process RSS on
+ * `checkIntervalMs` and toggles a single `isBackpressured` flag using
+ * a low/high-water hysteresis band. It does NOT evict, unload, or
+ * interrupt already-loaded packages — recovery is left to the kernel
+ * reclaiming pages as in-flight traffic completes.
+ */
+export interface MemoryGovernorConfig {
+   /** Hard ceiling for process RSS in bytes (the OOM-relevant figure). */
+   maxMemoryBytes: number;
+   /** Fraction of `maxMemoryBytes` at which the governor activates back-pressure (new package loads start returning HTTP 503). Must be in (0, 1) and strictly greater than `lowWaterFraction`. */
+   highWaterFraction: number;
+   /** Fraction of `maxMemoryBytes` at which the governor clears back-pressure (new package loads admitted again). Must be in (0, 1) and strictly less than `highWaterFraction`; the gap is the hysteresis band that prevents flap. */
+   lowWaterFraction: number;
+   /** Polling cadence for the RSS sampler, in milliseconds. */
+   checkIntervalMs: number;
+   /** When true, RSS crossings flip the back-pressure flag. When false, the governor still samples and emits metrics but never rejects requests — useful for a monitoring-only rollout before enabling the 503 behaviour. */
+   backpressureEnabled: boolean;
+}
+
+const DEFAULT_HIGH_WATER_FRACTION = 0.8;
+const DEFAULT_LOW_WATER_FRACTION = 0.7;
+const DEFAULT_CHECK_INTERVAL_MS = 5_000;
+const MIN_CHECK_INTERVAL_MS = 100;
+
+function parseIntEnv(name: string): number | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const value = Number.parseInt(raw, 10);
+   if (!Number.isFinite(value) || String(value) !== raw.trim()) {
+      throw new Error(
+         `Invalid value for ${name}: expected a base-10 integer, got "${raw}"`,
+      );
+   }
+   return value;
+}
+
+function parseFloatEnv(name: string): number | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const value = Number.parseFloat(raw);
+   if (!Number.isFinite(value)) {
+      throw new Error(
+         `Invalid value for ${name}: expected a finite number, got "${raw}"`,
+      );
+   }
+   return value;
+}
+
+function parseBoolEnv(name: string): boolean | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const normalised = raw.trim().toLowerCase();
+   if (["1", "true", "yes", "on"].includes(normalised)) return true;
+   if (["0", "false", "no", "off"].includes(normalised)) return false;
+   throw new Error(
+      `Invalid value for ${name}: expected a boolean (true/false), got "${raw}"`,
+   );
+}
+
+/**
+ * Parse memory-governor settings from environment variables and return
+ * either a fully-validated config or `null` when the feature is
+ * disabled. The feature is disabled iff `PUBLISHER_MAX_MEMORY_BYTES`
+ * is unset or set to `0`.
+ *
+ * Throws at startup on malformed input so a typo in a k8s manifest
+ * surfaces as a loud failure rather than silently disabling the cap.
+ */
+export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
+   const maxMemoryBytes = parseIntEnv("PUBLISHER_MAX_MEMORY_BYTES");
+   if (maxMemoryBytes === undefined || maxMemoryBytes === 0) {
+      return null;
+   }
+   if (maxMemoryBytes < 0) {
+      throw new Error(
+         `PUBLISHER_MAX_MEMORY_BYTES must be a positive integer (got ${maxMemoryBytes})`,
+      );
+   }
+
+   const highWaterFraction =
+      parseFloatEnv("PUBLISHER_MEMORY_HIGH_WATER_FRACTION") ??
+      DEFAULT_HIGH_WATER_FRACTION;
+   const lowWaterFraction =
+      parseFloatEnv("PUBLISHER_MEMORY_LOW_WATER_FRACTION") ??
+      DEFAULT_LOW_WATER_FRACTION;
+   const checkIntervalMs =
+      parseIntEnv("PUBLISHER_MEMORY_CHECK_INTERVAL_MS") ??
+      DEFAULT_CHECK_INTERVAL_MS;
+   const backpressureEnabled =
+      parseBoolEnv("PUBLISHER_MEMORY_BACKPRESSURE") ?? true;
+
+   if (highWaterFraction <= 0 || highWaterFraction >= 1) {
+      throw new Error(
+         `PUBLISHER_MEMORY_HIGH_WATER_FRACTION must be in (0, 1) (got ${highWaterFraction})`,
+      );
+   }
+   if (lowWaterFraction <= 0 || lowWaterFraction >= 1) {
+      throw new Error(
+         `PUBLISHER_MEMORY_LOW_WATER_FRACTION must be in (0, 1) (got ${lowWaterFraction})`,
+      );
+   }
+   if (lowWaterFraction >= highWaterFraction) {
+      throw new Error(
+         `PUBLISHER_MEMORY_LOW_WATER_FRACTION (${lowWaterFraction}) must be strictly less than PUBLISHER_MEMORY_HIGH_WATER_FRACTION (${highWaterFraction})`,
+      );
+   }
+   if (checkIntervalMs < MIN_CHECK_INTERVAL_MS) {
+      throw new Error(
+         `PUBLISHER_MEMORY_CHECK_INTERVAL_MS must be >= ${MIN_CHECK_INTERVAL_MS} (got ${checkIntervalMs})`,
+      );
+   }
+
+   return {
+      maxMemoryBytes,
+      highWaterFraction,
+      lowWaterFraction,
+      checkIntervalMs,
+      backpressureEnabled,
+   };
+};
+
 function substituteEnvVars(value: string): string {
    const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
 
@@ -77,13 +262,30 @@ function processConfigValue(value: unknown): unknown {
 }
 
 export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
-   const publisherConfigPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
-   if (!fs.existsSync(publisherConfigPath)) {
+   const resolved = resolvePublisherConfigPath(serverRoot);
+   if (!resolved) {
+      if (
+         process.env.PUBLISHER_CONFIG_PATH &&
+         process.env.PUBLISHER_CONFIG_PATH.length > 0
+      ) {
+         // Explicit --config was given but the path didn't exist. Loud
+         // failure here so a typo in the flag doesn't silently boot the
+         // server with an empty environment list.
+         logger.error(
+            `--config path not found: ${process.env.PUBLISHER_CONFIG_PATH}. Using default empty config.`,
+         );
+      }
       return {
          frozenConfig: false,
          environments: [],
       };
    }
+   const publisherConfigPath = resolved.path;
+   if (resolved.isBundledDefault) {
+      logger.info(
+         `No publisher.config.json found at ${path.join(serverRoot, PUBLISHER_CONFIG_NAME)}; falling back to bundled DuckDB-only default. Pass --config <path> or place a config in the server root to override.`,
+      );
+   }
 
    let rawConfig: unknown;
    try {
@@ -108,6 +310,24 @@ export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
    // Process environment variables in config values
    const processedConfig = processConfigValue(rawConfig);
 
+   // TODO: Remove this during projects cleanup
+   // Back-compat: the top-level key was renamed `projects` → `environments`.
+   // If a config still uses the old key, accept it once with a deprecation
+   // warning so existing on-disk configs don't silently parse as empty.
+   if (
+      processedConfig &&
+      typeof processedConfig === "object" &&
+      !("environments" in processedConfig) &&
+      "projects" in processedConfig
+   ) {
+      logger.warn(
+         `${PUBLISHER_CONFIG_NAME} uses deprecated "projects" key; rename to "environments".`,
+      );
+      (processedConfig as Record<string, unknown>).environments = (
+         processedConfig as Record<string, unknown>
+      ).projects;
+   }
+
    if (
       processedConfig &&
       typeof processedConfig === "object" &&

package/src/controller/compile.controller.ts

@@ -1,4 +1,4 @@
-import type { LogMessage } from "@malloydata/malloy";
+import type { GivenValue, LogMessage } from "@malloydata/malloy";
 import { EnvironmentStore } from "../service/environment_store";
 
 export class CompileController {
@@ -14,6 +14,7 @@ export class CompileController {
       modelName: string,
       source: string,
       includeSql: boolean = false,
+      givens?: Record<string, GivenValue>,
    ): Promise<{ status: string; problems: LogMessage[]; sql?: string }> {
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
@@ -24,6 +25,7 @@ export class CompileController {
          modelName,
          source,
          includeSql,
+         givens,
       );
 
       // Determine overall status based on presence of errors

package/src/controller/connection.controller.ts

@@ -91,7 +91,7 @@ function validateAdminAuthoredConnection(
 ): void {
    if (connectionName === "duckdb" || connectionConfig.name === "duckdb") {
       throw new BadRequestError(
-         "DuckDB connection name cannot be 'duckdb'; it is reserved for Publisher package sandboxes.",
+         "Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').",
       );
    }
 

package/src/controller/model.controller.ts

@@ -2,6 +2,7 @@ import { components } from "../api";
 import { ModelNotFoundError } from "../errors";
 import { EnvironmentStore } from "../service/environment_store";
 import type { FilterParams } from "../service/filter";
+import type { GivenValue } from "@malloydata/malloy";
 
 type ApiNotebook = components["schemas"]["Notebook"];
 type ApiModel = components["schemas"]["Model"];
@@ -97,6 +98,7 @@ export class ModelController {
       cellIndex: number,
       filterParams?: FilterParams,
       bypassFilters?: boolean,
+      givens?: Record<string, GivenValue>,
    ): Promise<{
       type: "code" | "markdown";
       text: string;
@@ -117,6 +119,11 @@ export class ModelController {
          throw new ModelNotFoundError(`${notebookPath} is a model`);
       }
 
-      return model.executeNotebookCell(cellIndex, filterParams, bypassFilters);
+      return model.executeNotebookCell(
+         cellIndex,
+         filterParams,
+         bypassFilters,
+         givens,
+      );
    }
 }

package/src/controller/package.controller.ts

@@ -1,6 +1,4 @@
-import * as path from "path";
 import { components } from "../api";
-import { PUBLISHER_DATA_DIR } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
 import { logger } from "../logger";
 import { EnvironmentStore } from "../service/environment_store";
@@ -37,15 +35,38 @@ export class PackageController {
          environmentName,
          false,
       );
-      const _package = await environment.getPackage(packageName, reload);
-      const packageLocation = _package.getPackageMetadata().location;
-      if (reload && packageLocation) {
-         await this.downloadPackage(
-            environmentName,
-            packageName,
-            packageLocation,
-         );
+
+      if (reload) {
+         // Resolve the package's source location from the currently-cached
+         // metadata WITHOUT triggering a stale-state reload. If a `location`
+         // is set, route the reload through `installPackage` so that
+         // download-then-load happens atomically; otherwise fall back to an
+         // in-place reload of the existing on-disk content.
+         let location: string | undefined;
+         try {
+            const cached = await environment.getPackage(packageName, false);
+            location = cached.getPackageMetadata().location;
+         } catch {
+            // Not previously loaded — nothing to reinstall from.
+         }
+         if (location) {
+            const reinstalled = await environment.installPackage(
+               packageName,
+               (stagingPath) =>
+                  this.downloadInto(
+                     environmentName,
+                     packageName,
+                     location,
+                     stagingPath,
+                  ),
+            );
+            return reinstalled.getPackageMetadata();
+         }
+         const _package = await environment.getPackage(packageName, true);
+         return _package.getPackageMetadata();
       }
+
+      const _package = await environment.getPackage(packageName, false);
       return _package.getPackageMetadata();
    }
 
@@ -60,21 +81,32 @@ export class PackageController {
       if (!body.name) {
          throw new BadRequestError("Package name is required");
       }
+      const packageName = body.name;
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
          false,
       );
+      let result;
       if (body.location) {
-         await this.downloadPackage(environmentName, body.name, body.location);
+         const bodyLocation = body.location;
+         result = await environment.installPackage(packageName, (stagingPath) =>
+            this.downloadInto(
+               environmentName,
+               packageName,
+               bodyLocation,
+               stagingPath,
+            ),
+         );
+      } else {
+         result = await environment.addPackage(packageName);
       }
-      const result = await environment.addPackage(body.name);
       await this.environmentStore.addPackageToDatabase(
          environmentName,
-         body.name,
+         packageName,
       );
 
       if (options?.autoLoadManifest === true) {
-         await this.tryLoadExistingManifest(environmentName, body.name);
+         await this.tryLoadExistingManifest(environmentName, packageName);
       }
 
       return result;
@@ -151,12 +183,20 @@ export class PackageController {
          false,
       );
       if (body.location) {
-         await this.downloadPackage(
-            environmentName,
-            packageName,
-            body.location,
+         // Re-install: stream the new content into a staging dir (no lock)
+         // and atomically swap it in (under the lock).
+         const bodyLocation = body.location;
+         await environment.installPackage(packageName, (stagingPath) =>
+            this.downloadInto(
+               environmentName,
+               packageName,
+               bodyLocation,
+               stagingPath,
+            ),
          );
       }
+      // Apply metadata changes (publisher.json) under the same per-package
+      // mutex via `Environment.updatePackage`.
       const result = await environment.updatePackage(packageName, body);
       await this.environmentStore.addPackageToDatabase(
          environmentName,
@@ -166,17 +206,18 @@ export class PackageController {
       return result;
    }
 
-   private async downloadPackage(
+   /**
+    * Run the right downloader for the given location into `targetPath`.
+    * This used to point at the canonical package directory, but the
+    * install pipeline now passes a sibling staging dir so the long-running
+    * download doesn't hold the per-package mutex.
+    */
+   private async downloadInto(
       environmentName: string,
       packageName: string,
       packageLocation: string,
+      targetPath: string,
    ) {
-      const absoluteTargetPath = path.join(
-         this.environmentStore.serverRootPath,
-         PUBLISHER_DATA_DIR,
-         environmentName,
-         packageName,
-      );
       const isCompressedFile = packageLocation.endsWith(".zip");
       if (
          packageLocation.startsWith("https://") ||
@@ -184,20 +225,20 @@ export class PackageController {
       ) {
          await this.environmentStore.downloadGitHubDirectory(
             packageLocation,
-            absoluteTargetPath,
+            targetPath,
          );
       } else if (packageLocation.startsWith("gs://")) {
          await this.environmentStore.downloadGcsDirectory(
             packageLocation,
             environmentName,
-            absoluteTargetPath,
+            targetPath,
             isCompressedFile,
          );
       } else if (packageLocation.startsWith("s3://")) {
          await this.environmentStore.downloadS3Directory(
             packageLocation,
             environmentName,
-            absoluteTargetPath,
+            targetPath,
             isCompressedFile,
          );
       }
@@ -207,7 +248,7 @@ export class PackageController {
          // so we need to mount them on the right place.
          await this.environmentStore.mountLocalDirectory(
             packageLocation,
-            absoluteTargetPath,
+            targetPath,
             environmentName,
             packageName,
          );

package/src/controller/query.controller.ts

@@ -4,6 +4,7 @@ import { API_PREFIX } from "../constants";
 import { ModelNotFoundError } from "../errors";
 import { EnvironmentStore } from "../service/environment_store";
 import type { FilterParams } from "../service/filter";
+import type { GivenValue } from "@malloydata/malloy";
 
 type ApiQuery = components["schemas"]["QueryResult"];
 
@@ -32,6 +33,7 @@ export class QueryController {
       compactJson: boolean = false,
       filterParams?: FilterParams,
       bypassFilters?: boolean,
+      givens?: Record<string, GivenValue>,
    ): Promise<ApiQuery> {
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
@@ -49,6 +51,7 @@ export class QueryController {
             query,
             filterParams,
             bypassFilters,
+            givens,
          );
          const renderLogs = validateRenderTags(result);
          return {

package/src/default-publisher.config.json

@@ -0,0 +1,23 @@
+{
+  "frozenConfig": false,
+  "environments": [
+    {
+      "name": "malloy-samples",
+      "packages": [
+        {
+          "name": "ecommerce",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/ecommerce"
+        },
+        {
+          "name": "imdb",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/imdb"
+        },
+        {
+          "name": "faa",
+          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa"
+        }
+      ],
+      "connections": []
+    }
+  ]
+}

package/src/errors.spec.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "bun:test";
+import {
+   BadRequestError,
+   ConnectionAuthError,
+   ConnectionError,
+   internalErrorToHttpError,
+} from "./errors";
+
+describe("internalErrorToHttpError", () => {
+   it("maps ConnectionAuthError to 422", () => {
+      const { status, json } = internalErrorToHttpError(
+         new ConnectionAuthError("creds rejected for db_x"),
+      );
+      expect(status).toBe(422);
+      expect(json).toEqual({
+         code: 422,
+         message: "creds rejected for db_x",
+      });
+   });
+
+   it("maps BadRequestError to 400", () => {
+      const { status, json } = internalErrorToHttpError(
+         new BadRequestError("bad input"),
+      );
+      expect(status).toBe(400);
+      expect(json).toEqual({ code: 400, message: "bad input" });
+   });
+
+   it("maps ConnectionError to 502 (distinct from auth, still retryable)", () => {
+      const { status, json } = internalErrorToHttpError(
+         new ConnectionError("upstream broken"),
+      );
+      expect(status).toBe(502);
+      expect(json).toEqual({ code: 502, message: "upstream broken" });
+   });
+
+   it("falls through to 500 for unrecognized errors", () => {
+      const { status, json } = internalErrorToHttpError(new Error("boom"));
+      expect(status).toBe(500);
+      expect(json.message).toBe("boom");
+   });
+});

package/src/errors.ts

@@ -16,6 +16,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(400, error.message);
    } else if (error instanceof ConnectionNotFoundError) {
       return httpError(404, error.message);
+   } else if (error instanceof ConnectionAuthError) {
+      return httpError(422, error.message);
    } else if (error instanceof ModelCompilationError) {
       return httpError(424, error.message);
    } else if (error instanceof ConnectionError) {
@@ -26,6 +28,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(409, error.message);
    } else if (error instanceof InvalidStateTransitionError) {
       return httpError(409, error.message);
+   } else if (error instanceof ServiceUnavailableError) {
+      return httpError(503, error.message);
    } else {
       return httpError(500, error.message);
    }
@@ -83,6 +87,12 @@ export class ConnectionError extends Error {
    }
 }
 
+export class ConnectionAuthError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
 export class ModelCompilationError extends Error {
    constructor(error: MalloyError) {
       super(error.message);
@@ -114,3 +124,14 @@ export class InvalidStateTransitionError extends Error {
       super(message);
    }
 }
+
+/**
+ * Thrown when the publisher is temporarily refusing a request to keep
+ * RSS under the configured `PUBLISHER_MAX_MEMORY_BYTES` cap. Mapped to
+ * HTTP 503 so an upstream proxy / client can retry with back-off.
+ */
+export class ServiceUnavailableError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}