@malloy-publisher/server 0.0.197 → 0.0.198-dev1

package/src/compile/protocol.ts

@@ -0,0 +1,251 @@
+/**
+ * Wire protocol between the main thread (CompileWorkerPool) and the
+ * compile worker threads. Messages flow in both directions over the
+ * worker_threads MessagePort:
+ *
+ *   main ──▶ worker:  CompileJobRequest      (start a compile)
+ *   worker ──▶ main:  CompileJobResult       (success)
+ *   worker ──▶ main:  CompileJobError        (failure)
+ *
+ *   worker ──▶ main:  SchemaForTablesRequest (proxy schema fetch)
+ *   worker ──▶ main:  SchemaForSqlRequest    (proxy SQL block schema)
+ *   main ──▶ worker:  SchemaForTablesResponse / SchemaForSqlResponse
+ *
+ *   main ──▶ worker:  ShutdownRequest        (graceful drain & exit)
+ *
+ * The protocol intentionally uses plain structured-clonable POJOs so
+ * `parentPort.postMessage` and `worker.postMessage` can transfer them
+ * via V8's structured clone — much cheaper than JSON.stringify for
+ * the multi-MB `modelDef` payloads that come back from compile.
+ *
+ * All requests are correlated by an opaque `requestId` string so the
+ * receiver can match responses without relying on FIFO ordering.
+ */
+
+import type {
+   Annotation,
+   SQLSourceDef,
+   TableSourceDef,
+} from "@malloydata/malloy";
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: main ──▶ worker (compile job)
+// ──────────────────────────────────────────────────────────────────────
+
+/**
+ * Connection metadata the worker needs to construct a stub
+ * `InfoConnection`. Resolved lazily — the worker asks the main thread
+ * for these on the first `lookupConnection(name)` call (see
+ * {@link ConnectionMetadataRequest}). We don't ship the full list
+ * upfront because the caller layer doesn't always know it; Malloy
+ * sees connection names only as `connection.table('...')`
+ * references inside the model.
+ */
+export interface ConnectionMetadata {
+   name: string;
+   dialectName: string;
+   digest: string;
+}
+
+export interface CompileJobRequest {
+   type: "compile";
+   requestId: string;
+   /** Absolute path to the package directory on disk. */
+   packagePath: string;
+   /** Path of the model file relative to `packagePath`. */
+   modelPath: string;
+   /** Name of the default connection (e.g. "duckdb"), or null. */
+   defaultConnectionName: string | null;
+   /** Optional row-build manifest passed through to the Runtime. */
+   buildManifest?: unknown;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: worker ──▶ main (compile result)
+// ──────────────────────────────────────────────────────────────────────
+
+/**
+ * Wire shape of a successful compile. Mirrors the fields the
+ * server's `Model` constructor needs to fully describe a `.malloy`
+ * file without holding a `ModelMaterializer` reference.
+ *
+ * The materializer itself is intentionally NOT shipped back — it
+ * binds to a Runtime that holds live native connection handles and
+ * cannot cross a worker_threads boundary. The main thread builds
+ * its own materializer lazily on the first query (see
+ * `Model.ensureMaterializer`).
+ */
+export interface CompileJobResult {
+   type: "compile-result";
+   requestId: string;
+   /** Whatever `await modelMaterializer.getModel()`._modelDef returned. */
+   modelDef: unknown;
+   /** Source-info entries (from imports + local sources). */
+   sourceInfos: unknown[];
+   /** Pre-extracted API source descriptors. */
+   sources: unknown[];
+   /** Pre-extracted API query descriptors. */
+   queries: unknown[];
+   /** Parsed `#(filter)` map, keyed by source name. */
+   filterMap: Array<[string, unknown[]]>;
+   /** Givens declared on the model, already in API shape so the main
+    *  thread can stash them on the `Model` without further conversion. */
+   givens?: unknown[];
+   /** Accumulated dataStyles (from HackyDataStylesAccumulator). */
+   dataStyles: unknown;
+   /** Wall-clock ms inside the worker for the actual compile. */
+   compileDurationMs: number;
+}
+
+export interface CompileJobError {
+   type: "compile-error";
+   requestId: string;
+   /** Serialized error — the main thread reconstructs an Error. */
+   error: SerializedError;
+}
+
+/**
+ * Error wire-shape. We cannot transfer Error instances directly
+ * across postMessage cleanly (Bun/Node behaviour diverges on stack
+ * propagation), so we ship a structured payload and reconstitute on
+ * the main thread.
+ */
+export interface SerializedError {
+   name: string;
+   message: string;
+   stack?: string;
+   /** Set when the error originated as a Malloy `MalloyError`. */
+   malloyProblems?: unknown[];
+   /** Set when the error originated as `ModelCompilationError`. */
+   isCompilationError?: boolean;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: worker ──▶ main (proxy connection metadata)
+// ──────────────────────────────────────────────────────────────────────
+
+export interface ConnectionMetadataRequest {
+   type: "connection-metadata";
+   requestId: string;
+   jobId: string;
+   connectionName: string;
+}
+
+export interface ConnectionMetadataResponse {
+   type: "connection-metadata-response";
+   requestId: string;
+   ok: true;
+   metadata: ConnectionMetadata;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: worker ──▶ main (proxy schema fetches)
+// ──────────────────────────────────────────────────────────────────────
+
+export interface SchemaForTablesRequest {
+   type: "schema-for-tables";
+   requestId: string;
+   /** Job this RPC belongs to (so main routes to the right config). */
+   jobId: string;
+   connectionName: string;
+   tables: Record<string, string>;
+   options: {
+      refreshTimestamp?: number;
+      modelAnnotation?: Annotation;
+   };
+}
+
+export interface SchemaForTablesResponse {
+   type: "schema-for-tables-response";
+   requestId: string;
+   ok: true;
+   schemas: Record<string, TableSourceDef>;
+   errors: Record<string, string>;
+}
+
+export interface SchemaForSqlRequest {
+   type: "schema-for-sql";
+   requestId: string;
+   jobId: string;
+   connectionName: string;
+   sentence: unknown;
+   options: {
+      refreshTimestamp?: number;
+      modelAnnotation?: Annotation;
+   };
+}
+
+export interface SchemaForSqlResponse {
+   type: "schema-for-sql-response";
+   requestId: string;
+   ok: true;
+   structDef?: SQLSourceDef;
+   error?: string;
+}
+
+export interface RpcErrorResponse {
+   type: "rpc-error";
+   requestId: string;
+   ok: false;
+   error: SerializedError;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Direction: worker ──▶ main (file read for imports)
+// ──────────────────────────────────────────────────────────────────────
+
+/**
+ * Workers read most files directly via fs (they run in the same
+ * filesystem namespace). This RPC exists for the rare case where the
+ * package URL reader has host-specific behaviour (e.g. virtual files,
+ * remote URLs) — we delegate back to the main thread's URL reader so
+ * compile semantics stay identical to the in-process path.
+ */
+export interface ReadUrlRequest {
+   type: "read-url";
+   requestId: string;
+   jobId: string;
+   url: string;
+}
+
+export interface ReadUrlResponse {
+   type: "read-url-response";
+   requestId: string;
+   ok: true;
+   contents: string;
+   invalidationKey?: string | number | null;
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Lifecycle
+// ──────────────────────────────────────────────────────────────────────
+
+export interface ShutdownRequest {
+   type: "shutdown";
+}
+
+export interface ReadyMessage {
+   type: "ready";
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// Union types for routing
+// ──────────────────────────────────────────────────────────────────────
+
+export type MainToWorkerMessage =
+   | CompileJobRequest
+   | ConnectionMetadataResponse
+   | SchemaForTablesResponse
+   | SchemaForSqlResponse
+   | ReadUrlResponse
+   | RpcErrorResponse
+   | ShutdownRequest;
+
+export type WorkerToMainMessage =
+   | CompileJobResult
+   | CompileJobError
+   | ConnectionMetadataRequest
+   | SchemaForTablesRequest
+   | SchemaForSqlRequest
+   | ReadUrlRequest
+   | ReadyMessage;

package/src/config.spec.ts

@@ -1081,3 +1081,84 @@ describe("Config path resolution (--config and bundled default)", () => {
       expect(result.environments).toEqual([]);
    });
 });
+
+describe("getMemoryGovernorConfig", () => {
+   const GOVERNOR_ENV_VARS = [
+      "PUBLISHER_MAX_MEMORY_BYTES",
+      "PUBLISHER_MEMORY_HIGH_WATER_FRACTION",
+      "PUBLISHER_MEMORY_LOW_WATER_FRACTION",
+      "PUBLISHER_MEMORY_CHECK_INTERVAL_MS",
+      "PUBLISHER_MEMORY_BACKPRESSURE",
+   ];
+
+   beforeEach(() => {
+      for (const v of GOVERNOR_ENV_VARS) delete process.env[v];
+   });
+   afterEach(() => {
+      for (const v of GOVERNOR_ENV_VARS) delete process.env[v];
+   });
+
+   it("returns null when PUBLISHER_MAX_MEMORY_BYTES is unset", async () => {
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(getMemoryGovernorConfig()).toBeNull();
+   });
+
+   it("parses defaults when only PUBLISHER_MAX_MEMORY_BYTES is set", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = String(2 * 1024 * 1024 * 1024);
+      const { getMemoryGovernorConfig } = await import("./config");
+      const cfg = getMemoryGovernorConfig();
+      expect(cfg).not.toBeNull();
+      expect(cfg!.maxMemoryBytes).toBe(2 * 1024 * 1024 * 1024);
+      expect(cfg!.backpressureEnabled).toBe(true);
+      expect(cfg!.highWaterFraction).toBeGreaterThan(cfg!.lowWaterFraction);
+   });
+
+   it("honours fraction and interval overrides", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "1000000000";
+      process.env.PUBLISHER_MEMORY_HIGH_WATER_FRACTION = "0.85";
+      process.env.PUBLISHER_MEMORY_LOW_WATER_FRACTION = "0.7";
+      process.env.PUBLISHER_MEMORY_CHECK_INTERVAL_MS = "10000";
+      process.env.PUBLISHER_MEMORY_BACKPRESSURE = "false";
+      const { getMemoryGovernorConfig } = await import("./config");
+      const cfg = getMemoryGovernorConfig();
+      expect(cfg).not.toBeNull();
+      expect(cfg!.highWaterFraction).toBe(0.85);
+      expect(cfg!.lowWaterFraction).toBe(0.7);
+      expect(cfg!.checkIntervalMs).toBe(10000);
+      expect(cfg!.backpressureEnabled).toBe(false);
+   });
+
+   it("treats PUBLISHER_MAX_MEMORY_BYTES=0 as disabled (returns null)", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "0";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(getMemoryGovernorConfig()).toBeNull();
+   });
+
+   it("rejects a negative PUBLISHER_MAX_MEMORY_BYTES", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "-1";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(() => getMemoryGovernorConfig()).toThrow();
+   });
+
+   it("rejects low >= high", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "1000000000";
+      process.env.PUBLISHER_MEMORY_HIGH_WATER_FRACTION = "0.7";
+      process.env.PUBLISHER_MEMORY_LOW_WATER_FRACTION = "0.8";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(() => getMemoryGovernorConfig()).toThrow();
+   });
+
+   it("rejects an out-of-range fraction", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "1000000000";
+      process.env.PUBLISHER_MEMORY_HIGH_WATER_FRACTION = "1.5";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(() => getMemoryGovernorConfig()).toThrow();
+   });
+
+   it("rejects a check interval below the safety floor", async () => {
+      process.env.PUBLISHER_MAX_MEMORY_BYTES = "1000000000";
+      process.env.PUBLISHER_MEMORY_CHECK_INTERVAL_MS = "10";
+      const { getMemoryGovernorConfig } = await import("./config");
+      expect(() => getMemoryGovernorConfig()).toThrow();
+   });
+});

package/src/config.ts

@@ -99,6 +99,132 @@ export type ProcessedPublisherConfig = {
    environments: ProcessedEnvironment[];
 };
 
+/**
+ * Tunables for {@link PackageMemoryGovernor}. All values are sourced
+ * from environment variables at startup; see {@link getMemoryGovernorConfig}
+ * for parsing and defaults.
+ *
+ * The governor is admission control only: it polls process RSS on
+ * `checkIntervalMs` and toggles a single `isBackpressured` flag using
+ * a low/high-water hysteresis band. It does NOT evict, unload, or
+ * interrupt already-loaded packages — recovery is left to the kernel
+ * reclaiming pages as in-flight traffic completes.
+ */
+export interface MemoryGovernorConfig {
+   /** Hard ceiling for process RSS in bytes (the OOM-relevant figure). */
+   maxMemoryBytes: number;
+   /** Fraction of `maxMemoryBytes` at which the governor activates back-pressure (new package loads start returning HTTP 503). Must be in (0, 1) and strictly greater than `lowWaterFraction`. */
+   highWaterFraction: number;
+   /** Fraction of `maxMemoryBytes` at which the governor clears back-pressure (new package loads admitted again). Must be in (0, 1) and strictly less than `highWaterFraction`; the gap is the hysteresis band that prevents flap. */
+   lowWaterFraction: number;
+   /** Polling cadence for the RSS sampler, in milliseconds. */
+   checkIntervalMs: number;
+   /** When true, RSS crossings flip the back-pressure flag. When false, the governor still samples and emits metrics but never rejects requests — useful for a monitoring-only rollout before enabling the 503 behaviour. */
+   backpressureEnabled: boolean;
+}
+
+const DEFAULT_HIGH_WATER_FRACTION = 0.8;
+const DEFAULT_LOW_WATER_FRACTION = 0.7;
+const DEFAULT_CHECK_INTERVAL_MS = 5_000;
+const MIN_CHECK_INTERVAL_MS = 100;
+
+function parseIntEnv(name: string): number | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const value = Number.parseInt(raw, 10);
+   if (!Number.isFinite(value) || String(value) !== raw.trim()) {
+      throw new Error(
+         `Invalid value for ${name}: expected a base-10 integer, got "${raw}"`,
+      );
+   }
+   return value;
+}
+
+function parseFloatEnv(name: string): number | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const value = Number.parseFloat(raw);
+   if (!Number.isFinite(value)) {
+      throw new Error(
+         `Invalid value for ${name}: expected a finite number, got "${raw}"`,
+      );
+   }
+   return value;
+}
+
+function parseBoolEnv(name: string): boolean | undefined {
+   const raw = process.env[name];
+   if (raw === undefined || raw.trim() === "") return undefined;
+   const normalised = raw.trim().toLowerCase();
+   if (["1", "true", "yes", "on"].includes(normalised)) return true;
+   if (["0", "false", "no", "off"].includes(normalised)) return false;
+   throw new Error(
+      `Invalid value for ${name}: expected a boolean (true/false), got "${raw}"`,
+   );
+}
+
+/**
+ * Parse memory-governor settings from environment variables and return
+ * either a fully-validated config or `null` when the feature is
+ * disabled. The feature is disabled iff `PUBLISHER_MAX_MEMORY_BYTES`
+ * is unset or set to `0`.
+ *
+ * Throws at startup on malformed input so a typo in a k8s manifest
+ * surfaces as a loud failure rather than silently disabling the cap.
+ */
+export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
+   const maxMemoryBytes = parseIntEnv("PUBLISHER_MAX_MEMORY_BYTES");
+   if (maxMemoryBytes === undefined || maxMemoryBytes === 0) {
+      return null;
+   }
+   if (maxMemoryBytes < 0) {
+      throw new Error(
+         `PUBLISHER_MAX_MEMORY_BYTES must be a positive integer (got ${maxMemoryBytes})`,
+      );
+   }
+
+   const highWaterFraction =
+      parseFloatEnv("PUBLISHER_MEMORY_HIGH_WATER_FRACTION") ??
+      DEFAULT_HIGH_WATER_FRACTION;
+   const lowWaterFraction =
+      parseFloatEnv("PUBLISHER_MEMORY_LOW_WATER_FRACTION") ??
+      DEFAULT_LOW_WATER_FRACTION;
+   const checkIntervalMs =
+      parseIntEnv("PUBLISHER_MEMORY_CHECK_INTERVAL_MS") ??
+      DEFAULT_CHECK_INTERVAL_MS;
+   const backpressureEnabled =
+      parseBoolEnv("PUBLISHER_MEMORY_BACKPRESSURE") ?? true;
+
+   if (highWaterFraction <= 0 || highWaterFraction >= 1) {
+      throw new Error(
+         `PUBLISHER_MEMORY_HIGH_WATER_FRACTION must be in (0, 1) (got ${highWaterFraction})`,
+      );
+   }
+   if (lowWaterFraction <= 0 || lowWaterFraction >= 1) {
+      throw new Error(
+         `PUBLISHER_MEMORY_LOW_WATER_FRACTION must be in (0, 1) (got ${lowWaterFraction})`,
+      );
+   }
+   if (lowWaterFraction >= highWaterFraction) {
+      throw new Error(
+         `PUBLISHER_MEMORY_LOW_WATER_FRACTION (${lowWaterFraction}) must be strictly less than PUBLISHER_MEMORY_HIGH_WATER_FRACTION (${highWaterFraction})`,
+      );
+   }
+   if (checkIntervalMs < MIN_CHECK_INTERVAL_MS) {
+      throw new Error(
+         `PUBLISHER_MEMORY_CHECK_INTERVAL_MS must be >= ${MIN_CHECK_INTERVAL_MS} (got ${checkIntervalMs})`,
+      );
+   }
+
+   return {
+      maxMemoryBytes,
+      highWaterFraction,
+      lowWaterFraction,
+      checkIntervalMs,
+      backpressureEnabled,
+   };
+};
+
 function substituteEnvVars(value: string): string {
    const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
 

package/src/controller/compile.controller.ts

@@ -1,4 +1,4 @@
-import type { LogMessage } from "@malloydata/malloy";
+import type { GivenValue, LogMessage } from "@malloydata/malloy";
 import { EnvironmentStore } from "../service/environment_store";
 
 export class CompileController {
@@ -14,6 +14,7 @@ export class CompileController {
       modelName: string,
       source: string,
       includeSql: boolean = false,
+      givens?: Record<string, GivenValue>,
    ): Promise<{ status: string; problems: LogMessage[]; sql?: string }> {
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
@@ -24,6 +25,7 @@ export class CompileController {
          modelName,
          source,
          includeSql,
+         givens,
       );
 
       // Determine overall status based on presence of errors

package/src/controller/model.controller.ts

@@ -2,6 +2,7 @@ import { components } from "../api";
 import { ModelNotFoundError } from "../errors";
 import { EnvironmentStore } from "../service/environment_store";
 import type { FilterParams } from "../service/filter";
+import type { GivenValue } from "@malloydata/malloy";
 
 type ApiNotebook = components["schemas"]["Notebook"];
 type ApiModel = components["schemas"]["Model"];
@@ -97,6 +98,7 @@ export class ModelController {
       cellIndex: number,
       filterParams?: FilterParams,
       bypassFilters?: boolean,
+      givens?: Record<string, GivenValue>,
    ): Promise<{
       type: "code" | "markdown";
       text: string;
@@ -117,6 +119,11 @@ export class ModelController {
          throw new ModelNotFoundError(`${notebookPath} is a model`);
       }
 
-      return model.executeNotebookCell(cellIndex, filterParams, bypassFilters);
+      return model.executeNotebookCell(
+         cellIndex,
+         filterParams,
+         bypassFilters,
+         givens,
+      );
    }
 }