@malloy-publisher/server 0.0.198-dev → 0.0.198-dev1

package/src/service/environment_admission.spec.ts

@@ -0,0 +1,180 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+
+import { ServiceUnavailableError } from "../errors";
+import { buildEnvironmentMalloyConfig } from "./connection";
+import { Environment } from "./environment";
+import type { PackageMemoryGovernor } from "./package_memory_governor";
+
+/**
+ * Minimal subset of {@link PackageMemoryGovernor} that
+ * `Environment.assertCanAdmitNewPackage` actually consults. Allows us
+ * to drive the gate from the tests without spinning the real OTel
+ * instrumentation pipeline.
+ */
+class StubGovernor {
+   public backpressured = false;
+   isBackpressured(): boolean {
+      return this.backpressured;
+   }
+}
+
+function makeEnvironment(envPath: string): Environment {
+   const malloyConfig = buildEnvironmentMalloyConfig([], envPath);
+   return new Environment("test-env", envPath, malloyConfig, []);
+}
+
+describe("Environment admission gate (memory governor choke point)", () => {
+   let envDir: string;
+
+   beforeEach(() => {
+      envDir = fs.mkdtempSync(
+         path.join(os.tmpdir(), "publisher-env-admission-"),
+      );
+   });
+
+   afterEach(() => {
+      fs.rmSync(envDir, { recursive: true, force: true });
+   });
+
+   it("admits new packages when no governor is attached (legacy behaviour)", async () => {
+      const env = makeEnvironment(envDir);
+      // No governor set; the gate must be a pure no-op. The package
+      // doesn't exist on disk, so addPackage rejects with
+      // PackageNotFoundError — that we get any error other than 503
+      // is the assertion.
+      let caught: unknown;
+      try {
+         await env.addPackage("does-not-exist");
+      } catch (err) {
+         caught = err;
+      }
+      expect(caught).toBeDefined();
+      expect(caught).not.toBeInstanceOf(ServiceUnavailableError);
+   });
+
+   it("rejects getPackage cache-miss with 503 when back-pressured", async () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      // No package by this name has ever been loaded, so this is the
+      // exact "lazy-load on cache miss" path Monty flagged. The gate
+      // must throw before Package.create touches the disk.
+      await expect(env.getPackage("ghost", false)).rejects.toBeInstanceOf(
+         ServiceUnavailableError,
+      );
+   });
+
+   it("rejects getPackage reload=true with 503 when back-pressured", async () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      await expect(env.getPackage("ghost", true)).rejects.toBeInstanceOf(
+         ServiceUnavailableError,
+      );
+   });
+
+   it("rejects addPackage with 503 when back-pressured (after the 404 check passes)", async () => {
+      // Create a real (empty) package directory so the existence
+      // check passes and the gate gets to run. Without this, the
+      // PackageNotFoundError would mask the 503 we want to assert.
+      const pkgName = "real-pkg";
+      fs.mkdirSync(path.join(envDir, pkgName));
+
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      await expect(env.addPackage(pkgName)).rejects.toBeInstanceOf(
+         ServiceUnavailableError,
+      );
+   });
+
+   it("returns 404 (not 503) when the package directory does not exist, even under pressure", async () => {
+      // 404 must take precedence over 503: a permanent "you forgot to
+      // upload the package" error should not be masked as a transient
+      // "retry later" — otherwise operators chase phantom memory
+      // problems while the real fix is a missing artifact.
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      let caught: unknown;
+      try {
+         await env.addPackage("never-existed");
+      } catch (err) {
+         caught = err;
+      }
+      expect(caught).toBeDefined();
+      expect(caught).not.toBeInstanceOf(ServiceUnavailableError);
+   });
+
+   it("allowAdmission=true bypasses the gate (for future warmup/probe callers)", async () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      // With the bypass, the gate must not fire — the call should
+      // proceed to the real loader and fail there with some other
+      // error (PackageNotFoundError-equivalent from Package.create on
+      // a non-existent directory). The assertion is "not 503".
+      let caught: unknown;
+      try {
+         await env.getPackage("ghost", false, { allowAdmission: true });
+      } catch (err) {
+         caught = err;
+      }
+      expect(caught).toBeDefined();
+      expect(caught).not.toBeInstanceOf(ServiceUnavailableError);
+   });
+
+   it("clearing back-pressure on the governor immediately re-admits new loads", async () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+
+      governor.backpressured = true;
+      await expect(env.getPackage("ghost", false)).rejects.toBeInstanceOf(
+         ServiceUnavailableError,
+      );
+
+      // Flip the flag (simulating the periodic poller crossing the
+      // low-water mark) and verify the next call no longer 503s.
+      governor.backpressured = false;
+      let caught: unknown;
+      try {
+         await env.getPackage("ghost", false);
+      } catch (err) {
+         caught = err;
+      }
+      expect(caught).toBeDefined();
+      expect(caught).not.toBeInstanceOf(ServiceUnavailableError);
+   });
+
+   it("detaching the governor (set null) reverts to legacy admit-everything", async () => {
+      const env = makeEnvironment(envDir);
+      const governor = new StubGovernor();
+      env.setMemoryGovernor(governor as unknown as PackageMemoryGovernor);
+      governor.backpressured = true;
+
+      env.setMemoryGovernor(null);
+
+      let caught: unknown;
+      try {
+         await env.getPackage("ghost", false);
+      } catch (err) {
+         caught = err;
+      }
+      expect(caught).toBeDefined();
+      expect(caught).not.toBeInstanceOf(ServiceUnavailableError);
+   });
+});

package/src/service/environment_store.ts

@@ -30,6 +30,7 @@ import { formatDuration, logger } from "../logger";
 import { Connection } from "../storage/DatabaseInterface";
 import { StorageConfig, StorageManager } from "../storage/StorageManager";
 import { Environment, PackageStatus } from "./environment";
+import type { PackageMemoryGovernor } from "./package_memory_governor";
 type ApiEnvironment = components["schemas"]["Environment"];
 
 const AZURE_SUPPORTED_SCHEMES = ["https://", "http://", "abfss://", "az://"];
@@ -101,6 +102,10 @@ export class EnvironmentStore {
       followRegionRedirects: true,
    });
    private gcsClient: Storage;
+   // Shared by every Environment so the back-pressure decision is
+   // process-wide. Set once at server start via setMemoryGovernor;
+   // new Environments pick it up at construction.
+   private memoryGovernor: PackageMemoryGovernor | null = null;
 
    constructor(serverRootPath: string) {
       this.serverRootPath = serverRootPath;
@@ -117,6 +122,19 @@ export class EnvironmentStore {
       this.finishedInitialization = this.initialize();
    }
 
+   /**
+    * Attach (or detach with `null`) the shared {@link PackageMemoryGovernor}.
+    * Propagated to every Environment so the back-pressure decision is
+    * process-wide, and remembered so any Environment created *after*
+    * this call also picks it up at construction.
+    */
+   public setMemoryGovernor(governor: PackageMemoryGovernor | null): void {
+      this.memoryGovernor = governor;
+      for (const env of this.environments.values()) {
+         env.setMemoryGovernor(governor);
+      }
+   }
+
    private async addConfiguredEnvironment(environment: ProcessedEnvironment) {
       try {
          await this.addEnvironment(
@@ -238,6 +256,9 @@ export class EnvironmentStore {
                               ...conn.config,
                            })),
                         );
+                        environmentInstance.setMemoryGovernor(
+                           this.memoryGovernor,
+                        );
 
                         // Get packages from database
                         const packages = await repository.listPackages(
@@ -837,6 +858,7 @@ export class EnvironmentStore {
          absoluteEnvironmentPath,
          environment.connections || [],
       );
+      newEnvironment.setMemoryGovernor(this.memoryGovernor);
 
       if (!newEnvironment.metadata) newEnvironment.metadata = {};
       newEnvironment.metadata.location = absoluteEnvironmentPath;

package/src/service/filter_integration.spec.ts

@@ -133,6 +133,55 @@ import "child_orders.malloy"
 run: child_orders -> summary
 `;
 
+// Model with a given: declaration — view filters rows by the given value
+const MODEL_WITH_GIVENS = `##! experimental.givens
+
+given: target_region :: string is 'US'
+
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure:
+      order_count is count()
+      total_amount is sum(amount)
+
+   view: by_given_region is {
+      where: region = $target_region
+      aggregate: order_count, total_amount
+   }
+}
+`;
+
+// Model with both a #(filter) annotation and a given: declaration to verify composition
+const MODEL_WITH_GIVENS_AND_FILTER = `##! experimental.givens
+
+given: target_region :: string is 'US'
+
+#(filter) dimension=status type=equal
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure:
+      order_count is count()
+      total_amount is sum(amount)
+
+   view: by_given_region is {
+      where: region = $target_region
+      aggregate: order_count, total_amount
+   }
+}
+`;
+
+const NOTEBOOK_GIVENS = `>>>markdown
+# Givens Test
+
+>>>malloy
+import "orders_givens.malloy"
+
+>>>malloy
+run: orders -> by_given_region
+`;
+
 beforeAll(async () => {
    await fs.mkdir(TEST_DB_DIR, { recursive: true });
    await fs.mkdir(TEST_PKG_DIR, { recursive: true });
@@ -657,6 +706,67 @@ describe("filter integration", () => {
          expect(markdownCell.type).toBe("markdown");
          expect(markdownCell.text).toContain("Test Notebook");
       });
+
+      it("applies givens to notebook cell execution", async () => {
+         await writeFile("orders_givens.malloy", MODEL_WITH_GIVENS);
+         await writeFile("givens_notebook.malloynb", NOTEBOOK_GIVENS);
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "givens_notebook.malloynb",
+            getConnections(),
+         );
+
+         // Cell 2: run: orders -> by_given_region with target_region overridden to 'EU'
+         // EU rows: (3,'EU','active',150) and (4,'EU','cancelled',75) → order_count=2, total_amount=225
+         const codeCell = await model.executeNotebookCell(
+            2,
+            undefined,
+            undefined,
+            { target_region: "EU" },
+         );
+         expect(codeCell.result).toBeDefined();
+
+         const notebookRows = parseNotebookResult(codeCell.result!);
+         expect(notebookRows.length).toBe(1);
+         expect(Number(notebookRows[0].order_count)).toBe(2);
+         expect(Number(notebookRows[0].total_amount)).toBe(225);
+      });
+
+      it("composes givens and filterParams in notebook cell execution", async () => {
+         await writeFile(
+            "orders_givens_filter.malloy",
+            MODEL_WITH_GIVENS_AND_FILTER,
+         );
+         await writeFile(
+            "givens_filter_notebook.malloynb",
+            NOTEBOOK_GIVENS.replace(
+               "orders_givens.malloy",
+               "orders_givens_filter.malloy",
+            ),
+         );
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "givens_filter_notebook.malloynb",
+            getConnections(),
+         );
+
+         // given restricts to APAC; filterParam restricts to active
+         // APAC + active: only (5,'APAC','active',300) → order_count=1, total_amount=300
+         const codeCell = await model.executeNotebookCell(
+            2,
+            { status: "active" },
+            undefined,
+            { target_region: "APAC" },
+         );
+         expect(codeCell.result).toBeDefined();
+
+         const notebookRows = parseNotebookResult(codeCell.result!);
+         expect(notebookRows.length).toBe(1);
+         expect(Number(notebookRows[0].order_count)).toBe(1);
+         expect(Number(notebookRows[0].total_amount)).toBe(300);
+      });
    });
 
    // -----------------------------------------------------------------------

package/src/service/givens_integration.spec.ts

@@ -0,0 +1,192 @@
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import { Connection } from "@malloydata/malloy";
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { Model } from "./model";
+
+const TEST_DIR = path.join(os.tmpdir(), "givens-integration-tests");
+const TEST_DB_DIR = path.join(TEST_DIR, "db");
+const TEST_DB_PATH = path.join(TEST_DB_DIR, "test.duckdb");
+const TEST_PKG_DIR = path.join(TEST_DIR, "pkg");
+
+let duckdbConnection: DuckDBConnection;
+
+const SEED_SQL = `
+CREATE TABLE IF NOT EXISTS orders (
+   order_id INTEGER,
+   region VARCHAR,
+   order_date DATE
+);
+INSERT INTO orders VALUES
+   (1, 'US', '2024-01-15'),
+   (2, 'EU', '2024-02-10'),
+   (3, 'APAC', '2024-03-05');
+`;
+
+const MODEL_WITH_GIVENS = `
+##! experimental.givens
+
+given: region_filter :: string is 'US'
+given: cutoff_date :: date is @2024-02-01
+
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure: order_count is count()
+}
+`;
+
+const MODEL_WITHOUT_GIVENS = `
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure: order_count is count()
+}
+`;
+
+const MODEL_WITH_ANNOTATED_GIVEN = `
+##! experimental.givens
+
+#(doc) Region code, e.g. US, EU
+#(label) Region
+given: region_filter :: string is 'US'
+
+source: orders is duckdb.table('orders') extend {
+   primary_key: order_id
+}
+`;
+
+beforeAll(async () => {
+   await fs.mkdir(TEST_DB_DIR, { recursive: true });
+   await fs.mkdir(TEST_PKG_DIR, { recursive: true });
+   duckdbConnection = new DuckDBConnection("duckdb", TEST_DB_PATH, TEST_DB_DIR);
+   for (const stmt of SEED_SQL.trim().split(";").filter(Boolean)) {
+      await duckdbConnection.runSQL(stmt.trim() + ";");
+   }
+   // Each fixture lives in its own file. Tests share `beforeAll` for harness
+   // setup but never edit these files at runtime, so no `beforeEach` /
+   // `afterEach` cleanup is needed.
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "orders.malloy"),
+      MODEL_WITH_GIVENS,
+      "utf-8",
+   );
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "orders_no_givens.malloy"),
+      MODEL_WITHOUT_GIVENS,
+      "utf-8",
+   );
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "orders_annotated.malloy"),
+      MODEL_WITH_ANNOTATED_GIVEN,
+      "utf-8",
+   );
+});
+
+afterAll(async () => {
+   try {
+      await duckdbConnection.close();
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      await fs.rm(TEST_DIR, { recursive: true, force: true });
+   } catch {
+      // Ignore cleanup errors
+   }
+});
+
+function getConnections(): Map<string, Connection> {
+   const map = new Map<string, Connection>();
+   map.set("duckdb", duckdbConnection);
+   return map;
+}
+
+describe("givens introspection", () => {
+   it("surfaces declared givens on the compiled-model response", async () => {
+      const model = await Model.create(
+         "test-pkg",
+         TEST_PKG_DIR,
+         "orders.malloy",
+         getConnections(),
+      );
+
+      const compiledModel = await model.getModel();
+
+      expect(compiledModel.givens).toBeDefined();
+      expect(compiledModel.givens).toHaveLength(2);
+
+      const byName = new Map(
+         (compiledModel.givens ?? []).map((g) => [g.name, g]),
+      );
+      const region = byName.get("region_filter");
+      const cutoff = byName.get("cutoff_date");
+
+      expect(region).toBeDefined();
+      expect(region?.type).toBe("string");
+      expect(cutoff).toBeDefined();
+      expect(cutoff?.type).toBe("date");
+   });
+
+   it("attaches the model-level givens list to every source", async () => {
+      const model = await Model.create(
+         "test-pkg",
+         TEST_PKG_DIR,
+         "orders.malloy",
+         getConnections(),
+      );
+
+      const sources = model.getSources();
+      expect(sources).toBeDefined();
+      expect(sources).toHaveLength(1);
+
+      const ordersSource = sources?.[0];
+      expect(ordersSource?.name).toBe("orders");
+      expect(ordersSource?.givens).toBeDefined();
+      expect(ordersSource?.givens).toHaveLength(2);
+
+      const names = (ordersSource?.givens ?? []).map((g) => g.name).sort();
+      expect(names).toEqual(["cutoff_date", "region_filter"]);
+   });
+
+   it("returns undefined for givens when the model declares none", async () => {
+      const model = await Model.create(
+         "test-pkg",
+         TEST_PKG_DIR,
+         "orders_no_givens.malloy",
+         getConnections(),
+      );
+
+      const compiledModel = await model.getModel();
+
+      // Absent rather than empty: matches how `sources`/`queries` behave when
+      // there are none, and lets OpenAPI clients distinguish "feature
+      // unsupported" from "supported but no declarations."
+      expect(compiledModel.givens).toBeUndefined();
+      expect(model.getSources()?.[0]?.givens).toBeUndefined();
+   });
+
+   it("surfaces only `#(...)` annotations, not pragmas or doc comments", async () => {
+      const model = await Model.create(
+         "test-pkg",
+         TEST_PKG_DIR,
+         "orders_annotated.malloy",
+         getConnections(),
+      );
+
+      const compiledModel = await model.getModel();
+
+      expect(compiledModel.givens).toHaveLength(1);
+      const region = compiledModel.givens?.[0];
+      expect(region?.name).toBe("region_filter");
+
+      // The model declares two `#(...)` annotations plus a `##!` pragma.
+      // Only the `#(...)` lines should land on the wire.
+      const annotations = region?.annotations ?? [];
+      expect(annotations.length).toBeGreaterThanOrEqual(2);
+      for (const line of annotations) {
+         expect(line.startsWith("#(")).toBe(true);
+      }
+      // Negative assertion: no pragma leakage.
+      expect(annotations.some((a) => a.startsWith("##!"))).toBe(false);
+   });
+});

package/src/service/manifest_service.spec.ts

@@ -39,6 +39,7 @@ function createMocks() {
    } as unknown as sinon.SinonStubbedInstance<ResourceRepository>;
 
    const reloadAllModels = sandbox.stub().resolves();
+   const reloadAllModelsForPackage = sandbox.stub().resolves();
 
    const pkg = {
       reloadAllModels,
@@ -46,6 +47,7 @@ function createMocks() {
 
    const environment = {
       getPackage: sandbox.stub().resolves(pkg),
+      reloadAllModelsForPackage,
    };
 
    const environmentStore = {
@@ -66,6 +68,7 @@ function createMocks() {
       environment,
       pkg,
       reloadAllModels,
+      reloadAllModelsForPackage,
       service,
    };
 }
@@ -153,7 +156,9 @@ describe("ManifestService", () => {
             ),
          ).toBe(true);
          expect(ctx.environment.getPackage.calledWith("pkg", false)).toBe(true);
-         expect(ctx.reloadAllModels.calledWith(manifest.entries)).toBe(true);
+         expect(
+            ctx.reloadAllModelsForPackage.calledWith("pkg", manifest.entries),
+         ).toBe(true);
       });
 
       it("should return an empty manifest when no entries exist", async () => {
@@ -170,7 +175,7 @@ describe("ManifestService", () => {
          );
 
          expect(result.entries).toEqual({});
-         expect(ctx.reloadAllModels.calledWith({})).toBe(true);
+         expect(ctx.reloadAllModelsForPackage.calledWith("pkg", {})).toBe(true);
       });
    });
 

package/src/service/manifest_service.ts

@@ -82,8 +82,14 @@ export class ManifestService {
          environmentName,
          false,
       );
-      const pkg = await environment.getPackage(packageName, false);
-      await pkg.reloadAllModels(manifest.entries);
+      // Ensure the package is loaded, then reload its models under the
+      // per-package mutex so the disk reads are serialized against
+      // installPackage / deletePackage.
+      await environment.getPackage(packageName, false);
+      await environment.reloadAllModelsForPackage(
+         packageName,
+         manifest.entries,
+      );
 
       logger.info("Reloaded manifest and recompiled models", {
          environmentId,

package/src/service/materialization_service.ts

@@ -376,8 +376,14 @@ export class MaterializationService {
                environmentName,
                false,
             );
-            const pkg = await environment.getPackage(packageName, false);
-            await pkg.reloadAllModels(updatedManifest.entries);
+            // Ensure the package is loaded, then reload models under the
+            // per-package mutex so the disk reads are serialized against
+            // installPackage / deletePackage.
+            await environment.getPackage(packageName, false);
+            await environment.reloadAllModelsForPackage(
+               packageName,
+               updatedManifest.entries,
+            );
          }
 
          await this.transitionExecution(executionId, "SUCCESS", {
@@ -604,8 +610,13 @@ export class MaterializationService {
       // ── STEP 2: COMPILE & PLAN ─────────────────────────────────────
       // `connections` is built lazily from the connection names the plan
       // actually targets — no upfront ATTACH on every environment connection.
+      // Hold the per-package mutex for the duration of the compile so the
+      // `fs.stat` + `runtime.loadModel` calls inside `compilePackageBuildPlan`
+      // are serialized against `installPackage` / `deletePackage`.
       const { graphs, sources, connectionDigests, connections } =
-         await this.compilePackageBuildPlan(pkg, signal);
+         await environment.withPackageLock(packageName, () =>
+            this.compilePackageBuildPlan(pkg, signal),
+         );
 
       if (graphs.length === 0) {
          logger.info("No persist sources to build");