@malloy-publisher/server 0.0.202 → 0.0.204

package/src/service/connection.ts

@@ -1084,7 +1084,7 @@ export function buildEnvironmentMalloyConfig(
             ...azureDuckDBCache.values(),
          ];
          const closeResults = await Promise.allSettled([
-            malloyConfig.releaseConnections(),
+            malloyConfig.shutdown("close"),
             ...wrapperPromises.map(async (promise) => {
                const connection = await promise;
                await connection.close();

package/src/service/environment.ts

@@ -736,7 +736,7 @@ export class Environment {
          );
          if (existingPackage !== undefined && reload) {
             this.retireConnectionGeneration(`package ${packageName}`, () =>
-               existingPackage.getMalloyConfig().releaseConnections(),
+               existingPackage.getMalloyConfig().shutdown("close"),
             );
          }
          this.packages.set(packageName, _package);
@@ -955,7 +955,7 @@ export class Environment {
 
          if (oldPackage) {
             this.retireConnectionGeneration(`package ${packageName}`, () =>
-               oldPackage.getMalloyConfig().releaseConnections(),
+               oldPackage.getMalloyConfig().shutdown("close"),
             );
          }
 
@@ -1139,7 +1139,7 @@ export class Environment {
          // any in-flight queries that already acquired a connection finish
          // before the underlying duckdb handle is released.
          this.retireConnectionGeneration(`package ${packageName}`, () =>
-            _package.getMalloyConfig().releaseConnections(),
+            _package.getMalloyConfig().shutdown("close"),
          );
 
          // Atomically rename the canonical tree out of the way so no reader
@@ -1235,7 +1235,7 @@ export class Environment {
       // they wrap. Without this, hard unload leaks per-package DuckDB handles.
       const packageReleases = await Promise.allSettled(
          Array.from(this.packages.values(), (pkg) =>
-            pkg.getMalloyConfig().releaseConnections(),
+            pkg.getMalloyConfig().shutdown("close"),
          ),
       );
       for (const result of packageReleases) {

package/src/service/environment_store.ts

@@ -708,13 +708,25 @@ export class EnvironmentStore {
    }
 
    public async getStatus() {
+      // Surface the memory governor's back-pressure as a "throttled"
+      // operational state so the control plane can stop routing new package
+      // loads/queries to a throttled worker. Draining takes precedence: a
+      // shutting-down server should keep reporting "draining". When the
+      // governor is disabled (no PUBLISHER_MAX_MEMORY_BYTES) this never fires.
+      const baseState = getOperationalState();
+      const operationalState = (
+         baseState !== "draining" &&
+         (this.memoryGovernor?.isBackpressured() ?? false)
+            ? "throttled"
+            : baseState
+      ) as components["schemas"]["ServerStatus"]["operationalState"];
+
       const status = {
          timestamp: Date.now(),
          environments: [] as Array<components["schemas"]["Environment"]>,
          initialized: this.isInitialized,
          frozenConfig: isPublisherConfigFrozen(this.serverRootPath),
-         operationalState:
-            getOperationalState() as components["schemas"]["ServerStatus"]["operationalState"],
+         operationalState,
       };
 
       const environments = await this.listEnvironments(true);

package/src/service/filter.spec.ts

@@ -423,7 +423,7 @@ describe("service/filter", () => {
          const query = "run: orders -> summary";
          const clause = "`status` = 'active'";
          expect(injectFilterRefinement(query, clause)).toBe(
-            "run: orders -> summary + {where: `status` = 'active'}",
+            "run: orders -> summary\n+ {where: `status` = 'active'}",
          );
       });
 
@@ -432,7 +432,7 @@ describe("service/filter", () => {
             "run: orders -> { group_by: status; aggregate: order_count }";
          const clause = "`region` = 'US'";
          expect(injectFilterRefinement(query, clause)).toBe(
-            "run: orders -> { group_by: status; aggregate: order_count } + {where: `region` = 'US'}",
+            "run: orders -> { group_by: status; aggregate: order_count }\n+ {where: `region` = 'US'}",
          );
       });
 
@@ -440,8 +440,19 @@ describe("service/filter", () => {
          const query = "run: orders -> summary   \n  ";
          const clause = "`status` = 'active'";
          expect(injectFilterRefinement(query, clause)).toBe(
-            "run: orders -> summary + {where: `status` = 'active'}",
+            "run: orders -> summary\n+ {where: `status` = 'active'}",
          );
       });
+
+      it("places refinement on its own line so a trailing comment cannot swallow it", () => {
+         const clause = "`org_id` = 'acme'";
+         // A trailing line comment must not extend over the injected filter.
+         for (const comment of ["//", "-- sneaky"]) {
+            const query = `run: orders -> { group_by: status } ${comment}`;
+            const refined = injectFilterRefinement(query, clause);
+            const lastLine = refined.split("\n").pop() ?? "";
+            expect(lastLine).toBe(`+ {where: ${clause}}`);
+         }
+      });
    });
 });

package/src/service/filter.ts

@@ -272,6 +272,10 @@ export function buildFilterClause(
  * Append a filter refinement to a Malloy query string.
  * Uses Malloy's `+ {where: ...}` refinement syntax.
  *
+ * The refinement is placed on its own line so that a trailing line comment
+ * (`//` or `--`) on the caller's query cannot extend over it and neutralize
+ * the filter.
+ *
  * If `filterClause` is empty, returns the original query unchanged.
  */
 export function injectFilterRefinement(
@@ -281,7 +285,7 @@ export function injectFilterRefinement(
    if (!filterClause) {
       return query;
    }
-   return `${query.trimEnd()} + {where: ${filterClause}}`;
+   return `${query.trimEnd()}\n+ {where: ${filterClause}}`;
 }
 
 // ---------------------------------------------------------------------------

package/src/service/filter_bypass.spec.ts

@@ -0,0 +1,418 @@
+/**
+ * Filter-param (`#(filter)`) enforcement across source-derivation paths.
+ *
+ * A `#(filter)` annotation marks a source as protected: a required filter (e.g.
+ * the implicit multi-tenant `org_id` boundary) must be supplied before the
+ * source can be read. The aim is to enforce that without narrowing the query
+ * shapes `execute_query` accepts.
+ *
+ * Cases guarded here:
+ *   - Direct read of a protected source: rejected (400) naming the missing
+ *     required filter; scoped to the supplied values once provided.
+ *   - Reaching a protected source under an ad-hoc name (alias / extend / chain):
+ *     enforced identically — the query still runs, scoped — rather than being
+ *     refused for its shape.
+ *   - Unprotected sources are unaffected; `bypassFilters` skips enforcement.
+ *
+ * Language-level escapes (import, raw tables, raw SQL) are out of scope here and
+ * covered in `restricted_mode.spec.ts`.
+ */
+
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import { Connection } from "@malloydata/malloy";
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { BadRequestError } from "../errors";
+import { Model } from "./model";
+import type { FilterParams } from "./filter";
+
+const TEST_DIR = path.join(os.tmpdir(), "filter-bypass-tests");
+const TEST_DB_DIR = path.join(TEST_DIR, "db");
+const TEST_DB_PATH = path.join(TEST_DB_DIR, "test.duckdb");
+const TEST_PKG_DIR = path.join(TEST_DIR, "pkg");
+
+let duckdbConnection: DuckDBConnection;
+
+// Three tenants. org_id is the partition key, so any cross-org leak shows up
+// directly as extra rows when we `group_by: org_id` (acme has 2 products).
+const SEED_SQL = `
+CREATE TABLE IF NOT EXISTS products (
+   org_id VARCHAR,
+   product_name VARCHAR
+);
+INSERT INTO products VALUES
+   ('acme', 'AcmeAnvil'),
+   ('acme', 'AcmeRocket'),
+   ('globex', 'GlobexPhone'),
+   ('initech', 'InitechStapler');
+
+CREATE TABLE IF NOT EXISTS orders (
+   org_id VARCHAR,
+   category VARCHAR,
+   region VARCHAR
+);
+INSERT INTO orders VALUES
+   ('acme', 'widgets', 'US'),
+   ('acme', 'gadgets', 'EU'),
+   ('globex', 'widgets', 'US'),
+   ('initech', 'gadgets', 'APAC');
+
+CREATE TABLE IF NOT EXISTS events (
+   org_id VARCHAR,
+   kind VARCHAR
+);
+INSERT INTO events VALUES
+   ('acme', 'click'),
+   ('globex', 'view'),
+   ('initech', 'scroll');
+`;
+
+// products: implicit Organization filter only (the multi-tenant boundary).
+const PRODUCTS_MODEL = `
+#(filter) name=Organization dimension=org_id type=equal implicit required
+source: products is duckdb.table('products') extend {
+   measure: n is count()
+   view: by_org is {
+      group_by: org_id, product_name
+      aggregate: n
+   }
+}
+`;
+
+// orders: implicit Organization + required-explicit Category + optional Region.
+const ORDERS_MODEL = `
+#(filter) name=Organization dimension=org_id type=equal implicit required
+#(filter) name=Category dimension=category type=equal required
+#(filter) name=Region dimension=region type=in
+source: orders is duckdb.table('orders') extend {
+   measure: n is count()
+}
+`;
+
+// A model that neither defines nor imports any protected source. Used as the
+// target for the unprotected-source regression.
+const ANALYTICS_MODEL = `
+source: metrics is duckdb.table('events') extend {
+   measure: c is count()
+}
+`;
+
+beforeAll(async () => {
+   await fs.mkdir(TEST_DB_DIR, { recursive: true });
+   await fs.mkdir(TEST_PKG_DIR, { recursive: true });
+   duckdbConnection = new DuckDBConnection("duckdb", TEST_DB_PATH, TEST_DB_DIR);
+   for (const stmt of SEED_SQL.trim().split(";").filter(Boolean)) {
+      await duckdbConnection.runSQL(stmt.trim() + ";");
+   }
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "products.malloy"),
+      PRODUCTS_MODEL,
+      "utf-8",
+   );
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "orders.malloy"),
+      ORDERS_MODEL,
+      "utf-8",
+   );
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "analytics.malloy"),
+      ANALYTICS_MODEL,
+      "utf-8",
+   );
+});
+
+afterAll(async () => {
+   try {
+      await duckdbConnection.close();
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      await fs.rm(TEST_DIR, { recursive: true, force: true });
+   } catch {
+      // Ignore cleanup errors
+   }
+});
+
+function getConnections(): Map<string, Connection> {
+   const map = new Map<string, Connection>();
+   map.set("duckdb", duckdbConnection);
+   return map;
+}
+
+type Row = Record<string, unknown>;
+
+function asRows(compactResult: unknown): Row[] {
+   return compactResult as Row[];
+}
+
+async function makeModel(modelPath: string): Promise<Model> {
+   return Model.create("test-pkg", TEST_PKG_DIR, modelPath, getConnections());
+}
+
+/** Run an ad-hoc query string and return the result rows. */
+async function runAdHoc(
+   model: Model,
+   query: string,
+   filterParams?: FilterParams,
+): Promise<Row[]> {
+   const { compactResult } = await model.getQueryResults(
+      undefined,
+      undefined,
+      query,
+      filterParams,
+   );
+   return asRows(compactResult);
+}
+
+/**
+ * Assert an ad-hoc query is rejected with a 400 whose message names a missing
+ * required filter (`expectedSubstring`, e.g. "Organization"). If the query
+ * instead succeeds, the assertion fails reporting the row count — a filter
+ * bypass / data leak slipped through.
+ */
+async function expectFilterRejected(
+   model: Model,
+   query: string,
+   filterParams: FilterParams | undefined,
+   expectedSubstring: string,
+): Promise<void> {
+   let leakedRows: number | undefined;
+   try {
+      const { compactResult } = await model.getQueryResults(
+         undefined,
+         undefined,
+         query,
+         filterParams,
+      );
+      leakedRows = asRows(compactResult).length;
+   } catch (error) {
+      expect(error).toBeInstanceOf(BadRequestError);
+      expect((error as Error).message).toContain(expectedSubstring);
+      return;
+   }
+   throw new Error(
+      `Expected a 400 naming "${expectedSubstring}", but the query succeeded ` +
+         `and returned ${leakedRows} rows (FILTER BYPASS / LEAK).`,
+   );
+}
+
+/** Assert every returned row is scoped to acme and the count matches. */
+function expectAcmeScoped(rows: Row[], expectedRows: number): void {
+   expect(rows.length).toBe(expectedRows);
+   for (const row of rows) expect(row.org_id).toBe("acme");
+}
+
+// ===========================================================================
+// Enforced: direct reads of a protected source.
+// ===========================================================================
+
+describe("direct read of a protected source is enforced", () => {
+   it("rejects a direct query with no filter params", async () => {
+      const model = await makeModel("products.malloy");
+      await expectFilterRejected(
+         model,
+         "run: products -> { group_by: org_id, product_name; aggregate: n is count() }",
+         undefined,
+         "Organization",
+      );
+   });
+
+   it("scopes a direct query when Organization is supplied", async () => {
+      const model = await makeModel("products.malloy");
+      const rows = await runAdHoc(
+         model,
+         "run: products -> { group_by: org_id, product_name; aggregate: n is count() }",
+         { Organization: "acme" },
+      );
+      expectAcmeScoped(rows, 2); // acme has exactly 2 products
+   });
+
+   // A predefined view on the trusted source is still the direct path.
+   it("enforces filters on a direct named-view read", async () => {
+      const model = await makeModel("products.malloy");
+      await expectFilterRejected(
+         model,
+         "run: products -> by_org",
+         undefined,
+         "Organization",
+      );
+      const rows = await runAdHoc(model, "run: products -> by_org", {
+         Organization: "acme",
+      });
+      expect(rows.length).toBe(2);
+   });
+
+   // orders — implicit Organization + required-explicit Category. Both required
+   // filters are enforced; the error names whichever is still missing.
+   it("rejects orders with no filter params (names Organization)", async () => {
+      const model = await makeModel("orders.malloy");
+      await expectFilterRejected(
+         model,
+         "run: orders -> { group_by: org_id, category; aggregate: n is count() }",
+         undefined,
+         "Organization",
+      );
+   });
+
+   it("rejects orders with only Category supplied (still missing Organization)", async () => {
+      const model = await makeModel("orders.malloy");
+      await expectFilterRejected(
+         model,
+         "run: orders -> { group_by: org_id, category; aggregate: n is count() }",
+         { Category: "widgets" },
+         "Organization",
+      );
+   });
+
+   it("rejects orders with only Organization supplied (still missing Category)", async () => {
+      const model = await makeModel("orders.malloy");
+      await expectFilterRejected(
+         model,
+         "run: orders -> { group_by: org_id, category; aggregate: n is count() }",
+         { Organization: "acme" },
+         "Category",
+      );
+   });
+
+   it("scopes orders when both Organization and Category are supplied", async () => {
+      const model = await makeModel("orders.malloy");
+      const rows = await runAdHoc(
+         model,
+         "run: orders -> { group_by: org_id, category; aggregate: n is count() }",
+         { Organization: "acme", Category: "widgets" },
+      );
+      expect(rows.length).toBe(1); // acme + widgets → one group
+      expect(rows[0].org_id).toBe("acme");
+      expect(rows[0].category).toBe("widgets");
+   });
+});
+
+// ===========================================================================
+// Enforced: alias / extend / chain of a protected source.
+//
+// Reaching a protected source under an ad-hoc name carries the SAME filter
+// requirement. The query is not rejected for its shape — it runs, scoped — so
+// the `execute_query` surface stays intact.
+// ===========================================================================
+
+interface EnforcedVector {
+   label: string;
+   modelPath: string;
+   query: string;
+   /** Required filter named in the missing-param rejection. */
+   missing: string;
+   /** Params satisfying every required filter on the protected source. */
+   validParams: FilterParams;
+   /** Rows expected once scoped to acme. */
+   expectedRows: number;
+}
+
+const enforcedVectors: EnforcedVector[] = [
+   {
+      // Plain alias — a pure rename of the protected source.
+      label: "alias (source a is products)",
+      modelPath: "products.malloy",
+      query: "source: a is products\nrun: a -> { group_by: org_id, product_name; aggregate: n is count() }",
+      missing: "Organization",
+      validParams: { Organization: "acme" },
+      expectedRows: 2,
+   },
+   {
+      // Extend with an extra measure. The body adds to the curated surface but
+      // does not touch the filter dimension, so enforcement is unaffected.
+      label: "extend (source e is products extend { … })",
+      modelPath: "products.malloy",
+      query: "source: e is products extend { measure: rc is count() }\nrun: e -> { group_by: org_id, product_name; aggregate: rc }",
+      missing: "Organization",
+      validParams: { Organization: "acme" },
+      expectedRows: 2,
+   },
+   {
+      // Chained derivation — protection is inherited link by link and resolved
+      // back to products.
+      label: "chained (a is products; b is a)",
+      modelPath: "products.malloy",
+      query: "source: a is products\nsource: b is a\nrun: b -> { group_by: org_id, product_name; aggregate: n is count() }",
+      missing: "Organization",
+      validParams: { Organization: "acme" },
+      expectedRows: 2,
+   },
+   {
+      // Ad-hoc `#(filter)` annotation in the query text declaring a NON-required
+      // Organization filter on the alias, attempting to drop the requirement.
+      // Annotations written in the query are not honored — only the protected
+      // source's own annotation counts — so the requirement still stands.
+      label: "ad-hoc #(filter) annotation override is ignored",
+      modelPath: "products.malloy",
+      query:
+         "#(filter) name=Organization dimension=org_id type=equal\n" +
+         "source: a is products extend {}\n" +
+         "run: a -> { group_by: org_id, product_name; aggregate: n is count() }",
+      missing: "Organization",
+      validParams: { Organization: "acme" },
+      expectedRows: 2,
+   },
+   {
+      // orders alias — confirms enforcement on a multi-required-filter source.
+      label: "orders alias (source a is orders)",
+      modelPath: "orders.malloy",
+      query: "source: a is orders\nrun: a -> { group_by: org_id, category; aggregate: n is count() }",
+      missing: "Organization",
+      validParams: { Organization: "acme", Category: "widgets" },
+      expectedRows: 1,
+   },
+   {
+      // orders extend.
+      label: "orders extend",
+      modelPath: "orders.malloy",
+      query: "source: e is orders extend { measure: rc is count() }\nrun: e -> { group_by: org_id, category; aggregate: rc }",
+      missing: "Organization",
+      validParams: { Organization: "acme", Category: "widgets" },
+      expectedRows: 1,
+   },
+];
+
+describe("alias/extend/chain of a protected source is enforced (not restricted)", () => {
+   for (const v of enforcedVectors) {
+      describe(v.label, () => {
+         it("rejects when the required filter is missing", async () => {
+            const model = await makeModel(v.modelPath);
+            await expectFilterRejected(model, v.query, undefined, v.missing);
+         });
+
+         it("runs scoped when valid filter params are supplied", async () => {
+            const model = await makeModel(v.modelPath);
+            const rows = await runAdHoc(model, v.query, v.validParams);
+            expectAcmeScoped(rows, v.expectedRows);
+         });
+      });
+   }
+});
+
+// ===========================================================================
+// Regression: unprotected sources stay open; bypassFilters short-circuits
+// filter enforcement.
+// ===========================================================================
+
+describe("regressions", () => {
+   it("runs an unprotected source with no filter params (no spurious demand)", async () => {
+      const model = await makeModel("analytics.malloy");
+      const rows = await runAdHoc(
+         model,
+         "run: metrics -> { group_by: org_id; aggregate: c is count() }",
+      );
+      expect(rows.length).toBe(3); // all three orgs, unfiltered
+   });
+
+   it("bypassFilters short-circuits filter enforcement on a derivation", async () => {
+      const model = await makeModel("products.malloy");
+      const { compactResult } = await model.getQueryResults(
+         undefined,
+         undefined,
+         "source: a is products\nrun: a -> { group_by: org_id; aggregate: n is count() }",
+         {},
+         true,
+      );
+      expect(asRows(compactResult).length).toBe(3);
+   });
+});

package/src/service/given.ts

@@ -13,14 +13,18 @@
  * deps, so it's safe to bundle into the worker entry).
  */
 
+import type { Annotations } from "@malloydata/malloy";
+import { isReservedRoute } from "./annotations";
+
 /**
  * Duck-typed shape of a Malloy SDK `Given` instance (the value type
- * of `Model.givens`).
+ * of `Model.givens`). `Given` itself isn't re-exported from the
+ * package root, but the `Annotations` view it returns is.
  */
 export interface MalloyGiven {
    readonly name: string;
    readonly type: { type: string; filterType?: string };
-   getTaglines(prefix?: RegExp): string[];
+   readonly annotations: Annotations;
 }
 
 /**
@@ -32,6 +36,14 @@ export interface MalloyGivenApi {
    name: string;
    type: string;
    annotations?: string[];
+   /**
+    * The given's default as a Malloy source literal — one literal per declared
+    * `type`. Examples across the type range: `'WN'` or `"WN"` (string), `2003`
+    * (number), `true` (boolean), `@2024-01-01` (date), `f'WN'` (filter). Omitted
+    * when the given has no default. Consumers render/prefill it per `type` (e.g.
+    * unquote a string).
+    */
+   default?: string;
 }
 
 /**
@@ -46,16 +58,18 @@ export interface MalloyGivenApi {
  *   location either; matching that floor. A future PR can add a
  *   sanitised package-relative path if a client needs it.
  *
- * - `default` / `defaultText` — Malloy's API only exposes the
- *   parsed `ConstantExpr` AST, not a rendered source string.
- *   Rendering it here would duplicate the Malloy printer. Add
- *   when Malloy surfaces a stringified accessor.
+ * - `default` is surfaced as the rendered source literal
+ *   (`given._internal.defaultText` — e.g. a string `'WN'`, number
+ *   `2003`, boolean `true`, date `@2024-01-01`, or filter `f'WN'`).
+ *   Malloy's public surface still exposes only the parsed `.default`
+ *   AST; `_internal.defaultText` is the already-rendered string, so we
+ *   forward it verbatim rather than re-implement the printer. Omitted
+ *   when the given has no default.
  *
- * `annotations` is restricted to `#(...)` declaration annotations
- * (the caller-facing kind, e.g. `#(doc)`). `getTaglines()` with no
- * prefix would also return `##` doc-comment lines and the
- * model-level `##!` pragma, which aren't part of the given's
- * surface contract.
+ * `annotations` is restricted to app-route annotations (bracketed,
+ * caller-facing, e.g. `#(doc)`), excluding Malloy's reserved routes
+ * (plain `#` tags, `#"` doc strings, `##!` pragmas), which aren't part
+ * of the given's surface contract.
  *
  * Type rendering: `GivenTypeDef` is typed as `AtomicTypeDef |
  * FilterExpressionParamTypeDef`, but Malloy's grammar only emits
@@ -75,6 +89,17 @@ export function malloyGivenToApi(given: MalloyGiven): MalloyGivenApi {
    return {
       name: given.name,
       type: renderedType,
-      annotations: given.getTaglines(/^#\(/),
+      annotations: given.annotations
+         .forRoute(undefined)
+         .filter((note) => !isReservedRoute(note.route))
+         .map((note) => note.text),
+      // `_internal.defaultText` is the already-rendered source literal of the
+      // given's default. It lives on Malloy's private `_internal` (the public
+      // surface exposes only the parsed `.default` AST node, not a stringified
+      // form), so we reach it through a localized cast rather than widening the
+      // duck-typed `MalloyGiven` — which would collide with the SDK `Given`'s
+      // own private `_internal` at every `as MalloyGiven` cast site.
+      default: (given as { _internal?: { defaultText?: string } })._internal
+         ?.defaultText,
    };
 }