@malloy-publisher/server 0.0.203 → 0.0.204

package/src/service/package_worker_path.spec.ts

@@ -167,6 +167,119 @@ describe("Package.create via worker pool", () => {
       }
    });
 
+   it("validates and surfaces a valid #(authorize) model through the worker", async () => {
+      writeManifest();
+      fs.writeFileSync(
+         path.join(tempDir, "gated.malloy"),
+         `##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$ROLE = 'analyst'"
+source: gated is duckdb.sql("select 1 as id")`,
+      );
+
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
+         const model = pkg.getModel("gated.malloy");
+         const apiModel = (await model!.getModel()) as {
+            sources?: { name?: string; authorize?: string[] }[];
+         };
+         // The worker compiled the authorize probe (no throw) and surfaced the
+         // effective expression list — proves worker-path validation runs.
+         expect(apiModel.sources?.[0]?.authorize).toEqual([
+            "$ROLE = 'analyst'",
+         ]);
+         expect(model!.getAuthorize("gated")).toEqual(["$ROLE = 'analyst'"]);
+      } finally {
+         await duckdb.close();
+      }
+   });
+
+   it("rejects a package whose #(authorize) references an unknown given (worker validation)", async () => {
+      writeManifest();
+      fs.writeFileSync(
+         path.join(tempDir, "badgate.malloy"),
+         `##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$NOPE = 'x'"
+source: gated is duckdb.sql("select 1 as id")`,
+      );
+
+      const { ModelCompilationError } = await import("../errors");
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         // Must surface as a 424 ModelCompilationError across the worker
+         // boundary, not a generic 500 — the worker serializes it with
+         // isCompilationError so the main thread re-wraps it.
+         await expect(
+            Package.create("env", "pkg", tempDir, malloyConfig),
+         ).rejects.toBeInstanceOf(ModelCompilationError);
+      } finally {
+         await duckdb.close();
+      }
+   });
+
+   it("validates a valid #(authorize) source in a .malloynb notebook through the worker", async () => {
+      writeManifest();
+      fs.writeFileSync(
+         path.join(tempDir, "gated.malloynb"),
+         `>>>markdown
+# Gated notebook
+
+>>>malloy
+##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$ROLE = 'analyst'"
+source: gated is duckdb.sql("select 1 as id")`,
+      );
+
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
+         const model = pkg.getModel("gated.malloynb");
+         // compileNotebookModel ran authorize validation (no throw) and
+         // surfaced the gate — the notebook compile path was previously
+         // unexercised by tests.
+         expect(model!.getAuthorize("gated")).toEqual(["$ROLE = 'analyst'"]);
+      } finally {
+         await duckdb.close();
+      }
+   });
+
+   it("rejects a .malloynb notebook whose #(authorize) references an unknown given", async () => {
+      writeManifest();
+      fs.writeFileSync(
+         path.join(tempDir, "badgate.malloynb"),
+         `>>>malloy
+##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$NOPE = 'x'"
+source: gated is duckdb.sql("select 1 as id")`,
+      );
+
+      const { ModelCompilationError } = await import("../errors");
+      const { malloyConfig, duckdb } = await makeMalloyConfig();
+      try {
+         await expect(
+            Package.create("env", "pkg", tempDir, malloyConfig),
+         ).rejects.toBeInstanceOf(ModelCompilationError);
+      } finally {
+         await duckdb.close();
+      }
+   });
+
    // NB: kept last in this describe — swapping the singleton for a
    // pre-shutdown pool also tears down the shared `pool` (the swap
    // implementation shuts down the outgoing singleton). Subsequent

package/src/service/quoting.ts

@@ -1,23 +1,3 @@
-/**
- * Minimal identifier-quoting surface. Every `Dialect` in `@malloydata/malloy`
- * implements this; we accept the duck type so tests can inject a fake without
- * instantiating a full dialect.
- */
-export interface Quoter {
-   quoteTablePath(seg: string): string;
-}
-
-/**
- * Quote a potentially schema-qualified table path (e.g. "schema.table")
- * by quoting each segment individually with the dialect's quoteTablePath.
- */
-export function quoteTablePath(path: string, dialect: Quoter): string {
-   return path
-      .split(".")
-      .map((seg) => dialect.quoteTablePath(seg))
-      .join(".");
-}
-
 /**
  * Split a possibly schema-qualified table name into its schema prefix
  * (including the trailing dot) and the bare table name.

package/src/service/restricted_mode.spec.ts

@@ -0,0 +1,299 @@
+/**
+ * Restricted-mode containment for untrusted ad-hoc query text.
+ *
+ * The `query` text that reaches `execute_query` is authored by an untrusted
+ * caller (an MCP/LLM client, a UI field, an HTTP body), but it runs against a
+ * warehouse connection that can see far more than any one model curates.
+ * Compiling that text with `loadRestrictedQuery` keeps the caller inside the
+ * model's published surface — its sources, views, dimensions and measures —
+ * and stops it from using Malloy as a general-purpose handle to the underlying
+ * database or filesystem.
+ *
+ * These tests are written from the publisher's threat model: each is a way an
+ * untrusted query could try to reach data or compute the model never exposed,
+ * paired with the assertion that restricted mode blocks it. They are not a
+ * re-test of Malloy's per-construct rejection logic — the point is the misuse
+ * scenario, not the grammar.
+ *
+ * The setup: the connection holds a `secrets` table that the `catalog` model
+ * never references. The model only exposes `widgets`. Any query that manages to
+ * return a row of `secrets` has escaped the curated surface.
+ */
+
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import { Connection } from "@malloydata/malloy";
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { Model } from "./model";
+
+const TEST_DIR = path.join(os.tmpdir(), "restricted-mode-tests");
+const TEST_DB_DIR = path.join(TEST_DIR, "db");
+const TEST_DB_PATH = path.join(TEST_DB_DIR, "test.duckdb");
+const TEST_PKG_DIR = path.join(TEST_DIR, "pkg");
+
+let duckdbConnection: DuckDBConnection;
+
+// `widgets` is the curated, model-exposed table. `secrets` lives in the same
+// connection but is never referenced by the model the caller queries — it
+// stands in for any table the deployment did not mean to publish.
+const SEED_SQL = `
+CREATE TABLE IF NOT EXISTS widgets (
+   region VARCHAR,
+   name VARCHAR
+);
+INSERT INTO widgets VALUES
+   ('US', 'Alpha'),
+   ('EU', 'Beta'),
+   ('APAC', 'Gamma');
+
+CREATE TABLE IF NOT EXISTS secrets (
+   id VARCHAR,
+   ssn VARCHAR
+);
+INSERT INTO secrets VALUES
+   ('1', '111-11-1111'),
+   ('2', '222-22-2222');
+`;
+
+// The model the ad-hoc queries are issued against. It publishes `widgets` and
+// nothing else — `secrets` is deliberately absent.
+const CATALOG_MODEL = `
+source: widgets is duckdb.table('widgets') extend {
+   measure: n is count()
+   view: by_region is {
+      group_by: region
+      aggregate: n
+   }
+}
+`;
+
+// A second model that DOES expose secrets. It is never loaded by the caller;
+// it exists only as the target of an `import` escalation attempt.
+const VAULT_MODEL = `
+source: vault is duckdb.table('secrets') extend {
+   measure: n is count()
+}
+`;
+
+beforeAll(async () => {
+   await fs.mkdir(TEST_DB_DIR, { recursive: true });
+   await fs.mkdir(TEST_PKG_DIR, { recursive: true });
+   duckdbConnection = new DuckDBConnection("duckdb", TEST_DB_PATH, TEST_DB_DIR);
+   for (const stmt of SEED_SQL.trim().split(";").filter(Boolean)) {
+      await duckdbConnection.runSQL(stmt.trim() + ";");
+   }
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "catalog.malloy"),
+      CATALOG_MODEL,
+      "utf-8",
+   );
+   await fs.writeFile(
+      path.join(TEST_PKG_DIR, "vault.malloy"),
+      VAULT_MODEL,
+      "utf-8",
+   );
+});
+
+afterAll(async () => {
+   try {
+      await duckdbConnection.close();
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      await fs.rm(TEST_DIR, { recursive: true, force: true });
+   } catch {
+      // Ignore cleanup errors
+   }
+});
+
+function getConnections(): Map<string, Connection> {
+   const map = new Map<string, Connection>();
+   map.set("duckdb", duckdbConnection);
+   return map;
+}
+
+type Row = Record<string, unknown>;
+
+function asRows(compactResult: unknown): Row[] {
+   return compactResult as Row[];
+}
+
+async function makeModel(modelPath: string): Promise<Model> {
+   return Model.create("test-pkg", TEST_PKG_DIR, modelPath, getConnections());
+}
+
+/** Run an ad-hoc query string and return the result rows. */
+async function runAdHoc(model: Model, query: string): Promise<Row[]> {
+   const { compactResult } = await model.getQueryResults(
+      undefined,
+      undefined,
+      query,
+   );
+   return asRows(compactResult);
+}
+
+/**
+ * Restricted-mode rejections surface as a Malloy compile error: the message
+ * quotes the offending source text and states the rule, and the underlying
+ * `problems` carry `code: 'restricted-construct-forbidden'`. We accept either
+ * signal so the assertion is robust to how `model.ts` re-wraps the error.
+ */
+function looksRestricted(error: unknown): boolean {
+   const message = ((error as Error)?.message ?? String(error)).toLowerCase();
+   if (message.includes("restricted")) return true;
+   const problems = (error as { problems?: Array<{ code?: string }> })
+      ?.problems;
+   return (
+      Array.isArray(problems) &&
+      problems.some((p) => (p.code ?? "").includes("restricted"))
+   );
+}
+
+/**
+ * Assert an untrusted ad-hoc query is blocked before it can reach unpublished
+ * data. If it instead succeeds, report the row count — the caller escaped the
+ * curated surface and that is the leak we are guarding against.
+ */
+async function expectBlocked(model: Model, query: string): Promise<void> {
+   let leakedRows: number | undefined;
+   try {
+      const rows = await runAdHoc(model, query);
+      leakedRows = rows.length;
+   } catch (error) {
+      expect(looksRestricted(error)).toBe(true);
+      return;
+   }
+   throw new Error(
+      `Expected the query to be blocked by restricted mode, but it succeeded ` +
+         `and returned ${leakedRows} rows (escaped the curated surface).`,
+   );
+}
+
+/**
+ * Same assertion as `expectBlocked`, but exercised through the named
+ * `sourceName`/`queryName` request shape rather than the free-form `query`
+ * field — those identifiers are concatenated into a `run: …` string and so are
+ * just as caller-controlled.
+ */
+async function expectNamedBlocked(
+   model: Model,
+   sourceName: string | undefined,
+   queryName: string,
+): Promise<void> {
+   let leakedRows: number | undefined;
+   try {
+      const { compactResult } = await model.getQueryResults(
+         sourceName,
+         queryName,
+      );
+      leakedRows = asRows(compactResult).length;
+   } catch (error) {
+      expect(looksRestricted(error)).toBe(true);
+      return;
+   }
+   throw new Error(
+      `Expected the named-path request to be blocked by restricted mode, but ` +
+         `it succeeded and returned ${leakedRows} rows (escaped the curated surface).`,
+   );
+}
+
+// ===========================================================================
+// The published surface stays fully usable — restriction must not break the
+// legitimate path it is wrapped around.
+// ===========================================================================
+
+describe("the curated model surface stays usable under restriction", () => {
+   it("runs an ad-hoc query over a published source", async () => {
+      const model = await makeModel("catalog.malloy");
+      const rows = await runAdHoc(
+         model,
+         "run: widgets -> { group_by: region; aggregate: n is count() }",
+      );
+      expect(rows.length).toBe(3); // US, EU, APAC
+   });
+
+   it("runs a published named view", async () => {
+      const model = await makeModel("catalog.malloy");
+      const rows = await runAdHoc(model, "run: widgets -> by_region");
+      expect(rows.length).toBe(3);
+   });
+});
+
+// ===========================================================================
+// Misuse vectors: an untrusted query trying to read `secrets`, which the
+// catalog model never published. Each must be blocked.
+// ===========================================================================
+
+describe("an untrusted query cannot reach data the model never published", () => {
+   // The connection can see every table; the model curated only `widgets`.
+   // Naming another table directly would turn the model into a handle on the
+   // whole warehouse.
+   it("cannot point a source at an arbitrary warehouse table", async () => {
+      const model = await makeModel("catalog.malloy");
+      await expectBlocked(
+         model,
+         "run: duckdb.table('secrets') -> { aggregate: c is count() }",
+      );
+   });
+
+   // Raw SQL would let the caller run anything the connection's credentials
+   // allow — arbitrary reads, cross-table joins, even writes on a writable role.
+   it("cannot execute arbitrary SQL against the connection", async () => {
+      const model = await makeModel("catalog.malloy");
+      await expectBlocked(
+         model,
+         'run: duckdb.sql("SELECT id, ssn FROM secrets") -> { group_by: ssn }',
+      );
+   });
+
+   // Importing another model would pull in surfaces the queried model chose not
+   // to expose (and the file path is caller-controlled).
+   it("cannot import another model to borrow its surface", async () => {
+      const model = await makeModel("catalog.malloy");
+      await expectBlocked(
+         model,
+         'import "vault.malloy"\n' +
+            "run: vault -> { aggregate: c is count() }",
+      );
+   });
+
+   // Combining the curated surface with a raw table — joining `secrets` onto the
+   // published `widgets` — must not slip a raw table past the restriction.
+   it("cannot smuggle a raw table in through a join on a published source", async () => {
+      const model = await makeModel("catalog.malloy");
+      await expectBlocked(
+         model,
+         "source: x is widgets extend {\n" +
+            "   join_cross: s is duckdb.table('secrets')\n" +
+            "}\n" +
+            "run: x -> { group_by: s.ssn }",
+      );
+   });
+});
+
+// ===========================================================================
+// The named `sourceName`/`queryName` request shape reaches the same compiler
+// path as ad-hoc text, so it must inherit the same restriction. A real name is
+// a bare identifier; anything that smuggles in a disallowed construct must be
+// blocked, while legitimate published names keep working.
+// ===========================================================================
+
+describe("the named source/view path is restricted too", () => {
+   it("blocks a disallowed construct supplied through the sourceName/queryName fields", async () => {
+      const model = await makeModel("catalog.malloy");
+      await expectNamedBlocked(
+         model,
+         "duckdb.table('secrets')",
+         "{ group_by: ssn }",
+      );
+   });
+
+   it("still runs a legitimate published source and view by name", async () => {
+      const model = await makeModel("catalog.malloy");
+      const { compactResult } = await model.getQueryResults(
+         "widgets",
+         "by_region",
+      );
+      expect(asRows(compactResult).length).toBe(3);
+   });
+});

package/src/service/source_extraction.ts

@@ -0,0 +1,226 @@
+/**
+ * Shared source / query introspection extracted from a compiled `ModelDef`.
+ *
+ * Both the in-process `Model.create` path (`service/model.ts`) and the
+ * package-load worker (`package_load/package_load_worker.ts`, which runs in a
+ * separate bundle and serializes the result over the worker protocol) need to
+ * walk a `ModelDef` and produce the same `sources` / `queries` shapes plus the
+ * `#(filter)` `filterMap`. These two call sites used to carry byte-for-byte
+ * copies of this logic; keeping them in lockstep by hand was a standing hazard
+ * (a change to one silently diverged from the other). This module is the single
+ * source of truth — the two callers differ only in how they type the result
+ * (generated API types vs. worker wire types — structurally identical, so each
+ * casts at its boundary) and in how they report a filter parse failure (the
+ * service logs a warning; the worker has no logger and stays silent), which is
+ * threaded through the optional `onParseError` callback.
+ */
+
+import {
+   isSourceDef,
+   ModelDef,
+   NamedModelObject,
+   NamedQueryDef,
+   StructDef,
+   TurtleDef,
+} from "@malloydata/malloy";
+import { annotationTexts, modelAnnotations } from "./annotations";
+import { collectAuthorizeExprs, type AuthorizeMap } from "./authorize";
+import { parseFilters, type FilterDefinition } from "./filter";
+
+/** A `#(filter)` definition enriched with the dimension's Malloy type. */
+export interface ExtractedFilter {
+   name: string;
+   dimension: string;
+   type: string;
+   implicit: boolean;
+   required: boolean;
+   dimensionType: string | undefined;
+}
+
+export interface ExtractedView {
+   name: string;
+   annotations: string[] | undefined;
+}
+
+/**
+ * Structural source shape both callers cast to their own typed view
+ * (`ApiSource` in the service, `ApiSourceWire` in the worker). `givens` is
+ * attached verbatim from the caller-supplied list, so it stays `unknown` here.
+ */
+export interface ExtractedSource {
+   name: string;
+   annotations: string[] | undefined;
+   views: ExtractedView[];
+   filters: ExtractedFilter[] | undefined;
+   givens: unknown;
+   /**
+    * Effective `#(authorize)` / `##(authorize)` expressions gating this source:
+    * file-level expressions first, then the source's own. Undefined when the
+    * source carries no authorize annotations. Surfaced for introspection;
+    * enforcement happens server-side.
+    */
+   authorize: string[] | undefined;
+}
+
+export interface ExtractedQuery {
+   name: string;
+   sourceName: string | undefined;
+   annotations: string[] | undefined;
+}
+
+/**
+ * Extract every source from a compiled model, parsing `#(filter)` annotations
+ * along the way.
+ *
+ * Filters are collected by walking the `annotations.inherits` chain so that
+ * filters declared on a base source flow to an extending source. The chain runs
+ * child → parent, so we collect child-first then reverse — `parseFilters` uses
+ * "last wins" dedup, which lets a child's `#(filter)` override the base's.
+ *
+ * `givens` is attached unchanged to every source (Malloy exposes givens at the
+ * model level, not per-source). `onParseError`, when supplied, is invoked with
+ * the source name and error if a source's `#(filter)` annotations fail to parse;
+ * filter extraction then continues. Authorize parse errors are NOT routed here —
+ * they propagate (a malformed gate fails model load) so a security gate is never
+ * silently dropped.
+ *
+ * Authorize (`#(authorize)` / `##(authorize)`) is collected from the source's
+ * own `blockNotes` only — we do NOT walk the `inherits` chain. Note Malloy's
+ * behavior for `X is Y extend {...}`: if X declares its own `#(authorize)`,
+ * X.blockNotes holds only X's gates (Y's are dropped — the intended "curated
+ * re-exposure"); if X declares none, Malloy surfaces Y's blockNotes on X, so
+ * the base gate carries to the un-annotated extension (a safe default — a
+ * locked base stays locked unless an extension explicitly re-exposes itself).
+ * This carry happens through `blockNotes`, not the `inherits` chain, so reading
+ * own-blockNotes is sufficient. Joins are a separate concern and are not gated.
+ * The effective list per source is the file-level `##(authorize)` expressions
+ * (from `modelDef.annotations.notes`) followed by the source's own
+ * `#(authorize)` expressions, evaluated as one OR disjunction at request time.
+ */
+export function extractSourcesFromModelDef(
+   modelDef: ModelDef,
+   givens: unknown,
+   onParseError?: (sourceName: string, err: unknown) => void,
+): {
+   sources: ExtractedSource[];
+   filterMap: Map<string, FilterDefinition[]>;
+   authorizeMap: AuthorizeMap;
+} {
+   const filterMap = new Map<string, FilterDefinition[]>();
+   const authorizeMap: AuthorizeMap = new Map();
+
+   // File-level ##(authorize) is collected once and prepended to every source.
+   // Unlike filters, a malformed authorize annotation is NOT swallowed: the
+   // parse error propagates so the model fails to load loudly (caught per-model
+   // upstream and turned into a compilationError). Silently dropping a gate —
+   // and in the worker path there is no onParseError callback, so it would be
+   // truly silent — could leave a source that the author meant to lock looking
+   // unrestricted.
+   const fileLevelAuthorize = collectAuthorizeExprs(
+      (modelAnnotations(modelDef).notes ?? []).map((note) => note.text),
+   );
+
+   const sources: ExtractedSource[] = Object.values(modelDef.contents)
+      .filter((obj) => isSourceDef(obj))
+      .map((sourceObj) => {
+         const struct = sourceObj as StructDef;
+         const sourceName = struct.as || struct.name;
+         const annotations = annotationTexts(struct.annotations);
+
+         const collected: string[][] = [];
+         let cur = struct.annotations;
+         while (cur) {
+            if (cur.blockNotes) {
+               collected.push(cur.blockNotes.map((note) => note.text));
+            }
+            cur = cur.inherits;
+         }
+         const allAnnotations = collected.reverse().flat();
+
+         let filters: ExtractedFilter[] | undefined;
+         if (allAnnotations.length > 0) {
+            try {
+               const parsed = parseFilters(allAnnotations);
+               if (parsed.length > 0) {
+                  filterMap.set(sourceName, parsed);
+                  const fields = struct.fields;
+                  filters = parsed.map((f) => {
+                     const field = fields.find(
+                        (fd) => (fd.as || fd.name) === f.dimension,
+                     );
+                     return {
+                        name: f.name,
+                        dimension: f.dimension,
+                        type: f.type,
+                        implicit: f.implicit,
+                        required: f.required,
+                        dimensionType: field?.type as string | undefined,
+                     };
+                  });
+               }
+            } catch (err) {
+               onParseError?.(sourceName, err);
+            }
+         }
+
+         // Authorize: the source's OWN #(authorize) annotations only — no
+         // inherits walk. File-level ##(authorize) is prepended so file gates
+         // and source gates form one OR disjunction. A malformed annotation
+         // propagates (model fails to load) rather than silently dropping the
+         // gate — see the file-level note above.
+         const ownNotes = (struct.annotations?.blockNotes ?? []).map(
+            (note) => note.text,
+         );
+         const effective = [
+            ...fileLevelAuthorize,
+            ...collectAuthorizeExprs(ownNotes),
+         ];
+         let authorize: string[] | undefined;
+         if (effective.length > 0) {
+            authorizeMap.set(sourceName, effective);
+            authorize = effective;
+         }
+
+         const views: ExtractedView[] = struct.fields
+            .filter((field) => field.type === "turtle")
+            .filter((turtle) =>
+               // Filter out non-reduce views (e.g. indexes).
+               (turtle as TurtleDef).pipeline
+                  .map((stage) => stage.type)
+                  .every((type) => type === "reduce"),
+            )
+            .map((turtle) => ({
+               name: turtle.as || turtle.name,
+               annotations: annotationTexts(turtle.annotations),
+            }));
+
+         return {
+            name: sourceName,
+            annotations,
+            views,
+            filters,
+            givens,
+            authorize,
+         };
+      });
+
+   return { sources, filterMap, authorizeMap };
+}
+
+/** Extract every named query from a compiled model. */
+export function extractQueriesFromModelDef(
+   modelDef: ModelDef,
+): ExtractedQuery[] {
+   const isNamedQuery = (obj: NamedModelObject): obj is NamedQueryDef =>
+      obj.type === "query";
+   return Object.values(modelDef.contents)
+      .filter(isNamedQuery)
+      .map((queryObj) => ({
+         name: queryObj.as || queryObj.name,
+         sourceName:
+            typeof queryObj.structRef === "string"
+               ? queryObj.structRef
+               : undefined,
+         annotations: annotationTexts(queryObj.annotations),
+      }));
+}