@malloy-publisher/server 0.0.228 → 0.0.230

package/src/mcp/skills/skills_bundle.spec.ts

@@ -1,4 +1,6 @@
 import { describe, expect, it } from "bun:test";
+import * as path from "node:path";
+import { buildSkills, type SkillEntry } from "./build_skills_bundle";
 import bundle from "./skills_bundle.json";
 
 const skills = (
@@ -7,9 +9,36 @@ const skills = (
    }
 ).skills;
 
+/** The repo's top-level skills/, which this bundle is generated from. */
+const sourceDir = path.join(
+   import.meta.dir,
+   "..",
+   "..",
+   "..",
+   "..",
+   "..",
+   "skills",
+);
+
+/**
+ * Codepoint order. The committed bundle is sorted with localeCompare, which
+ * depends on the runtime's locale, and this suite runs on three platforms.
+ * Membership and content are what matter here; file order is cosmetic.
+ */
+const byName = (a: SkillEntry, b: SkillEntry) =>
+   a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
+
 describe("skills_bundle.json (generated dual-channel asset)", () => {
-   it("contains the full ported skill set", () => {
-      expect(skills.length).toBeGreaterThanOrEqual(24);
+   it("is in sync with skills/", () => {
+      // Regenerate with:
+      //   bun run src/mcp/skills/build_skills_bundle.ts ../../skills
+      expect([...skills].sort(byName)).toEqual(
+         [...buildSkills(sourceDir)].sort(byName),
+      );
+   });
+
+   it("is not empty", () => {
+      expect(skills.length).toBeGreaterThan(0);
    });
 
    it("every skill has a nonempty name, description, and body", () => {

package/src/query_param_utils.ts

@@ -16,3 +16,14 @@ export function normalizeQueryArray(value: unknown): string[] | undefined {
    if (Array.isArray(value)) return value.map(String);
    return [String(value)];
 }
+
+/**
+ * Parse an Express query param as a non-negative integer, or `undefined` when
+ * it is absent or not a finite integer. Degrades a garbage value (`?limit=abc`)
+ * to "unset" rather than passing `NaN` down into a SQL `LIMIT`/`OFFSET` bind.
+ */
+export function parseNonNegativeIntParam(value: unknown): number | undefined {
+   if (value === undefined || value === null) return undefined;
+   const parsed = parseInt(String(value), 10);
+   return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined;
+}

package/src/server.ts

@@ -34,7 +34,11 @@ import {
 } from "./instrumentation";
 import { logger, loggerMiddleware } from "./logger";
 
-import { getMemoryGovernorConfig } from "./config";
+import {
+   getExtensionFetchPolicy,
+   getMaterializationSchedulerConfig,
+   getMemoryGovernorConfig,
+} from "./config";
 import { setFilterDeprecationHeaders } from "./filter_deprecation";
 import { checkHeapConfiguration } from "./heap_check";
 import { queryConcurrency } from "./query_concurrency";
@@ -43,8 +47,12 @@ import { ThemeController } from "./controller/theme.controller";
 import { initializeMcpServer } from "./mcp/server";
 import { registerLegacyRoutes } from "./server-old";
 import { EnvironmentStore } from "./service/environment_store";
+import { MaterializationScheduler } from "./service/materialization_scheduler";
 import { MaterializationService } from "./service/materialization_service";
-import { normalizeQueryArray } from "./query_param_utils";
+import {
+   normalizeQueryArray,
+   parseNonNegativeIntParam,
+} from "./query_param_utils";
 import { PackageMemoryGovernor } from "./service/package_memory_governor";
 import { ThemeStore } from "./service/theme_store";
 import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
@@ -183,7 +191,7 @@ app.use(httpMetricsMiddleware);
 // looks low for the default caps" advisory so operators don't
 // chase OOMKills before checking the obvious config.
 checkHeapConfiguration();
-const environmentStore = new EnvironmentStore(SERVER_ROOT);
+export const environmentStore = new EnvironmentStore(SERVER_ROOT);
 const watchModeController = new WatchModeController(environmentStore);
 const connectionController = new ConnectionController(environmentStore);
 const modelController = new ModelController(environmentStore);
@@ -192,6 +200,12 @@ const modelController = new ModelController(environmentStore);
 // that Environment.getPackage / addPackage consult before allocating
 // any new package — the server responds with HTTP 503 instead of
 // OOM-killing the pod.
+// Validate the DuckDB extension-fetch policy at boot so an unrecognised value
+// (e.g. a k8s-manifest typo) fails the boot loudly here, rather than surfacing
+// on the first query that resolves a DuckDB connection — which matters most for
+// an operator relying on `local-only` for a no-network guarantee. Logging the
+// resolved policy also records the posture the server booted with.
+logger.info(`DuckDB extension-fetch policy: ${getExtensionFetchPolicy()}`);
 const memoryGovernorConfig = getMemoryGovernorConfig();
 const memoryGovernor = memoryGovernorConfig
    ? new PackageMemoryGovernor(memoryGovernorConfig)
@@ -206,6 +220,32 @@ const materializationService = new MaterializationService(environmentStore);
 const materializationController = new MaterializationController(
    materializationService,
 );
+/**
+ * Construct and start the standalone materialization scheduler from environment
+ * config, or return null when the feature is disabled
+ * (`PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` unset/false — the default, so an
+ * orchestrated deployment never runs it). Extracted from the module body so a
+ * test can drive the real env-var → {@link getMaterializationSchedulerConfig} →
+ * construct → `start()`/`unref()` → timer path an operator uses; the
+ * module-level singleton below is armed once at import and can't be re-created
+ * per test.
+ */
+export function startMaterializationSchedulerFromEnv(
+   store: EnvironmentStore,
+   service: MaterializationService,
+): MaterializationScheduler | null {
+   const config = getMaterializationSchedulerConfig();
+   if (!config) return null;
+   const scheduler = new MaterializationScheduler(store, service, config);
+   scheduler.start();
+   return scheduler;
+}
+
+// Standalone materialization scheduler: opt-in via
+// PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER (default off, so an orchestrated
+// deployment — whose control plane drives materialization — never runs it). The
+// sweep timer is `unref`'d, so it never keeps the process alive on shutdown.
+startMaterializationSchedulerFromEnv(environmentStore, materializationService);
 const themeStore = new ThemeStore(environmentStore.storageManager, SERVER_ROOT);
 const themeController = new ThemeController(themeStore, SERVER_ROOT);
 
@@ -1319,6 +1359,31 @@ app.post(
    },
 );
 
+// Environment-scoped aggregate: every materialization across all packages in
+// the env, newest first. Nested under `/packages` as the collection-level
+// sibling of the per-package `/packages/:packageName/materializations` list.
+// MUST stay registered ahead of `/packages/:packageName` below so the literal
+// `materializations` segment wins the match; consequently `materializations` is
+// a reserved package name at this position (a package can never be named that).
+app.get(
+   `${API_PREFIX}/environments/:environmentName/packages/materializations`,
+   async (req, res) => {
+      try {
+         const limit = parseNonNegativeIntParam(req.query.limit);
+         const offset = parseNonNegativeIntParam(req.query.offset);
+         const builds =
+            await materializationController.listEnvironmentMaterializations(
+               req.params.environmentName,
+               { limit, offset },
+            );
+         res.status(200).json(builds);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 app.get(
    `${API_PREFIX}/environments/:environmentName/packages/:packageName`,
    async (req, res) => {
@@ -1636,6 +1701,10 @@ app.post(
 );
 
 // ==================== MATERIALIZATION ROUTES ====================
+// The environment-scoped aggregate list (every materialization across all
+// packages) is registered up in the package routes as
+// `/packages/materializations`, ahead of `/packages/:packageName`, so the
+// literal wins the match — see that route for the ordering contract.
 
 app.post(
    `${API_PREFIX}/environments/:environmentName/packages/:packageName/materializations`,
@@ -1658,12 +1727,8 @@ app.get(
    `${API_PREFIX}/environments/:environmentName/packages/:packageName/materializations`,
    async (req, res) => {
       try {
-         const limit = req.query.limit
-            ? parseInt(req.query.limit as string, 10)
-            : undefined;
-         const offset = req.query.offset
-            ? parseInt(req.query.offset as string, 10)
-            : undefined;
+         const limit = parseNonNegativeIntParam(req.query.limit);
+         const offset = parseNonNegativeIntParam(req.query.offset);
          const builds = await materializationController.listMaterializations(
             req.params.environmentName,
             req.params.packageName,

package/src/service/authorize.spec.ts

@@ -3,8 +3,30 @@ import {
    collectAuthorizeExprs,
    isProbeTrue,
    parseAuthorizeAnnotation,
+   referencedGivenNames,
 } from "./authorize";
 
+describe("referencedGivenNames", () => {
+   it("returns the $NAME tokens deduped in first-seen order", () => {
+      expect(
+         referencedGivenNames("$ROLE = 'admin' and $ROLE != $PRIOR"),
+      ).toEqual(["ROLE", "PRIOR"]);
+   });
+
+   it("ignores a $NAME inside a string literal (not a real reference)", () => {
+      // Otherwise a joined gate's referenced-given count is inflated and the
+      // full-coverage check wrongly denies a correctly-authorized request.
+      expect(referencedGivenNames("$ROLE = 'the $BOSS role'")).toEqual([
+         "ROLE",
+      ]);
+      expect(referencedGivenNames("'$A $B $C'")).toEqual([]);
+      expect(referencedGivenNames("$X = 'it\\'s $Y' or $Z = 1")).toEqual([
+         "X",
+         "Z",
+      ]);
+   });
+});
+
 describe("isProbeTrue", () => {
    it("grants only on a genuine true / 1 / 'true'", () => {
       expect(isProbeTrue(true)).toBe(true);

package/src/service/authorize.ts

@@ -27,6 +27,12 @@ const FILE_PREFIX = "##(authorize)";
 /** source name → effective authorize expressions (file-level then source-level). */
 export type AuthorizeMap = Map<string, string[]>;
 
+/** A `given:` declaration to prepend to a probe so it compiles standalone. */
+export interface ProbeGivenDecl {
+   name: string;
+   type: string;
+}
+
 /**
  * Build the synthetic probe query that evaluates a source's authorize
  * expressions. Each expression becomes a boolean `select` column over a
@@ -37,18 +43,119 @@ export type AuthorizeMap = Map<string, string[]>;
  * compile errors); running it evaluates the gate. The reserved dummy column
  * name is deliberately obscure so a real authorize expression is unlikely to
  * collide with it — a bare field reference in an expression is meant to fail.
+ *
+ * `givenDecls`, when supplied, are prepended as the probe's OWN `given:`
+ * block, making the probe self-contained: it compiles against whatever
+ * givens it declares itself rather than depending on the ambient model's
+ * given namespace. This is what lets {@link evaluateAuthorize} gate a joined
+ * source whose givens live in a model that isn't the one the probe is
+ * compiled against (see the two-hop transitive-import case in
+ * `docs/authorize.md`). Compile-time validation (`validateAuthorizeProbes`)
+ * calls this with no decls, so it still validates against the ambient
+ * namespace of the model it's compiling in.
  */
-export function buildAuthorizeProbe(exprs: string[]): string {
+export function buildAuthorizeProbe(
+   exprs: string[],
+   givenDecls: ProbeGivenDecl[] = [],
+): string {
    const selects = exprs
       .map((expr, i) => `__auth_${i} is (${expr})`)
       .join("\n      ");
-   return `run: duckdb.sql("SELECT 1 AS __authorize_probe_row") -> {
+   const givenBlock =
+      givenDecls.length > 0
+         ? `given:\n${givenDecls.map((g) => `  ${g.name} :: ${g.type}`).join("\n")}\n\n`
+         : "";
+   return `${givenBlock}run: duckdb.sql("SELECT 1 AS __authorize_probe_row") -> {
     select:
       ${selects}
     limit: 1
   }`;
 }
 
+const GIVEN_REF_PATTERN = /\$([A-Za-z_][A-Za-z0-9_]*)/g;
+// Malloy string literals are single-quoted; a `$NAME` inside one is literal
+// text, not a given reference. Strip literals (honoring `\'` escapes) before
+// scanning, so e.g. `$ROLE = 'the $BOSS role'` references only ROLE — otherwise
+// a joined gate's referenced-count is inflated and the full-coverage check
+// wrongly denies a correctly-authorized request.
+const STRING_LITERAL_PATTERN = /'(?:\\.|[^'\\])*'/g;
+
+/**
+ * Given names an authorize expression references (`$NAME` tokens), deduped,
+ * in first-seen order. Used to figure out which givens a self-contained
+ * probe needs to declare for a given expression.
+ */
+export function referencedGivenNames(expr: string): string[] {
+   const scanned = expr.replace(STRING_LITERAL_PATTERN, "''");
+   const names: string[] = [];
+   const seen = new Set<string>();
+   for (const match of scanned.matchAll(GIVEN_REF_PATTERN)) {
+      const name = match[1];
+      if (!seen.has(name)) {
+         seen.add(name);
+         names.push(name);
+      }
+   }
+   return names;
+}
+
+/**
+ * Infer a Malloy given type from a caller-supplied JS value, so a
+ * self-contained probe can declare exactly the givens an expression
+ * references without depending on the ambient model's own given namespace —
+ * which a joined source pulled in from a different (possibly multi-hop
+ * transitively imported) file may not share. Returns `null` for a value with
+ * no sensible Malloy type (null, a plain object, an empty untyped array);
+ * callers skip declaring that given, which fails the probe closed —
+ * consistent with the existing "referenced given has no value" behavior.
+ */
+function inferGivenType(value: GivenValue): string | null {
+   if (typeof value === "string") return "string";
+   if (typeof value === "number" || typeof value === "bigint") return "number";
+   if (typeof value === "boolean") return "boolean";
+   if (value instanceof Date) return "timestamp";
+   if (Array.isArray(value)) {
+      if (value.length === 0) return null;
+      const elementType = inferGivenType(value[0]);
+      return elementType ? `${elementType}[]` : null;
+   }
+   return null;
+}
+
+/**
+ * Build the self-contained `given:` declarations + bound values a probe
+ * needs for one expression, from whatever the caller supplied. A referenced
+ * name that's absent from `givens` (or whose value has no inferrable type) is
+ * simply left undeclared — the probe then fails to compile for that
+ * expression (an undeclared `$NAME`), which {@link evaluateAuthorize} already
+ * treats as "this disjunct can't be evaluated" and denies that branch.
+ *
+ * `declaredTypes`, when supplied, is preferred over inferring the type from
+ * the caller's JS value — the gate author's own `given:` declaration is the
+ * source of truth (e.g. `$LEVEL > 3` should compare numerically even if the
+ * caller sends `"5"` as a string). Only covers a name declared within one
+ * import hop of the entry model (the same reach as `this.givens` — see
+ * `docs/authorize.md`); a gate on a source reached through a deeper
+ * transitive import falls back to inferring from the value, same as before.
+ */
+function bindProbeGivens(
+   expr: string,
+   givens: Record<string, GivenValue>,
+   declaredTypes?: Map<string, string>,
+): { decls: ProbeGivenDecl[]; bound: Record<string, GivenValue> } {
+   const decls: ProbeGivenDecl[] = [];
+   const bound: Record<string, GivenValue> = {};
+   for (const name of referencedGivenNames(expr)) {
+      if (!(name in givens)) continue;
+      const value = givens[name];
+      const type = declaredTypes?.get(name) ?? inferGivenType(value);
+      if (!type) continue;
+      decls.push({ name, type });
+      bound[name] = value;
+   }
+   return { decls, bound };
+}
+
 /**
  * Strict, fail-closed truthiness for a probe result cell. DuckDB (the probe's
  * connection) returns a native boolean, but normalize defensively: only a real
@@ -74,6 +181,25 @@ interface AuthorizeProbeExecutor {
    };
 }
 
+/**
+ * Run one probe (already-built query text) and report whether it evaluated
+ * true. Any throw (compile error, runtime "no value" for a referenced given,
+ * a missing/malformed result) propagates to the caller, which decides how to
+ * react — {@link evaluateAuthorize} treats it as "can't evaluate this way,
+ * try something else" rather than granting.
+ */
+async function runProbe(
+   executor: AuthorizeProbeExecutor,
+   probeText: string,
+   givens: Record<string, GivenValue>,
+): Promise<boolean> {
+   const result = await executor
+      .loadQuery(probeText)
+      .run({ rowLimit: 1, givens });
+   const row = result?.data?.value?.[0];
+   return !!(row && isProbeTrue(row.__auth_0));
+}
+
 /**
  * Evaluate a source's authorize disjunction against the supplied givens.
  * Returns true if ANY expression evaluates true (OR semantics).
@@ -88,31 +214,162 @@ interface AuthorizeProbeExecutor {
  * Per-branch failures are swallowed to false (fail closed at the branch level);
  * access is granted only if some branch genuinely returns true. Short-circuits
  * on the first true. Returns false (→ caller denies) if none grant.
+ *
+ * Each expression is tried TWICE if needed, in one of two orders depending on
+ * `selfContainedFirst`:
+ *
+ *  - AMBIENT FIRST (`selfContainedFirst` false — the default, used only for
+ *    the run target's OWN source gate): compile against `executor`'s own
+ *    given namespace, passing the full supplied `givens` — the original
+ *    approach, unchanged for the common case where `executor` (always the
+ *    ENTRY model's materializer) already has the referenced givens in scope
+ *    (same file, or a directly imported one — Malloy merges one level of
+ *    import into a model's given namespace). Falls back to the
+ *    self-contained probe below only if the ambient probe THROWS.
+ *
+ *  - SELF-CONTAINED FIRST (`selfContainedFirst` true — used for every gate
+ *    reached via a join/derivation/composite-member walk): declare just the
+ *    givens this expression references ({@link bindProbeGivens}), preferring
+ *    `declaredTypes` (the gate author's own declaration) over inferring from
+ *    the caller-supplied value, so the probe compiles independently of
+ *    `executor`'s namespace. This is what isolates a JOINED source's gate
+ *    from the entry model's own ambient given namespace: if the entry model
+ *    happens to declare its OWN given of the SAME NAME the joined gate
+ *    references, an ambient-first probe would compile successfully against
+ *    the ENTRY model's default/value instead of failing closed — silently
+ *    granting access based on an unrelated given. An expression referencing
+ *    NO givens at all (a constant/public gate, e.g. `#(authorize) "true"`)
+ *    has nothing ambient to isolate from, so it's evaluated with a no-decls
+ *    self-contained probe and returned directly — no fallback needed.
+ *    Otherwise, the probe is attempted ONLY when EVERY referenced given got
+ *    a decl — a probe that declares just SOME of them is not actually
+ *    isolated: `executor.loadQuery` still compiles it against the ENTRY
+ *    model's own materializer, so an undeclared referenced name resolves via
+ *    the entry model's own ambient value/default for that name INSTEAD OF
+ *    THROWING — reopening the collision hole for a partially supplied
+ *    multi-given gate without ever reaching a catch block. So "not every
+ *    referenced name could be bound" denies immediately, with NO probe
+ *    attempted and NO ambient fallback. Once every referenced name IS
+ *    declared, ambient is tried as a last resort only if that fully-declared
+ *    probe itself throws for some OTHER reason — safe at that point since no
+ *    referenced name is left for an entry default to decide.
+ *
+ * Either order still lets a gate on a source reached through a multi-hop
+ * transitive import evaluate correctly — Malloy does not flatten a `given:`
+ * declaration through more than one level of import, so the entry model's own
+ * namespace can be missing a given that's declared two-or-more imports away
+ * from a JOINED source's home file (see `docs/authorize.md`).
  */
 export async function evaluateAuthorize(
    executor: AuthorizeProbeExecutor,
    exprs: string[],
    givens: Record<string, GivenValue>,
+   declaredTypes?: Map<string, string>,
+   options?: { selfContainedFirst?: boolean },
 ): Promise<boolean> {
+   const selfContainedFirst = options?.selfContainedFirst ?? false;
    for (const expr of exprs) {
+      if (selfContainedFirst) {
+         if (
+            await evaluateSelfContainedFirst(
+               executor,
+               expr,
+               givens,
+               declaredTypes,
+            )
+         ) {
+            return true;
+         }
+         continue;
+      }
       try {
-         const result = await executor
-            .loadQuery(buildAuthorizeProbe([expr]))
-            .run({ rowLimit: 1, givens });
-         const row = result?.data?.value?.[0];
-         if (row && isProbeTrue(row.__auth_0)) {
+         if (await runProbe(executor, buildAuthorizeProbe([expr]), givens)) {
             return true;
          }
+         continue;
       } catch {
-         // This disjunct can't be evaluated (e.g. a referenced given has no
-         // value) — treat as not-granting and try the next. It does not fail
-         // the whole request, which is what keeps OR semantics intact.
+         // Ambient compile/eval failed — fall through to the self-contained
+         // retry below rather than immediately treating this disjunct as
+         // not-granting.
+      }
+      try {
+         const { decls, bound } = bindProbeGivens(expr, givens, declaredTypes);
+         if (decls.length === 0) continue; // nothing left to try — deny
+         if (
+            await runProbe(executor, buildAuthorizeProbe([expr], decls), bound)
+         ) {
+            return true;
+         }
+      } catch {
+         // Still can't be evaluated — deny this branch and try the next.
+         // Does not fail the whole request, which is what keeps OR
+         // semantics intact.
          continue;
       }
    }
    return false;
 }
 
+/**
+ * One expression's self-contained-first evaluation (see
+ * {@link evaluateAuthorize}'s `selfContainedFirst` mode). A probe is only
+ * ever attempted once every given the expression references has been
+ * declared — a partially-declared probe still compiles against the ENTRY
+ * model's own materializer, so an undeclared name silently resolves via the
+ * entry's own ambient value/default instead of throwing, which is not
+ * isolation at all. Ambient is tried as a last resort only when the
+ * fully-declared self-contained probe itself throws for some other reason.
+ */
+async function evaluateSelfContainedFirst(
+   executor: AuthorizeProbeExecutor,
+   expr: string,
+   givens: Record<string, GivenValue>,
+   declaredTypes?: Map<string, string>,
+): Promise<boolean> {
+   const referenced = referencedGivenNames(expr);
+   if (referenced.length === 0) {
+      // The expression references NO givens at all — a constant/public gate
+      // (e.g. `#(authorize) "true"`). There's nothing ambient to isolate
+      // from, so run the no-decls probe (still fully self-contained: it
+      // declares no givens and is handed none) and use its result directly.
+      // This is NOT the "unsatisfiable" case below — that's specifically
+      // about a referenced given the caller can't supply.
+      try {
+         return await runProbe(executor, buildAuthorizeProbe([expr]), {});
+      } catch {
+         return false;
+      }
+   }
+   const { decls, bound } = bindProbeGivens(expr, givens, declaredTypes);
+   if (decls.length !== referenced.length) {
+      // Not every referenced given could be bound — deny. A probe that
+      // declares only SOME of the referenced names is not actually isolated:
+      // `executor.loadQuery` compiles it against the ENTRY model's own
+      // materializer, so an undeclared referenced given resolves via the
+      // ENTRY model's own ambient value/default for that name instead of
+      // throwing — reopening the name-collision hole for a partially
+      // supplied multi-given gate. Only attempt a probe once every
+      // referenced name can be declared (and thus shadowed) ourselves.
+      return false;
+   }
+   try {
+      return await runProbe(
+         executor,
+         buildAuthorizeProbe([expr], decls),
+         bound,
+      );
+   } catch {
+      // Every referenced given was supplied and declared, so a throw here
+      // isn't the collision case — there's no unsupplied name left for an
+      // entry default to decide. Ambient is a safe last resort.
+      try {
+         return await runProbe(executor, buildAuthorizeProbe([expr]), givens);
+      } catch {
+         return false;
+      }
+   }
+}
+
 /** A source plus its effective authorize expressions. */
 interface SourceWithAuthorize {
    name?: string;