@malloy-publisher/server 0.0.204 → 0.0.206

package/src/materialization_metrics.ts

@@ -0,0 +1,66 @@
+/**
+ * Centralized telemetry for the two-round materialization protocol.
+ *
+ * Operators need to answer "are builds completing, how long do the two
+ * rounds take, and where are they failing?" without grepping logs. The
+ * counter carries `round` (`round1`|`round2`) and `outcome`
+ * (`success`|`failed`|`cancelled`); the histogram carries `round` so a
+ * dashboard can render Round 1 (compile+plan) vs Round 2 (build) latency
+ * side by side.
+ *
+ * Lazy init for the same reason as {@link ./query_cap_metrics}: instruments
+ * created before `setGlobalMeterProvider` bind to a NoOp meter
+ * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
+ */
+
+import { metrics, type Counter, type Histogram } from "@opentelemetry/api";
+
+export type MaterializationRound = "round1" | "round2" | "auto";
+export type MaterializationOutcome = "success" | "failed" | "cancelled";
+
+let roundCounter: Counter | null = null;
+let roundDuration: Histogram | null = null;
+
+function ensureTelemetry(): { counter: Counter; duration: Histogram } {
+   if (roundCounter && roundDuration) {
+      return { counter: roundCounter, duration: roundDuration };
+   }
+   const meter = metrics.getMeter("publisher");
+   if (!roundCounter) {
+      roundCounter = meter.createCounter(
+         "publisher_materialization_rounds_total",
+         {
+            description:
+               "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled').",
+         },
+      );
+   }
+   if (!roundDuration) {
+      roundDuration = meter.createHistogram(
+         "publisher_materialization_round_duration_ms",
+         {
+            description:
+               "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
+            unit: "ms",
+         },
+      );
+   }
+   return { counter: roundCounter, duration: roundDuration };
+}
+
+/** Record the outcome and duration of a materialization round. */
+export function recordMaterializationRound(
+   round: MaterializationRound,
+   outcome: MaterializationOutcome,
+   durationMs: number,
+): void {
+   const { counter, duration } = ensureTelemetry();
+   counter.add(1, { round, outcome });
+   duration.record(durationMs, { round });
+}
+
+/** Visible for tests. Drops cached instruments so a fresh MeterProvider can capture emissions. */
+export function resetMaterializationTelemetryForTesting(): void {
+   roundCounter = null;
+   roundDuration = null;
+}

package/src/mcp/error_messages.spec.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "bun:test";
+import { AccessDeniedError } from "../errors";
+import { getMalloyErrorDetails } from "./error_messages";
+
+describe("getMalloyErrorDetails — access-denied branch", () => {
+   it("recognizes an authorize denial and gives access-relevant (not syntax) advice", () => {
+      const details = getMalloyErrorDetails(
+         "executeQuery",
+         "env/pkg/model.malloy",
+         new AccessDeniedError('Access denied for source "gated".'),
+      );
+
+      // Message carries the source name, never the gate expression.
+      expect(details.message).toContain('Access denied for source "gated".');
+
+      // The suggestion is about satisfying access (givens/role), and the
+      // generic Malloy-syntax suggestions are replaced, not appended.
+      expect(details.suggestions).toHaveLength(1);
+      expect(details.suggestions[0]).toMatch(/given|authorize|restricted/i);
+      // Not the generic "check the database connection / consult the language
+      // docs" advice that an unrecognized error would yield.
+      expect(details.suggestions.join(" ")).not.toMatch(
+         /database connection configuration|language documentation/i,
+      );
+   });
+
+   it("still falls back to generic suggestions for an unrecognized error", () => {
+      const details = getMalloyErrorDetails(
+         "executeQuery",
+         "env/pkg/model.malloy",
+         new Error("something unexpected"),
+      );
+      expect(details.suggestions.length).toBeGreaterThan(1);
+   });
+});

package/src/mcp/error_messages.ts

@@ -126,8 +126,21 @@ export function getMalloyErrorDetails(
       const invalidRequestMatch = error.message.match(
          /Invalid query request\\. Query OR queryName must be defined/i,
       );
+      // `#(authorize)` gate denial. The message names only the source (gate
+      // logic is never leaked), so the suggestion is about satisfying access,
+      // not fixing syntax — otherwise this falls through to generic Malloy
+      // syntax advice that misleads the caller.
+      const accessDeniedMatch = error.message.match(
+         /Access denied for source "([^"]+)"/i,
+      );
 
-      if (viewNotFoundMatch) {
+      if (accessDeniedMatch) {
+         refined = true;
+         const [, sourceName] = accessDeniedMatch;
+         suggestions = [
+            `Suggestion: Access to source '${sourceName}' is restricted by an #(authorize) gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`,
+         ];
+      } else if (viewNotFoundMatch) {
          refined = true;
          const [, viewName, sourceName] = viewNotFoundMatch;
          suggestions.unshift(

package/src/mcp/handler_utils.ts

@@ -3,6 +3,7 @@ import type { ResourceMetadata } from "@modelcontextprotocol/sdk/server/mcp";
 import type { ReadResourceResult } from "@modelcontextprotocol/sdk/types.js";
 import { EnvironmentStore } from "../service/environment_store";
 import {
+   AccessDeniedError,
    PackageNotFoundError,
    ModelNotFoundError,
    ModelCompilationError,
@@ -167,6 +168,17 @@ export async function getModelForQuery(
             `${environmentName}/${packageName}/${modelPath}`,
             error,
          );
+      } else if (error instanceof AccessDeniedError) {
+         // An #(authorize) denial during model setup. Funnel through
+         // getMalloyErrorDetails (which recognizes the access-denied message and
+         // gives supply-the-givens guidance) so a 403 never surfaces as an
+         // opaque internal error. Defensive: the gate fires during query
+         // execution today, not setup, but this keeps every error class homed.
+         errorDetails = getMalloyErrorDetails(
+            "executeQuery (load model)",
+            `${environmentName}/${packageName}/${modelPath}`,
+            error,
+         );
       } else if (error instanceof ServiceUnavailableError) {
          // Back-pressure: don't dress this up as a 404/500. Surface the
          // server's own message so the MCP caller knows to retry.

package/src/package_load/package_load_pool.ts

@@ -228,7 +228,13 @@ export interface LoadPackageJob {
  * don't allocate a Map for models the caller never looks at.
  */
 export interface LoadPackageOutcome {
-   packageMetadata: { name?: string; description?: string };
+   packageMetadata: {
+      name?: string;
+      description?: string;
+      explores?: string[];
+      queryableSources?: "declared" | "all";
+      manifestLocation?: string | null;
+   };
    models: Array<
       Omit<SerializedModel, "modelDef" | "sourceInfos"> & {
          modelDef?: ModelDef;

package/src/package_load/package_load_worker.ts

@@ -75,6 +75,7 @@ import { fileURLToPath, pathToFileURL } from "url";
 
 import {
    MODEL_FILE_SUFFIX,
+   normalizeModelPath,
    NOTEBOOK_FILE_SUFFIX,
    PACKAGE_MANIFEST_NAME,
 } from "../constants";
@@ -375,16 +376,38 @@ function buildWorkerMalloyConfig(job: LoadPackageRequest): MalloyConfig {
 // stays decoupled from the main-thread service module graph)
 // ──────────────────────────────────────────────────────────────────────
 
-async function readPackageMetadata(
-   packagePath: string,
-): Promise<{ name?: string; description?: string }> {
+async function readPackageMetadata(packagePath: string): Promise<{
+   name?: string;
+   description?: string;
+   explores?: string[];
+   queryableSources?: "declared" | "all";
+   manifestLocation?: string | null;
+}> {
    const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
    const contents = await fs.promises.readFile(manifestPath, "utf8");
    const parsed = JSON.parse(contents) as {
       name?: string;
       description?: string;
+      explores?: string[];
+      queryableSources?: unknown;
+      manifestLocation?: unknown;
+   };
+   return {
+      name: parsed.name,
+      description: parsed.description,
+      explores: Array.isArray(parsed.explores)
+         ? parsed.explores.map(normalizeModelPath)
+         : undefined,
+      // Default + invalid fall back to "declared" (fail-safe: queryable ==
+      // discoverable). Only an explicit "all" opts out of the query boundary.
+      queryableSources: parsed.queryableSources === "all" ? "all" : "declared",
+      // URI (gs://, s3://, file://, or local path) of the control-plane-computed
+      // build manifest. The main thread fetches + binds it after load.
+      manifestLocation:
+         typeof parsed.manifestLocation === "string"
+            ? parsed.manifestLocation
+            : null,
    };
-   return { name: parsed.name, description: parsed.description };
 }
 
 async function listPackageFiles(packagePath: string): Promise<string[]> {
@@ -400,6 +423,18 @@ function filterModelPaths(allRelative: string[]): string[] {
    );
 }
 
+// `normalizeModelPath` (shared, from ../constants) runs here at parse time so
+// the keys stored in Package.models are already normalized, and the API
+// publish/update path normalizes its `explores` input through the same helper —
+// so on-disk and API-written manifests share one representation.
+
+// explores validation is intentionally NOT done here. The worker is the
+// shared load path (startup, reload, AND publish), but the policy differs by
+// context — strict-reject at publish, warn-and-fail-safe at load — so it lives
+// on the main thread (see Package.getInvalidExplores / loadViaWorker and the
+// publish path in package.controller). An unmatched entry simply lists nothing
+// (listModels filters by set membership), which is the fail-safe outcome.
+
 // ──────────────────────────────────────────────────────────────────────
 // Model compile (mirrors service/model.ts but produces SerializedModel)
 // ──────────────────────────────────────────────────────────────────────
@@ -558,6 +593,10 @@ async function compileMalloyModel(
       modelDef,
       modelInfo: modelDefToModelInfo(modelDef),
       sourceInfos,
+      // `sources`/`queries` ship complete (authorize + filter enforcement and
+      // join resolution read the full set); the Model's discovery accessors
+      // filter them down to the export closure (`modelDef.exports`) to match
+      // `modelInfo`/`sourceInfos`.
       sources,
       queries,
       filterMap: Array.from(filterMap.entries()),
@@ -800,6 +839,7 @@ async function loadPackage(
 
    const allFiles = await listPackageFiles(job.packagePath);
    const modelPaths = filterModelPaths(allFiles);
+
    const models = await Promise.all(
       modelPaths.map((modelPath) =>
          compileOneModel(job, malloyConfig, modelPath),

package/src/package_load/protocol.ts

@@ -173,7 +173,13 @@ export interface SerializedNotebookCell {
 export interface LoadPackageResult {
    type: "load-package-result";
    requestId: string;
-   packageMetadata: { name?: string; description?: string };
+   packageMetadata: {
+      name?: string;
+      description?: string;
+      explores?: string[];
+      queryableSources?: "declared" | "all";
+      manifestLocation?: string | null;
+   };
    models: SerializedModel[];
    /** Wall-clock ms inside the worker for the full package load. */
    loadDurationMs: number;

package/src/runtime/publisher.js

@@ -0,0 +1,318 @@
+// Publisher runtime helper for in-package HTML dashboards.
+// Served by the Publisher server at /sdk/publisher.js. Hand-authored vanilla
+// JS — no bundler. Loaded via <script src="/sdk/publisher.js">.
+//
+// Exposes window.Publisher with:
+//   - Publisher.query(model, malloy, opts?)     → Promise<rows[]>
+//   - Publisher.queryFull(model, malloy, opts?) → Promise<MalloyResult>  (envelope for <malloy-render>)
+//   - Publisher.embed(selector, { src, height?, token? })
+//   - Publisher.context  ({ environment, package } inferred from URL)
+//   - Publisher.setToken(token)  (override Bearer token; default uses cookies)
+//
+// When loaded inside an iframe served from /environments/<env>/packages/<pkg>/...,
+// the runtime auto-subscribes to a Server-Sent Events live-reload stream
+// (GET .../events) and reloads the page on file changes. It also posts size
+// updates to the parent window so Publisher.embed() in the host can resize
+// the iframe.
+//
+// The "publisher:resize" postMessage protocol below is the SAME contract the
+// SPA host consumes. Its canonical definition lives in
+// packages/sdk/src/utils/pageEmbed.ts (PUBLISHER_RESIZE_MESSAGE_TYPE /
+// PublisherResizeMessage). This file is build-step-free vanilla JS and can't
+// import it, so keep the message type/shape here in sync with that module.
+
+(function () {
+   "use strict";
+
+   // --- Context inference -------------------------------------------------
+   // URL shape: /environments/<env>/packages/<pkg>/<file>
+   //
+   // location.pathname is URL-encoded, so we MUST decode the captured
+   // segments here. Without this step, a name with a space (e.g.
+   // "demo env") would arrive as "demo%20env" — and the encodeURIComponent
+   // we apply when building API URLs (below) would produce "demo%2520env",
+   // which Publisher then 404s on.
+   var pathMatch = location.pathname.match(
+      /^\/environments\/([^/]+)\/packages\/([^/]+)\//,
+   );
+   function safeDecode(s) {
+      try {
+         return decodeURIComponent(s);
+      } catch (_e) {
+         return s;
+      }
+   }
+   var ctx = pathMatch
+      ? {
+           environment: safeDecode(pathMatch[1]),
+           package: safeDecode(pathMatch[2]),
+        }
+      : {};
+
+   var apiBase = location.origin + "/api/v0";
+   var bearerToken = null;
+
+   function authHeaders() {
+      return bearerToken ? { Authorization: "Bearer " + bearerToken } : {};
+   }
+
+   // --- Query helpers -----------------------------------------------------
+   function resolveTarget(opts) {
+      var env = (opts && opts.environment) || ctx.environment;
+      var pkg = (opts && opts.package) || ctx.package;
+      if (!env || !pkg) {
+         throw new Error(
+            "Publisher: no environment/package; either serve the page from " +
+               "/environments/<env>/packages/<pkg>/... or pass { environment, package } in opts.",
+         );
+      }
+      return { env: env, pkg: pkg };
+   }
+
+   async function rawQuery(modelPath, malloyQuery, opts, compactJson) {
+      opts = opts || {};
+      var target = resolveTarget(opts);
+      var url =
+         apiBase +
+         "/environments/" +
+         encodeURIComponent(target.env) +
+         "/packages/" +
+         encodeURIComponent(target.pkg) +
+         "/models/" +
+         modelPath.split("/").map(encodeURIComponent).join("/") +
+         "/query";
+      var body = { compactJson: compactJson };
+      if (malloyQuery) body.query = malloyQuery;
+      if (opts.sourceName) body.sourceName = opts.sourceName;
+      if (opts.queryName) body.queryName = opts.queryName;
+      if (opts.filterParams) body.filterParams = opts.filterParams;
+      if (opts.bypassFilters) body.bypassFilters = true;
+
+      var headers = Object.assign(
+         { "content-type": "application/json" },
+         authHeaders(),
+      );
+      var res = await fetch(url, {
+         method: "POST",
+         credentials: "include",
+         headers: headers,
+         body: JSON.stringify(body),
+      });
+      var json;
+      try {
+         json = await res.json();
+      } catch (_e) {
+         throw new Error(
+            "Publisher: server returned non-JSON response (" + res.status + ")",
+         );
+      }
+      if (!res.ok) {
+         var msg = (json && json.message) || res.statusText || "Query failed";
+         var err = new Error("Publisher.query: " + msg);
+         err.response = json;
+         err.status = res.status;
+         throw err;
+      }
+      // The server's QueryResult always has `result` as a JSON-encoded string.
+      // Parse it before handing it back so callers see real JS values.
+      return JSON.parse(json.result);
+   }
+
+   function query(modelPath, malloyQuery, opts) {
+      return rawQuery(modelPath, malloyQuery, opts, true);
+   }
+   function queryFull(modelPath, malloyQuery, opts) {
+      return rawQuery(modelPath, malloyQuery, opts, false);
+   }
+
+   // --- Embed helper (host page) -----------------------------------------
+   function embed(selector, options) {
+      options = options || {};
+      var host =
+         typeof selector === "string"
+            ? document.querySelector(selector)
+            : selector;
+      if (!host) {
+         throw new Error("Publisher.embed: selector did not match an element");
+      }
+      if (!options.src) {
+         throw new Error("Publisher.embed: opts.src is required");
+      }
+      var iframe = document.createElement("iframe");
+      iframe.src = options.token
+         ? options.src +
+           (options.src.indexOf("?") === -1 ? "?" : "&") +
+           "embed_token=" +
+           encodeURIComponent(options.token)
+         : options.src;
+      iframe.style.border = "0";
+      iframe.style.width = "100%";
+      iframe.style.display = "block";
+      if (options.height) {
+         iframe.style.height =
+            typeof options.height === "number"
+               ? options.height + "px"
+               : options.height;
+      } else {
+         iframe.style.height = "0px"; // will be sized via postMessage
+      }
+      if (options.allow) iframe.allow = options.allow;
+      iframe.setAttribute(
+         "sandbox",
+         "allow-scripts allow-same-origin allow-forms",
+      );
+
+      // Resize listener
+      function onMessage(e) {
+         if (!e.data || e.data.type !== "publisher:resize") return;
+         if (e.source !== iframe.contentWindow) return;
+         if (typeof e.data.height === "number") {
+            iframe.style.height = Math.max(0, e.data.height) + "px";
+         }
+      }
+      window.addEventListener("message", onMessage);
+      // Best-effort cleanup if the host removes the iframe
+      var observer = new MutationObserver(function () {
+         if (!host.contains(iframe)) {
+            window.removeEventListener("message", onMessage);
+            observer.disconnect();
+         }
+      });
+      observer.observe(host, { childList: true, subtree: false });
+
+      host.appendChild(iframe);
+      return {
+         iframe: iframe,
+         destroy: function () {
+            window.removeEventListener("message", onMessage);
+            observer.disconnect();
+            if (iframe.parentNode) iframe.parentNode.removeChild(iframe);
+         },
+      };
+   }
+
+   // --- When this runtime is itself inside an iframe ---------------------
+   // Post size updates upstream + listen for live-reload SSE events.
+   function setUpEmbeddedSelfBehaviors() {
+      var inIframe = (function () {
+         try {
+            return window.self !== window.top;
+         } catch (_e) {
+            return true; // cross-origin parent — assume embedded
+         }
+      })();
+
+      if (inIframe) {
+         var lastHeight = -1;
+         function measureContentHeight() {
+            // We want the "ink height" — where the last piece of visible
+            // content ends. NOT document.body.scrollHeight: any rule like
+            // `body { min-height: 100vh }` (extremely common in dashboards
+            // that look nice standalone) inflates scrollHeight to match
+            // whatever the iframe's current viewport is, creating a
+            // feedback loop where the iframe ratchets up but never shrinks.
+            //
+            // Sum the lowest bottom edge across body's children, in
+            // document coordinates. This ignores body padding, min-height,
+            // and CSS that just fills the viewport.
+            var body = document.body;
+            if (!body) return document.documentElement.scrollHeight;
+            var maxBottom = 0;
+            var kids = body.children;
+            for (var i = 0; i < kids.length; i++) {
+               var rect = kids[i].getBoundingClientRect();
+               if (rect.bottom > maxBottom) maxBottom = rect.bottom;
+            }
+            if (maxBottom <= 0) {
+               // Fallback for empty body / hidden children
+               return document.documentElement.scrollHeight;
+            }
+            var scrollTop =
+               window.scrollY ||
+               document.documentElement.scrollTop ||
+               document.body.scrollTop ||
+               0;
+            // Add body bottom padding (rect.bottom is content-box bottom,
+            // body padding isn't part of any child's rect).
+            var bodyStyle = window.getComputedStyle(body);
+            var pad = parseFloat(bodyStyle.paddingBottom) || 0;
+            return Math.ceil(maxBottom + scrollTop + pad);
+         }
+         function postSize() {
+            var h = measureContentHeight();
+            if (h !== lastHeight) {
+               lastHeight = h;
+               try {
+                  window.parent.postMessage(
+                     { type: "publisher:resize", height: h },
+                     "*",
+                  );
+               } catch (_e) {
+                  /* ignore */
+               }
+            }
+         }
+         // Initial + observe content changes
+         if (document.readyState === "loading") {
+            document.addEventListener("DOMContentLoaded", postSize);
+         } else {
+            postSize();
+         }
+         window.addEventListener("load", postSize);
+         if (typeof ResizeObserver !== "undefined") {
+            var ro = new ResizeObserver(postSize);
+            // Observe documentElement so we catch any layout change
+            ro.observe(document.documentElement);
+         } else {
+            // Fallback: poll once a second
+            setInterval(postSize, 1000);
+         }
+      }
+   }
+
+   // --- SSE live reload --------------------------------------------------
+   function setUpLiveReload() {
+      if (!ctx.environment || !ctx.package) return;
+      if (typeof EventSource === "undefined") return;
+      var url =
+         apiBase +
+         "/environments/" +
+         encodeURIComponent(ctx.environment) +
+         "/packages/" +
+         encodeURIComponent(ctx.package) +
+         "/events";
+      try {
+         var es = new EventSource(url, { withCredentials: true });
+         var pending = false;
+         es.addEventListener("changed", function () {
+            if (pending) return;
+            pending = true;
+            // Tiny debounce to coalesce a flurry of saves
+            setTimeout(function () {
+               location.reload();
+            }, 100);
+         });
+         es.onerror = function () {
+            // Browser will auto-reconnect; nothing to do.
+         };
+      } catch (_e) {
+         // SSE may be blocked (e.g. corp proxy) — non-fatal.
+      }
+   }
+
+   // --- Public API --------------------------------------------------------
+   window.Publisher = {
+      query: query,
+      queryFull: queryFull,
+      embed: embed,
+      context: ctx,
+      setToken: function (token) {
+         bearerToken = token || null;
+      },
+   };
+
+   // Auto-init the in-iframe behaviors and live-reload subscription.
+   // Both are no-ops if not applicable.
+   setUpEmbeddedSelfBehaviors();
+   setUpLiveReload();
+})();