@malloy-publisher/server 0.0.203 → 0.0.205

package/src/package_load/package_load_pool.ts

@@ -67,7 +67,6 @@
  *   it resolves.
  */
 import type {
-   Annotation,
    FetchSchemaOptions,
    InfoConnection,
    LookupConnection,
@@ -811,15 +810,11 @@ export class PackageLoadPool {
 
 function buildFetchOptions(options: {
    refreshTimestamp?: number;
-   modelAnnotation?: Annotation;
 }): FetchSchemaOptions {
    const out: FetchSchemaOptions = {};
    if (options.refreshTimestamp !== undefined) {
       out.refreshTimestamp = options.refreshTimestamp;
    }
-   if (options.modelAnnotation !== undefined) {
-      out.modelAnnotation = options.modelAnnotation;
-   }
    return out;
 }
 

package/src/package_load/package_load_worker.ts

@@ -46,7 +46,6 @@
  */
 import {
    contextOverlay,
-   type Annotation,
    type BuildManifestEntry,
    type Connection,
    type FetchSchemaOptions,
@@ -56,16 +55,12 @@ import {
    type ModelDef,
    type ModelMaterializer,
    modelDefToModelInfo,
-   type NamedModelObject,
    type NamedQueryDef,
    type Query,
    Runtime,
    type SQLSourceDef,
    type SQLSourceRequest,
-   type StructDef,
    type TableSourceDef,
-   type TurtleDef,
-   isSourceDef,
 } from "@malloydata/malloy";
 import * as Malloy from "@malloydata/malloy-interfaces";
 import {
@@ -84,7 +79,13 @@ import {
    PACKAGE_MANIFEST_NAME,
 } from "../constants";
 import { HackyDataStylesAccumulator } from "../data_styles";
-import { parseFilters, type FilterDefinition } from "../service/filter";
+import { ModelCompilationError } from "../errors";
+import { validateAuthorizeProbes } from "../service/authorize";
+import { type FilterDefinition } from "../service/filter";
+import {
+   extractQueriesFromModelDef,
+   extractSourcesFromModelDef,
+} from "../service/source_extraction";
 import {
    malloyGivenToApi,
    type MalloyGiven,
@@ -271,15 +272,13 @@ class ProxyConnection {
 
 function serializeFetchOptions(options: FetchSchemaOptions): {
    refreshTimestamp?: number;
-   modelAnnotation?: Annotation;
 } {
-   const out: { refreshTimestamp?: number; modelAnnotation?: Annotation } = {};
+   const out: {
+      refreshTimestamp?: number;
+   } = {};
    if (options.refreshTimestamp !== undefined) {
       out.refreshTimestamp = options.refreshTimestamp;
    }
-   if (options.modelAnnotation !== undefined) {
-      out.modelAnnotation = options.modelAnnotation;
-   }
    return out;
 }
 
@@ -411,6 +410,7 @@ interface ApiSourceWire {
    views?: { name: string; annotations?: string[] }[];
    filters?: unknown[];
    givens?: unknown[];
+   authorize?: string[];
 }
 interface ApiQueryWire {
    name: string;
@@ -472,96 +472,20 @@ function appendLocalSourceInfos(
    }
 }
 
+// Source / query introspection is shared with the in-process path; see
+// service/source_extraction.ts. The worker has no logger, so a filter parse
+// failure is swallowed silently (no `onParseError` callback) — matching the
+// prior worker-local behavior.
 function extractSources(
-   modelPath: string,
    modelDef: ModelDef,
    givens: ApiGivenWire[] | undefined,
 ): { sources: ApiSourceWire[]; filterMap: Map<string, FilterDefinition[]> } {
-   const filterMap = new Map<string, FilterDefinition[]>();
-   const sources: ApiSourceWire[] = Object.values(modelDef.contents)
-      .filter((obj) => isSourceDef(obj))
-      .map((sourceObj) => {
-         const sourceName =
-            (sourceObj as StructDef).as || (sourceObj as StructDef).name;
-         const annotations = (sourceObj as StructDef).annotation?.blockNotes
-            ?.filter((note) => note.at.url.includes(modelPath))
-            .map((note) => note.text);
-
-         const collected: string[][] = [];
-         let cur: Annotation | undefined = (sourceObj as StructDef).annotation;
-         while (cur) {
-            if (cur.blockNotes) {
-               collected.push(cur.blockNotes.map((note) => note.text));
-            }
-            cur = cur.inherits;
-         }
-         const allAnnotations = collected.reverse().flat();
-         let filters: unknown[] | undefined;
-         if (allAnnotations.length > 0) {
-            try {
-               const parsed = parseFilters(allAnnotations);
-               if (parsed.length > 0) {
-                  filterMap.set(sourceName, parsed);
-                  const fields = (sourceObj as StructDef).fields;
-                  filters = parsed.map((f) => {
-                     const field = fields.find(
-                        (fd) => (fd.as || fd.name) === f.dimension,
-                     );
-                     return {
-                        name: f.name,
-                        dimension: f.dimension,
-                        type: f.type,
-                        implicit: f.implicit,
-                        required: f.required,
-                        dimensionType: field?.type as string | undefined,
-                     };
-                  });
-               }
-            } catch {
-               /* parse errors are warnings; matches in-process */
-            }
-         }
-
-         const views = (sourceObj as StructDef).fields
-            .filter((f) => f.type === "turtle")
-            .filter((turtle) =>
-               (turtle as TurtleDef).pipeline
-                  .map((stage) => stage.type)
-                  .every((type) => type === "reduce"),
-            )
-            .map((turtle) => ({
-               name: turtle.as || turtle.name,
-               annotations: turtle?.annotation?.blockNotes
-                  ?.filter((note) => note.at.url.includes(modelPath))
-                  .map((note) => note.text),
-            }));
-
-         return {
-            name: sourceName,
-            annotations,
-            views,
-            filters,
-            givens,
-         } as ApiSourceWire;
-      });
-
-   return { sources, filterMap };
+   const { sources, filterMap } = extractSourcesFromModelDef(modelDef, givens);
+   return { sources: sources as unknown as ApiSourceWire[], filterMap };
 }
 
-function extractQueries(modelPath: string, modelDef: ModelDef): ApiQueryWire[] {
-   const isNamedQuery = (obj: NamedModelObject): obj is NamedQueryDef =>
-      obj.type === "query";
-   return Object.values(modelDef.contents)
-      .filter(isNamedQuery)
-      .map((q) => ({
-         name: q.as || q.name,
-         sourceName: typeof q.structRef === "string" ? q.structRef : undefined,
-         annotations: q?.annotation?.blockNotes
-            ?.filter((note: { at: { url: string } }) =>
-               note.at.url.includes(modelPath),
-            )
-            .map((note: { text: string }) => note.text),
-      }));
+function extractQueries(modelDef: ModelDef): ApiQueryWire[] {
+   return extractQueriesFromModelDef(modelDef) as ApiQueryWire[];
 }
 
 function buildRuntimeForModel(
@@ -621,8 +545,12 @@ async function compileMalloyModel(
    );
    appendLocalSourceInfos(modelDef, sourceInfos, importedNames);
 
-   const { sources, filterMap } = extractSources(modelPath, modelDef, givens);
-   const queries = extractQueries(modelPath, modelDef);
+   const { sources, filterMap } = extractSources(modelDef, givens);
+   const queries = extractQueries(modelDef);
+   // Validate #(authorize) at compile time (shared with Model.create). Throws
+   // on an unknown given / source-field reference; compileOneModel's catch
+   // turns it into this model's compilationError.
+   await validateAuthorizeProbes(mm, sources);
 
    return {
       modelPath,
@@ -798,10 +726,12 @@ async function compileNotebookModel(
          collected.importedNames,
       );
       finalSourceInfos = collected.sourceInfos;
-      const extracted = extractSources(modelPath, finalModelDef, finalGivens);
+      const extracted = extractSources(finalModelDef, finalGivens);
       finalSources = extracted.sources;
       finalFilterMap = extracted.filterMap;
-      finalQueries = extractQueries(modelPath, finalModelDef);
+      finalQueries = extractQueries(finalModelDef);
+      // Validate #(authorize) at compile time (shared with Model.create).
+      await validateAuthorizeProbes(mm, finalSources);
    }
 
    return {
@@ -899,6 +829,18 @@ function serializeError(error: unknown): SerializedError {
          isCompilationError: true,
       };
    }
+   // ModelCompilationError (e.g. an invalid #(authorize) annotation caught by
+   // validateAuthorizeProbes) carries no Malloy `problems`, but it must keep its
+   // compilation-error classification across the worker boundary so the main
+   // thread re-wraps it as a 424, not a generic 500.
+   if (error instanceof ModelCompilationError) {
+      return {
+         name: error.name,
+         message: error.message,
+         stack: error.stack,
+         isCompilationError: true,
+      };
+   }
    if (error instanceof Error) {
       return {
          name: error.name,

package/src/package_load/protocol.ts

@@ -64,11 +64,7 @@
  * cheaper than `JSON.stringify` for the multi-MB modelDef payloads.
  */
 
-import type {
-   Annotation,
-   SQLSourceDef,
-   TableSourceDef,
-} from "@malloydata/malloy";
+import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
 
 // ──────────────────────────────────────────────────────────────────────
 // Direction: main ──▶ worker (load-package job)
@@ -236,7 +232,6 @@ export interface SchemaForTablesRequest {
    tables: Record<string, string>;
    options: {
       refreshTimestamp?: number;
-      modelAnnotation?: Annotation;
    };
 }
 
@@ -256,7 +251,6 @@ export interface SchemaForSqlRequest {
    sentence: unknown;
    options: {
       refreshTimestamp?: number;
-      modelAnnotation?: Annotation;
    };
 }
 

package/src/runtime/publisher.js

@@ -0,0 +1,318 @@
+// Publisher runtime helper for in-package HTML dashboards.
+// Served by the Publisher server at /sdk/publisher.js. Hand-authored vanilla
+// JS — no bundler. Loaded via <script src="/sdk/publisher.js">.
+//
+// Exposes window.Publisher with:
+//   - Publisher.query(model, malloy, opts?)     → Promise<rows[]>
+//   - Publisher.queryFull(model, malloy, opts?) → Promise<MalloyResult>  (envelope for <malloy-render>)
+//   - Publisher.embed(selector, { src, height?, token? })
+//   - Publisher.context  ({ environment, package } inferred from URL)
+//   - Publisher.setToken(token)  (override Bearer token; default uses cookies)
+//
+// When loaded inside an iframe served from /environments/<env>/packages/<pkg>/...,
+// the runtime auto-subscribes to a Server-Sent Events live-reload stream
+// (GET .../events) and reloads the page on file changes. It also posts size
+// updates to the parent window so Publisher.embed() in the host can resize
+// the iframe.
+//
+// The "publisher:resize" postMessage protocol below is the SAME contract the
+// SPA host consumes. Its canonical definition lives in
+// packages/sdk/src/utils/pageEmbed.ts (PUBLISHER_RESIZE_MESSAGE_TYPE /
+// PublisherResizeMessage). This file is build-step-free vanilla JS and can't
+// import it, so keep the message type/shape here in sync with that module.
+
+(function () {
+   "use strict";
+
+   // --- Context inference -------------------------------------------------
+   // URL shape: /environments/<env>/packages/<pkg>/<file>
+   //
+   // location.pathname is URL-encoded, so we MUST decode the captured
+   // segments here. Without this step, a name with a space (e.g.
+   // "demo env") would arrive as "demo%20env" — and the encodeURIComponent
+   // we apply when building API URLs (below) would produce "demo%2520env",
+   // which Publisher then 404s on.
+   var pathMatch = location.pathname.match(
+      /^\/environments\/([^/]+)\/packages\/([^/]+)\//,
+   );
+   function safeDecode(s) {
+      try {
+         return decodeURIComponent(s);
+      } catch (_e) {
+         return s;
+      }
+   }
+   var ctx = pathMatch
+      ? {
+           environment: safeDecode(pathMatch[1]),
+           package: safeDecode(pathMatch[2]),
+        }
+      : {};
+
+   var apiBase = location.origin + "/api/v0";
+   var bearerToken = null;
+
+   function authHeaders() {
+      return bearerToken ? { Authorization: "Bearer " + bearerToken } : {};
+   }
+
+   // --- Query helpers -----------------------------------------------------
+   function resolveTarget(opts) {
+      var env = (opts && opts.environment) || ctx.environment;
+      var pkg = (opts && opts.package) || ctx.package;
+      if (!env || !pkg) {
+         throw new Error(
+            "Publisher: no environment/package; either serve the page from " +
+               "/environments/<env>/packages/<pkg>/... or pass { environment, package } in opts.",
+         );
+      }
+      return { env: env, pkg: pkg };
+   }
+
+   async function rawQuery(modelPath, malloyQuery, opts, compactJson) {
+      opts = opts || {};
+      var target = resolveTarget(opts);
+      var url =
+         apiBase +
+         "/environments/" +
+         encodeURIComponent(target.env) +
+         "/packages/" +
+         encodeURIComponent(target.pkg) +
+         "/models/" +
+         modelPath.split("/").map(encodeURIComponent).join("/") +
+         "/query";
+      var body = { compactJson: compactJson };
+      if (malloyQuery) body.query = malloyQuery;
+      if (opts.sourceName) body.sourceName = opts.sourceName;
+      if (opts.queryName) body.queryName = opts.queryName;
+      if (opts.filterParams) body.filterParams = opts.filterParams;
+      if (opts.bypassFilters) body.bypassFilters = true;
+
+      var headers = Object.assign(
+         { "content-type": "application/json" },
+         authHeaders(),
+      );
+      var res = await fetch(url, {
+         method: "POST",
+         credentials: "include",
+         headers: headers,
+         body: JSON.stringify(body),
+      });
+      var json;
+      try {
+         json = await res.json();
+      } catch (_e) {
+         throw new Error(
+            "Publisher: server returned non-JSON response (" + res.status + ")",
+         );
+      }
+      if (!res.ok) {
+         var msg = (json && json.message) || res.statusText || "Query failed";
+         var err = new Error("Publisher.query: " + msg);
+         err.response = json;
+         err.status = res.status;
+         throw err;
+      }
+      // The server's QueryResult always has `result` as a JSON-encoded string.
+      // Parse it before handing it back so callers see real JS values.
+      return JSON.parse(json.result);
+   }
+
+   function query(modelPath, malloyQuery, opts) {
+      return rawQuery(modelPath, malloyQuery, opts, true);
+   }
+   function queryFull(modelPath, malloyQuery, opts) {
+      return rawQuery(modelPath, malloyQuery, opts, false);
+   }
+
+   // --- Embed helper (host page) -----------------------------------------
+   function embed(selector, options) {
+      options = options || {};
+      var host =
+         typeof selector === "string"
+            ? document.querySelector(selector)
+            : selector;
+      if (!host) {
+         throw new Error("Publisher.embed: selector did not match an element");
+      }
+      if (!options.src) {
+         throw new Error("Publisher.embed: opts.src is required");
+      }
+      var iframe = document.createElement("iframe");
+      iframe.src = options.token
+         ? options.src +
+           (options.src.indexOf("?") === -1 ? "?" : "&") +
+           "embed_token=" +
+           encodeURIComponent(options.token)
+         : options.src;
+      iframe.style.border = "0";
+      iframe.style.width = "100%";
+      iframe.style.display = "block";
+      if (options.height) {
+         iframe.style.height =
+            typeof options.height === "number"
+               ? options.height + "px"
+               : options.height;
+      } else {
+         iframe.style.height = "0px"; // will be sized via postMessage
+      }
+      if (options.allow) iframe.allow = options.allow;
+      iframe.setAttribute(
+         "sandbox",
+         "allow-scripts allow-same-origin allow-forms",
+      );
+
+      // Resize listener
+      function onMessage(e) {
+         if (!e.data || e.data.type !== "publisher:resize") return;
+         if (e.source !== iframe.contentWindow) return;
+         if (typeof e.data.height === "number") {
+            iframe.style.height = Math.max(0, e.data.height) + "px";
+         }
+      }
+      window.addEventListener("message", onMessage);
+      // Best-effort cleanup if the host removes the iframe
+      var observer = new MutationObserver(function () {
+         if (!host.contains(iframe)) {
+            window.removeEventListener("message", onMessage);
+            observer.disconnect();
+         }
+      });
+      observer.observe(host, { childList: true, subtree: false });
+
+      host.appendChild(iframe);
+      return {
+         iframe: iframe,
+         destroy: function () {
+            window.removeEventListener("message", onMessage);
+            observer.disconnect();
+            if (iframe.parentNode) iframe.parentNode.removeChild(iframe);
+         },
+      };
+   }
+
+   // --- When this runtime is itself inside an iframe ---------------------
+   // Post size updates upstream + listen for live-reload SSE events.
+   function setUpEmbeddedSelfBehaviors() {
+      var inIframe = (function () {
+         try {
+            return window.self !== window.top;
+         } catch (_e) {
+            return true; // cross-origin parent — assume embedded
+         }
+      })();
+
+      if (inIframe) {
+         var lastHeight = -1;
+         function measureContentHeight() {
+            // We want the "ink height" — where the last piece of visible
+            // content ends. NOT document.body.scrollHeight: any rule like
+            // `body { min-height: 100vh }` (extremely common in dashboards
+            // that look nice standalone) inflates scrollHeight to match
+            // whatever the iframe's current viewport is, creating a
+            // feedback loop where the iframe ratchets up but never shrinks.
+            //
+            // Sum the lowest bottom edge across body's children, in
+            // document coordinates. This ignores body padding, min-height,
+            // and CSS that just fills the viewport.
+            var body = document.body;
+            if (!body) return document.documentElement.scrollHeight;
+            var maxBottom = 0;
+            var kids = body.children;
+            for (var i = 0; i < kids.length; i++) {
+               var rect = kids[i].getBoundingClientRect();
+               if (rect.bottom > maxBottom) maxBottom = rect.bottom;
+            }
+            if (maxBottom <= 0) {
+               // Fallback for empty body / hidden children
+               return document.documentElement.scrollHeight;
+            }
+            var scrollTop =
+               window.scrollY ||
+               document.documentElement.scrollTop ||
+               document.body.scrollTop ||
+               0;
+            // Add body bottom padding (rect.bottom is content-box bottom,
+            // body padding isn't part of any child's rect).
+            var bodyStyle = window.getComputedStyle(body);
+            var pad = parseFloat(bodyStyle.paddingBottom) || 0;
+            return Math.ceil(maxBottom + scrollTop + pad);
+         }
+         function postSize() {
+            var h = measureContentHeight();
+            if (h !== lastHeight) {
+               lastHeight = h;
+               try {
+                  window.parent.postMessage(
+                     { type: "publisher:resize", height: h },
+                     "*",
+                  );
+               } catch (_e) {
+                  /* ignore */
+               }
+            }
+         }
+         // Initial + observe content changes
+         if (document.readyState === "loading") {
+            document.addEventListener("DOMContentLoaded", postSize);
+         } else {
+            postSize();
+         }
+         window.addEventListener("load", postSize);
+         if (typeof ResizeObserver !== "undefined") {
+            var ro = new ResizeObserver(postSize);
+            // Observe documentElement so we catch any layout change
+            ro.observe(document.documentElement);
+         } else {
+            // Fallback: poll once a second
+            setInterval(postSize, 1000);
+         }
+      }
+   }
+
+   // --- SSE live reload --------------------------------------------------
+   function setUpLiveReload() {
+      if (!ctx.environment || !ctx.package) return;
+      if (typeof EventSource === "undefined") return;
+      var url =
+         apiBase +
+         "/environments/" +
+         encodeURIComponent(ctx.environment) +
+         "/packages/" +
+         encodeURIComponent(ctx.package) +
+         "/events";
+      try {
+         var es = new EventSource(url, { withCredentials: true });
+         var pending = false;
+         es.addEventListener("changed", function () {
+            if (pending) return;
+            pending = true;
+            // Tiny debounce to coalesce a flurry of saves
+            setTimeout(function () {
+               location.reload();
+            }, 100);
+         });
+         es.onerror = function () {
+            // Browser will auto-reconnect; nothing to do.
+         };
+      } catch (_e) {
+         // SSE may be blocked (e.g. corp proxy) — non-fatal.
+      }
+   }
+
+   // --- Public API --------------------------------------------------------
+   window.Publisher = {
+      query: query,
+      queryFull: queryFull,
+      embed: embed,
+      context: ctx,
+      setToken: function (token) {
+         bearerToken = token || null;
+      },
+   };
+
+   // Auto-init the in-iframe behaviors and live-reload subscription.
+   // Both are no-ops if not applicable.
+   setUpEmbeddedSelfBehaviors();
+   setUpLiveReload();
+})();