@malloy-publisher/server 0.9.1 → 0.9.3

package/dist/package_load_worker.mjs

@@ -13195,7 +13195,7 @@ function toQueryTextProblem(problem) {
     ...problem.at ? { at: { range: problem.at.range } } : {}
   };
 }
-var GENERIC_INTERNAL_MESSAGE = "Internal server error.", GENERIC_UPSTREAM_MESSAGE = "Upstream connection error.", MAX_LOGGED_DETAIL_CHARS = 2000, FILESYSTEM_ACCESS_DESCRIPTIONS, NotImplementedError, BadRequestError, InvalidArgumentError, CompileRefusedError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, DashboardNotFoundError, ConnectionNotFoundError, TableNotFoundError, ConnectionError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, PackageManifestError, MaterializationEligibilityError, PublisherConfigError, ServerConfigurationError, FrozenConfigError, AccessDeniedError, QueryCompileError, NotQueryableError, OffSurfaceError, MaterializationNotFoundError, MaterializationConflictError, WriteConflictError, WriteRolledBackError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
+var GENERIC_INTERNAL_MESSAGE = "Internal server error.", GENERIC_UPSTREAM_MESSAGE = "Upstream connection error.", MAX_LOGGED_DETAIL_CHARS = 2000, FILESYSTEM_ACCESS_DESCRIPTIONS, NotImplementedError, BadRequestError, InvalidArgumentError, CompileRefusedError, RenderTagRefusedError, UnparseableTextError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, DashboardNotFoundError, ConnectionNotFoundError, TableNotFoundError, ConnectionError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, PackageManifestError, MaterializationEligibilityError, PublisherConfigError, ServerConfigurationError, FrozenConfigError, AccessDeniedError, QueryCompileError, NotQueryableError, OffSurfaceError, MaterializationNotFoundError, MaterializationConflictError, WriteConflictError, WriteRolledBackError, WriteVerifyError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
 var init_errors = __esm(() => {
   init_constants();
   init_logger();
@@ -13218,6 +13218,15 @@ var init_errors = __esm(() => {
   };
   CompileRefusedError = class CompileRefusedError extends BadRequestError {
   };
+  RenderTagRefusedError = class RenderTagRefusedError extends CompileRefusedError {
+  };
+  UnparseableTextError = class UnparseableTextError extends CompileRefusedError {
+    detail;
+    constructor(message, detail) {
+      super(message);
+      this.detail = detail;
+    }
+  };
   EnvironmentNotFoundError = class EnvironmentNotFoundError extends Error {
     constructor(message) {
       super(message);
@@ -13348,11 +13357,17 @@ var init_errors = __esm(() => {
     }
   };
   WriteRolledBackError = class WriteRolledBackError extends Error {
-    constructor(message) {
-      super(message);
+    constructor(message, options) {
+      super(message, options);
       this.name = "WriteRolledBackError";
     }
   };
+  WriteVerifyError = class WriteVerifyError extends Error {
+    constructor(message, options) {
+      super(message, options);
+      this.name = "WriteVerifyError";
+    }
+  };
   InvalidStateTransitionError = class InvalidStateTransitionError extends Error {
     constructor(message) {
       super(message);
@@ -13407,6 +13422,7 @@ var init_authorize_routes = __esm(() => {
 
 // src/service/authorize.ts
 import { payloadOf, routeOf } from "@malloydata/malloy";
+import { createRequire as createRequire2 } from "module";
 function noteRoute(text) {
   return routeOf({ value: text.trimStart() });
 }
@@ -13455,7 +13471,7 @@ function motlyAuthorizePayloadPattern(route) {
   return new RegExp(`^[ \\t]*[([{<][ \\t]*(?:${nearMissWordAlternation(route)})[ \\t]*[)\\]}>]`, "iu");
 }
 function malformedAuthorizeAttemptPattern(route) {
-  return new RegExp(`^##?\\|?[ \\t]*[([{<]?[ \\t]*(?:${nearMissWordAlternation(route)})`, "iu");
+  return new RegExp(`^##?\\|?[ \\t]*(?:[([{<][ \\t]*)?(?:${nearMissWordAlternation(route)})`, "iu");
 }
 function authorizeNoteContent(text) {
   const route = canonicalAuthorizeRoute(noteRoute(text));
@@ -13466,12 +13482,144 @@ function authorizeNoteContent(text) {
 function authorizeAnnotationRoute(text) {
   return authorizeNoteContent(text)?.route;
 }
-function hasCallerAuthorizeAnnotation(callerText) {
-  return AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText);
+function loadMalloyParser() {
+  try {
+    const entry = createRequire2(import.meta.url).resolve("@malloydata/malloy");
+    const loaded = createRequire2(entry)("./lang/run-malloy-parser");
+    return typeof loaded.makeMalloyParser === "function" ? loaded.makeMalloyParser : null;
+  } catch {
+    return null;
+  }
+}
+function malloyParserFactory() {
+  if (makeMalloyParser === undefined)
+    makeMalloyParser = parserLoader();
+  return makeMalloyParser ?? undefined;
+}
+function lastCallerGuardRefusalKind() {
+  return lastRefusalKind;
+}
+function proseRanges(text) {
+  if (lastLexed.text !== text)
+    lastLexed = { text, prose: lexProseRanges(text) };
+  return lastLexed.prose;
 }
-function assertNoCallerAuthorizeAnnotation(callerText) {
+function lexProseRanges(text) {
+  const factory = malloyParserFactory();
+  lexerMissing = !factory;
+  if (!factory) {
+    if (!warnedNoParser) {
+      warnedNoParser = true;
+      console.warn("Prose-aware caller-annotation guard disabled: @malloydata/malloy ./lang/run-malloy-parser could not be loaded; falling back to whole-text matching.");
+    }
+    return;
+  }
+  try {
+    const counter = () => ({
+      errors: 0,
+      syntaxError() {
+        this.errors++;
+      }
+    });
+    const lexerErrors = counter();
+    const parserErrors = counter();
+    const { tokenStream, parser } = factory(text, {
+      lexerErrorListener: lexerErrors,
+      parserErrorListener: parserErrors
+    });
+    parser.malloyDocument();
+    const vocabulary = tokenStream.tokenSource?.vocabulary;
+    if (lexerErrors.errors > 0 || parserErrors.errors > 0 || typeof vocabulary?.getSymbolicName !== "function")
+      return;
+    const utf16 = [];
+    for (let i = 0;i < text.length; ) {
+      utf16.push(i);
+      i += text.codePointAt(i) > 65535 ? 2 : 1;
+    }
+    utf16.push(text.length);
+    const rangeOf = (token) => {
+      const from = utf16[token.startIndex];
+      const to = utf16[token.stopIndex + 1];
+      return from === undefined || to === undefined || to < from ? undefined : [from, to];
+    };
+    const prose = [];
+    let block;
+    for (const token of tokenStream.getTokens()) {
+      const name = vocabulary.getSymbolicName(token.type);
+      const range = rangeOf(token);
+      if (name === "BLOCK_ANNOTATION_TEXT" || name === "BLOCK_ANNOTATION_END") {
+        if (!block || !range || token.startIndex !== block.next)
+          return;
+        if (name === "BLOCK_ANNOTATION_END") {
+          block = undefined;
+        } else {
+          if (block.prose)
+            prose.push(range);
+          block.next = token.stopIndex + 1;
+        }
+        continue;
+      }
+      if (block)
+        return;
+      if (name === "BLOCK_ANNOTATION_BEGIN" || name === "DOC_BLOCK_ANNOTATION_BEGIN") {
+        if (!range)
+          return;
+        block = {
+          prose: PROSE_ROUTES.has(noteRoute(text.slice(...range)) ?? ""),
+          next: token.stopIndex + 1
+        };
+      } else if (name === "ANNOTATION" || name === "DOC_ANNOTATION") {
+        if (!range)
+          return;
+        if (PROSE_ROUTES.has(noteRoute(text.slice(...range)) ?? ""))
+          prose.push([range[0] + 1, range[1]]);
+      }
+    }
+    return block ? undefined : prose;
+  } catch {
+    return;
+  }
+}
+function hasCallerAuthorizeAnnotation(callerText, precedingText = "") {
+  if (!AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText))
+    return false;
+  const text = precedingText + callerText;
+  const bodies = proseRanges(text);
+  lastRefusalKind = bodies !== undefined ? "lexed" : lexerMissing ? "no_lexer" : "whole_text";
+  if (bodies === undefined)
+    return true;
+  const sticky = new RegExp(AUTHORIZE_TAG_LIKE, "iuy");
+  const callerStart = precedingText.length;
+  const from = text.lastIndexOf(`
+`, callerStart - 1) + 1;
+  let body = 0;
+  for (let at = text.indexOf("#", from);at !== -1; at = text.indexOf("#", at + 1)) {
+    sticky.lastIndex = at;
+    const hit = sticky.exec(text);
+    if (!hit)
+      continue;
+    const end = at + hit[0].length;
+    if (end <= callerStart)
+      continue;
+    while (body < bodies.length && bodies[body][1] <= at)
+      body++;
+    const [start, stop] = bodies[body] ?? [Infinity, -Infinity];
+    if (!(start <= at && end <= stop))
+      return true;
+  }
+  return false;
+}
+function assertNoCallerAuthorizeAnnotation(callerText, precedingText = "") {
+  if (hasCallerAuthorizeAnnotation(callerText, precedingText))
+    refuseCallerAuthorizeAnnotation();
+}
+function assertNoAuthorizeTagLike(callerText) {
   if (!AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText))
     return;
+  lastRefusalKind = "whole_text";
+  refuseCallerAuthorizeAnnotation();
+}
+function refuseCallerAuthorizeAnnotation() {
   throw new BadRequestError("An `authorize` annotation is not permitted in caller-submitted Malloy " + "text. Access gates are declared by the model author on the source; a " + "request cannot introduce, replace, or relax one. To validate a gate " + "you are authoring, save it to the package's model file and reload the " + "package — model load validates every `#(authorize)` annotation it " + "declares.");
 }
 function containsAuthorizeAnnotationTag(texts) {
@@ -13669,13 +13817,19 @@ ${rewrites}
 ` + `Test that the rewrite gates the same rows and check the row count ` + `matches what the string form served.`
   });
 }
-var AUTHORIZE_TAG_LIKE, AUTHORIZE_ANNOTATION_ANYWHERE, REGEXP_SPECIAL, GIVEN_REF_PATTERN, SINGLE_QUOTE_ESCAPE, DOUBLE_QUOTE_ESCAPE, STRING_LITERAL_PATTERN, everyMemberOf = () => (...members) => members, ROW_LEVEL_GATE_REJECTION_CAUSES, WHOLE_BODY_QUOTED_STRING, WHOLE_BODY_SINGLE_QUOTED_STRING;
+var AUTHORIZE_TAG_LIKE, AUTHORIZE_ANNOTATION_ANYWHERE, REGEXP_SPECIAL, PROSE_ROUTES, makeMalloyParser, parserLoader, warnedNoParser = false, lastLexed, lastRefusalKind = "lexed", lexerMissing = false, GIVEN_REF_PATTERN, SINGLE_QUOTE_ESCAPE, DOUBLE_QUOTE_ESCAPE, STRING_LITERAL_PATTERN, everyMemberOf = () => (...members) => members, ROW_LEVEL_GATE_REJECTION_CAUSES, WHOLE_BODY_QUOTED_STRING, WHOLE_BODY_SINGLE_QUOTED_STRING;
 var init_authorize = __esm(() => {
   init_errors();
   init_authorize_routes();
-  AUTHORIZE_TAG_LIKE = String.raw`##?\|?[ \t]*[([{<]?[ \t]*(?:(?:(?:row|source)[-_]?)?authorize|access[-_]?filter)(?=[)\]}>]|[ \t]|$)`;
+  AUTHORIZE_TAG_LIKE = String.raw`##?\|?[ \t]*(?:[([{<][ \t]*)?(?:(?:(?:row|source)[-_]?)?authorize|access[-_]?filter)(?=[)\]}>]|[ \t]|$)`;
   AUTHORIZE_ANNOTATION_ANYWHERE = new RegExp(AUTHORIZE_TAG_LIKE, "iu");
   REGEXP_SPECIAL = /[.*+?^${}()|[\]\\]/g;
+  PROSE_ROUTES = new Set(["markdown", "text"]);
+  parserLoader = loadMalloyParser;
+  lastLexed = {
+    text: "",
+    prose: []
+  };
   GIVEN_REF_PATTERN = /\$([A-Za-z_][A-Za-z0-9_]*)/g;
   SINGLE_QUOTE_ESCAPE = /\\(['\\])/g;
   DOUBLE_QUOTE_ESCAPE = /\\(["\\])/g;
@@ -13688,9 +13842,12 @@ var init_authorize = __esm(() => {
 // src/authorize_metrics.ts
 function recordAuthorizeGuardRejection(field) {
   guardRejectionCounter ??= publisherMeter().createCounter("publisher_authorize_guard_rejected_total", {
-    description: "Requests rejected with 400 for declaring an `#(authorize)` annotation in caller-submitted Malloy text. Label: field ('query'|'source_name'|'query_name'|'compile_source')."
+    description: "Requests rejected with 400 for declaring an `#(authorize)` annotation in caller-submitted Malloy text. Labels: field ('query'|'source_name'|'query_name'|'compile_source'), match ('lexed' when Malloy's lexer placed a tag outside prose, 'whole_text' when the text could not be lexed cleanly or is a name, 'no_lexer' when Malloy's parser failed to load so every prose-bearing text refuses)."
+  });
+  guardRejectionCounter.add(1, {
+    field,
+    match: lastCallerGuardRefusalKind()
   });
-  guardRejectionCounter.add(1, { field });
 }
 function recordAuthorizeBypass(entryPoint) {
   bypassCounter ??= publisherMeter().createCounter("publisher_authorize_bypass_total", {
@@ -14817,6 +14974,44 @@ var init_gate_classification = __esm(() => {
   init_gate_registry_walk();
 });
 
+// src/service/model_info.ts
+import { modelDefToModelInfo } from "@malloydata/malloy";
+function modelInfoOf(modelDef) {
+  try {
+    return modelDefToModelInfo(modelDef);
+  } catch (err) {
+    if (!isUnboundGiven(err) || !modelDef.givens)
+      throw err;
+    try {
+      return modelDefToModelInfo({
+        ...modelDef,
+        givens: withPlaceholderDefaults(modelDef.givens)
+      });
+    } catch {
+      throw err;
+    }
+  }
+}
+function isUnboundGiven(err) {
+  return err?.code === "compiler-given-no-value";
+}
+function withPlaceholderDefaults(givens) {
+  const out = {};
+  for (const [id, given] of Object.entries(givens)) {
+    if (given.default !== undefined) {
+      out[id] = given;
+      continue;
+    }
+    const isFilter = given.type.type === "filter expression";
+    out[id] = {
+      ...given,
+      default: isFilter ? { node: "filterLiteral", filterSrc: "" } : { node: "null" }
+    };
+  }
+  return out;
+}
+var init_model_info = () => {};
+
 // src/service/query_text.ts
 import { ParseUtil } from "@malloydata/malloy-tag";
 function extractRunTargetSourceName(query) {
@@ -15748,6 +15943,17 @@ function isNotebookModelPath(modelPath) {
   const segments = modelPath.split("/");
   return segments.length === 2 && segments[0] === NOTEBOOKS_DIR;
 }
+function isDocumentModelPath(modelPath) {
+  if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
+    return false;
+  const segments = modelPath.split("/");
+  return segments.length === 2 && (segments[0] === NOTEBOOKS_DIR || segments[0] === DASHBOARDS_DIR);
+}
+function documentKind(modelPath, tagKind) {
+  if (tagKind === "notebook" || tagKind === "dashboard")
+    return tagKind;
+  return isNotebookModelPath(modelPath) ? "notebook" : "dashboard";
+}
 function routeOfNote(text) {
   return routeOf3({ value: text.trimStart() });
 }
@@ -15775,34 +15981,104 @@ function dashboardDescriptionNotes(notes) {
   return notes.filter((n) => line !== undefined && (n.at?.range.start.line ?? 0) > line).map((n) => n.text);
 }
 function hasArtifactLineOutsideBlocks(source, artifactLine) {
+  return findArtifactLineOutsideBlocks(source, artifactLine) !== undefined;
+}
+function blockCloserFinder(lines) {
+  const byKey = new Map;
+  lines.forEach((line, at) => {
+    const indent = /^[ \t]*/.exec(line)?.[0].length ?? 0;
+    for (const closer of ["|##", "|#"])
+      if (closesBlock(line, 0, indent, closer)) {
+        const key = `${indent}${closer}`;
+        const ats = byKey.get(key);
+        if (ats)
+          ats.push(at);
+        else
+          byKey.set(key, [at]);
+        break;
+      }
+  });
+  const cursor = new Map;
+  return (column, closer, after) => {
+    const key = `${column}${closer}`;
+    const ats = byKey.get(key) ?? [];
+    let k = cursor.get(key) ?? 0;
+    while (k < ats.length && ats[k] <= after)
+      k++;
+    cursor.set(key, k);
+    return ats[k] ?? -1;
+  };
+}
+function findArtifactLineOutsideBlocks(source, artifactLine) {
   const lines = source.split(/\r\n|\r|\n/);
+  const closerAfter = blockCloserFinder(lines);
   for (let i = 0;i < lines.length; i++) {
     const trimmed = lines[i].trimStart();
     const opener = /^(#{1,2})\|/.exec(trimmed);
     if (opener) {
-      const wanted = `|${opener[1]}`;
-      if (trimmed.includes(wanted, opener[0].length))
-        continue;
       const column = lines[i].length - trimmed.length;
-      let end = -1;
-      for (let at = i + 1;at < lines.length; at++)
-        if (closesBlock(lines[at], 0, column, wanted)) {
-          end = at;
-          break;
-        }
+      const end = closerAfter(column, `|${opener[1]}`, i);
       if (end !== -1) {
+        const block = [trimmed, ...lines.slice(i + 1, end)].join(`
+`);
+        if (artifactLine.test(block))
+          return block;
         i = end;
         continue;
       }
     }
     if (artifactLine.test(trimmed))
-      return true;
+      return trimmed;
   }
-  return false;
+  return lexedArtifactNote(source, lines, artifactLine);
+}
+function lexedArtifactNote(source, lines, artifactLine) {
+  if (!lines.some((line) => /\S.*##/.test(line) && !/^\s*##/.test(line))) {
+    return;
+  }
+  const translator = new MalloyTranslator(NOTEBOOK_PARSE_URL, null, {
+    urls: { [NOTEBOOK_PARSE_URL]: source }
+  });
+  const parse = translator.parseStep.step(translator).parse;
+  const stream = parse?.tokenStream;
+  const tokens = stream?.getTokens?.() ?? [];
+  const name = (type) => stream?.tokenSource?.vocabulary?.getSymbolicName(type);
+  const text = (i) => source.slice(tokens[i].startIndex, tokens[i].stopIndex + 1);
+  for (let i = 0;i < tokens.length; i++) {
+    const kind = name(tokens[i].type);
+    if (kind === "DOC_ANNOTATION" && artifactLine.test(text(i))) {
+      return text(i).trim();
+    }
+    if (kind !== "DOC_BLOCK_ANNOTATION_BEGIN")
+      continue;
+    let body = "";
+    let j = i;
+    while (name(tokens[j].type) !== "BLOCK_ANNOTATION_END") {
+      body += text(j);
+      if (++j >= tokens.length || name(tokens[j].type) === "EOF") {
+        return;
+      }
+    }
+    if (artifactLine.test(body))
+      return body;
+    i = j;
+  }
+  return;
+}
+function artifactKindOfNotes(notes) {
+  const tag = motlyTag(notes.map((note) => note.text).filter(isArtifactNoteText));
+  return tagText(tag?.tag("artifact"), "kind");
 }
 function claimsToBeANotebook(source) {
   return hasArtifactLineOutsideBlocks(source, ARTIFACT_NOTE);
 }
+function artifactTagText(source) {
+  return findArtifactLineOutsideBlocks(source, ARTIFACT_NOTE);
+}
+function artifactKindInText(source) {
+  const line = artifactTagText(source);
+  return line === undefined ? undefined : tagText(motlyTag([line])?.tag("artifact"), "kind");
+}
 function parseNotebookText(text) {
   try {
     const { problems, parse } = translateToParse(text);
@@ -16062,7 +16338,9 @@ ${body}`;
         codeLine: (codeLineOf(span.startCp, span.stopCp) ?? span.startLine) - span.startLine,
         ...caption !== undefined && { caption },
         startLine: span.startLine,
-        endLine: span.endLine
+        endLine: span.endLine,
+        start: span.start,
+        end: span.end
       });
       continue;
     }
@@ -16116,7 +16394,7 @@ ${body}`;
     });
   }
   const runCount = items.filter((item) => item.kind === "statement" && item.run).length;
-  if (runCount !== modelDef.queryList.length) {
+  if (modelDef && runCount !== modelDef.queryList.length) {
     return refuse({
       line: 1,
       message: `Line 1: the notebook reader found ${runCount} run: statements where Malloy compiled ${modelDef.queryList.length}, so its query cells cannot be matched to their results. Fix: none in the file; report it, since the reader and the compiler disagree.`
@@ -16131,7 +16409,7 @@ ${body}`;
     if (item.kind === "statement") {
       lineRun = undefined;
       const queryIndex = item.run ? runsSeen++ : undefined;
-      cells.push(item.run && belowTag ? {
+      const pushed = item.run && belowTag ? {
         kind: "query",
         type: "code",
         text: item.text,
@@ -16160,7 +16438,9 @@ ${body}`;
         },
         startLine: item.startLine,
         endLine: item.endLine
-      });
+      };
+      cellOffsets.set(pushed, [item.start, item.end]);
+      cells.push(pushed);
       return;
     }
     const { note } = item;
@@ -16239,9 +16519,12 @@ function readMarkdownBlocks(parse, text) {
   return blocks;
 }
 function notebookReaderProblem(modelPath, text, modelDef, url) {
-  if (!isNotebookModelPath(modelPath))
+  if (!isDocumentModelPath(modelPath))
     return;
-  if (artifactNoteLine(ownModelNoteObjects(modelDef)) === undefined)
+  const notes = ownModelNoteObjects(modelDef);
+  if (artifactNoteLine(notes) === undefined)
+    return;
+  if (documentKind(modelPath, artifactKindOfNotes(notes)) !== "notebook")
     return;
   const parse = parseNotebookText(text);
   const error = isNotebookReaderError(parse) ? parse : readNotebookCells(parse, modelDef, text).error;
@@ -16261,7 +16544,7 @@ function notebookReaderProblem(modelPath, text, modelDef, url) {
     }
   };
 }
-var NOTEBOOKS_DIR = "notebooks", ARTIFACT_NOTE, MARKDOWN_ROUTE = "markdown", attachedNowhereFix = (block) => block ? "write it as a floating `##|(markdown)` block closed by `|##` for prose that stands on its own, or move it directly above the statement it describes." : "write it as a floating `##(markdown)` line for prose that stands on its own, or move it directly above the statement it describes.", isRuleNode = (node) => node !== undefined && node.ruleIndex !== undefined && (node.childCount ?? 0) > 0, callAccessor = (node, name) => {
+var NOTEBOOKS_DIR = "notebooks", DASHBOARDS_DIR = "dashboards", ARTIFACT_NOTE, ANY_ARTIFACT_NOTE, MARKDOWN_ROUTE = "markdown", attachedNowhereFix = (block) => block ? "write it as a floating `##|(markdown)` block closed by `|##` for prose that stands on its own, or move it directly above the statement it describes." : "write it as a floating `##(markdown)` line for prose that stands on its own, or move it directly above the statement it describes.", cellOffsets, isRuleNode = (node) => node !== undefined && node.ruleIndex !== undefined && (node.childCount ?? 0) > 0, callAccessor = (node, name) => {
   const accessor = node[name];
   return typeof accessor === "function" ? accessor.call(node) : undefined;
 }, STATEMENT_ACCESSORS, NOTEBOOK_PARSE_URL = "file:///publisher-notebook-reader/notebook.malloy", NOTE_TOKENS, TEXT_BLOCK_NAME;
@@ -16270,7 +16553,63 @@ var init_notebook = __esm(() => {
   init_annotations();
   init_query_text();
   init_motly();
-  ARTIFACT_NOTE = /^##[ \t]*artifact\b/;
+  ARTIFACT_NOTE = {
+    test(note) {
+      const prefix = /^##\|?(?:[ \t\r\n]|$)/.exec(note);
+      if (!prefix)
+        return false;
+      const bare = /[0-9A-Za-z_\u00C0-\u024F\u1E00-\u1EFF]+/y;
+      let depth = 0;
+      let before = "";
+      for (let i = prefix[0].length;i < note.length; i++) {
+        const c = note[i];
+        if (/\s/.test(c))
+          continue;
+        if (note.startsWith("-...", i)) {
+          i += 3;
+          before = "";
+          continue;
+        }
+        bare.lastIndex = i;
+        const word = bare.exec(note)?.[0];
+        if (c === '"' || c === "'") {
+          const fence = note.startsWith(c.repeat(3), i) ? c.repeat(3) : c;
+          i += fence.length;
+          while (i < note.length && !note.startsWith(fence, i))
+            i += note[i] === "\\" ? 2 : 1;
+          i += fence.length - 1;
+        } else if (c === "`") {
+          let end = i + 1;
+          while (end < note.length && note[end] !== "`" && note[end] !== `
+`)
+            end += note[end] === "\\" ? 2 : 1;
+          const name = note.slice(i + 1, end);
+          if (depth === 0 && name === "artifact" && !/[=.-]/.test(before))
+            return true;
+          i = end;
+        } else if (c === "#") {
+          const eol = note.indexOf(`
+`, i);
+          if (eol < 0)
+            return false;
+          i = eol;
+          continue;
+        } else if (c === "{" || c === "[")
+          depth++;
+        else if (c === "}" || c === "]")
+          depth--;
+        else if (word) {
+          if (depth === 0 && word === "artifact" && !/[=.-]/.test(before))
+            return true;
+          i += word.length - 1;
+        }
+        before = c;
+      }
+      return false;
+    }
+  };
+  ANY_ARTIFACT_NOTE = /^#{1,2}\|?(?=[ \t\r\n])(?:[^"'\\]|\\.|"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')*?(?<![\w\u00C0-\u024F\u1E00-\u1EFF.$-])artifact(?![\w\u00C0-\u024F\u1E00-\u1EFF])/;
+  cellOffsets = new WeakMap;
   STATEMENT_ACCESSORS = [
     ["runStatement", "run"],
     ["docAnnotations", "notes"],
@@ -22975,6 +23314,7 @@ function malloyGivenToApi(given) {
     type: renderedType,
     annotations: allNotes.filter((note) => !isReservedRoute(note.route) && note.route !== MARKDOWN_ROUTE).map((note) => note.text),
     ...readGivenControlSpec(allNotes.map((note) => note.text)),
+    ...allNotes.some((note) => note.route === "secure") && { secure: true },
     default: given._internal?.defaultText
   };
 }
@@ -23088,9 +23428,12 @@ var init_given = __esm(() => {
 });
 
 // src/service/dashboard.ts
-import { isSourceDef as isSourceDef5 } from "@malloydata/malloy";
+import { isJoined as isJoined3, isSourceDef as isSourceDef5 } from "@malloydata/malloy";
+function queryTiles(tiles) {
+  return (tiles ?? []).filter((tile) => tile.kind === "query");
+}
 function filterPublisherOwnedRenderLogs(logs, modelPath) {
-  const couldBeATile = isDashboardModelPath(modelPath);
+  const couldBeATile = isDocumentModelPath(modelPath);
   return logs.filter((log) => {
     const match = UNKNOWN_RENDER_TAG.exec(log.message ?? "");
     if (!match)
@@ -23115,6 +23458,24 @@ function dashboardSlug(modelPath) {
 function matchesDocumentedDashboardName(slug) {
   return /^[a-zA-Z0-9_-]+$/.test(slug);
 }
+function readTileEntries(artifact) {
+  return artifact.array("tiles")?.flatMap((entry) => {
+    const text = tagText(entry);
+    if (text === undefined)
+      return [];
+    if (tagText(entry, "kind") !== "text")
+      return [{ kind: "query", expression: text }];
+    const colspan = positiveInteger(tagNumeric(entry, "colspan"));
+    return [
+      {
+        kind: "text",
+        name: text,
+        ...colspan !== undefined && { colspan },
+        ...entry.has("break") && { break: true }
+      }
+    ];
+  });
+}
 function readArtifactTag(tag, declaredType) {
   const artifact = tag.tag("artifact");
   if (!artifact)
@@ -23123,12 +23484,32 @@ function readArtifactTag(tag, declaredType) {
   const autorun = readAutorun(artifact);
   return {
     title: tagText(artifact, "title"),
-    tiles: artifact.array("tiles")?.filter((tile) => tagText(tile, "kind") !== "text").map((tile) => tagText(tile)).filter((tile) => tile !== undefined),
+    kind: tagText(artifact, "kind"),
+    tiles: readTileEntries(artifact),
     dashboardColumns: tag.tag("dashboard")?.has("columns") ? positiveInteger(tagNumeric(tag.tag("dashboard"), "columns")) : artifact.array("tiles") ? positiveInteger(tagNumeric(artifact, "dashboard_columns")) : undefined,
     givens,
     autorun
   };
 }
+function readDocumentTileExpressions(notes) {
+  const tag = motlyTag(notes);
+  const composite = tag ? readArtifactTag(tag, () => {
+    return;
+  }) : undefined;
+  return (composite?.tiles ?? []).flatMap((entry) => entry.kind === "text" ? [] : [entry.expression]);
+}
+function readTextBlocks(notes) {
+  return notes.flatMap((note) => {
+    const newline = note.indexOf(`
+`);
+    const opener = parseMarkdownOpener(newline === -1 ? note : note.slice(0, newline));
+    if (opener?.level !== 2 || !opener.name)
+      return [];
+    const body = newline === -1 ? "" : note.slice(newline + 1);
+    return [{ name: opener.name, body: body.replace(/\r\n?/g, `
+`) }];
+  });
+}
 function positiveInteger(value) {
   return value !== undefined && Number.isInteger(value) && value >= 1 ? value : undefined;
 }
@@ -23176,6 +23557,11 @@ function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames, gatesB
     for (const field of obj.fields) {
       const fieldName = field.as || field.name;
       fieldNames.add(fieldName);
+      if (isJoined3(field)) {
+        for (const joined of field.fields) {
+          fieldNames.add(`${fieldName}.${joined.as || joined.name}`);
+        }
+      }
       if (field.type === "turtle") {
         const refs = new Set(sourceGivens);
         collectGivenRefs(field.pipeline, refs);
@@ -23246,7 +23632,11 @@ function tagTextArray(tag, ...path) {
   return items?.map((item) => tagText(item)).filter(isString);
 }
 function resolveTileGivens(tile, facts) {
-  return referencedTileGivens(tile, facts)?.filter((name) => facts.givens.has(name));
+  const reads = referencedTileGivens(tile, facts);
+  if (reads === undefined)
+    return;
+  const gateReads = facts.compiledTileGivens?.get(normalizeTileExpression(tile))?.gateReads ?? [];
+  return Array.from(new Set([...reads, ...gateReads])).filter((name) => facts.givens.has(name));
 }
 function resolveTileLayout(tile, facts) {
   const annotations = tileAnnotations(tile, facts);
@@ -23258,7 +23648,15 @@ function tileAnnotations(tile, facts) {
 }
 function referencedTileGivens(tile, facts) {
   const normalized = normalizeTileExpression(tile);
-  return facts.viewGivens.get(normalized) ?? facts.queries.find((query) => query.name === normalized)?.givens;
+  const statically = facts.viewGivens.get(normalized) ?? facts.queries.find((query) => query.name === normalized)?.givens;
+  const compiled = facts.compiledTileGivens?.get(normalized)?.reads;
+  if (!compiled)
+    return statically;
+  const rank = (name) => {
+    const at = statically?.indexOf(name) ?? -1;
+    return at === -1 ? Number.MAX_SAFE_INTEGER : at;
+  };
+  return [...compiled].sort((a, b) => rank(a) - rank(b));
 }
 function buildGivenSpecs(names, facts) {
   const declarations = facts.givens;
@@ -23283,6 +23681,9 @@ function givenSpec(declaration, facts) {
     type: declaration.type,
     default: declaration.type.startsWith("filter<") && declaration.default !== undefined ? unwrapFilterLiteral(declaration.default) : declaration.default,
     annotations: declaration.annotations.filter((text) => /^##?\(/.test(text) && !isMarkdownNote(text)),
+    ...declaration.annotations.some((text) => /^#\(secure\)/.test(text)) && {
+      secure: true
+    },
     ...control
   };
 }
@@ -23291,7 +23692,7 @@ function factsCarryArtifactTag(facts) {
     return true;
   return facts.queries.some((query) => Boolean(motlyTag(query.annotations)?.tag("artifact")));
 }
-function buildDashboardManifest(facts) {
+function buildDashboardManifest(facts, options) {
   const name = dashboardSlug(facts.modelPath);
   const base = {
     name,
@@ -23300,26 +23701,44 @@ function buildDashboardManifest(facts) {
   const modelTag = motlyTag(facts.modelAnnotations);
   const declaredType = (name2) => facts.givens.get(name2)?.type;
   const composite = modelTag ? readArtifactTag(modelTag, declaredType) : undefined;
-  if (composite?.tiles?.length) {
-    const tiles = composite.tiles.map((query) => {
+  const kind = documentKind(facts.modelPath, composite?.kind);
+  if (composite?.tiles && (composite.tiles.length > 0 || kind === "notebook")) {
+    const blocks = readTextBlocks(facts.modelAnnotations);
+    const tiles = composite.tiles.map((entry) => {
+      if (entry.kind === "text") {
+        return {
+          kind: "text",
+          name: entry.name,
+          markdown: blocks.find((block) => block.name === entry.name)?.body ?? "",
+          ...entry.colspan !== undefined && { colspan: entry.colspan },
+          ...entry.break && { break: true }
+        };
+      }
+      const query = entry.expression;
+      if (options?.restrictedTiles?.has(normalizeTileExpression(query))) {
+        return { kind: "query", query, restricted: true };
+      }
       const givenNames = resolveTileGivens(query, facts);
       const layout = resolveTileLayout(query, facts);
       return {
+        kind: "query",
         query,
         ...givenNames ? { givenNames } : {},
         ...layout
       };
     });
+    const queries = queryTiles(tiles).filter((tile) => !tile.restricted);
     const doc = docCommentTitleAndDescription(facts.descriptionNotes ?? facts.modelAnnotations, composite.title);
     return {
       ...base,
+      kind,
       title: doc.title ?? name,
       description: doc.description,
       tiles,
-      dashboardColumns: composite.dashboardColumns,
+      dashboardColumns: kind === "notebook" ? 1 : composite.dashboardColumns,
       startingGivens: composite.givens,
       autorun: composite.autorun,
-      givens: buildGivenSpecs(tiles.some((tile) => tile.givenNames === undefined) ? Array.from(facts.givens.keys()) : tiles.flatMap((tile) => tile.givenNames ?? []), facts)
+      givens: buildGivenSpecs(queries.some((tile) => tile.givenNames === undefined) ? Array.from(facts.givens.keys()) : queries.flatMap((tile) => (referencedTileGivens(tile.query, facts) ?? []).filter((name2) => facts.givens.has(name2))), facts)
     };
   }
   for (const query of facts.queries) {
@@ -23330,6 +23749,7 @@ function buildDashboardManifest(facts) {
     const doc = docCommentTitleAndDescription(query.annotations, artifact.title);
     return {
       ...base,
+      kind: "dashboard",
       title: doc.title ?? name,
       description: doc.description,
       query: query.name,
@@ -23371,19 +23791,33 @@ function lintDashboard(facts, manifest) {
       continue;
     add(`\`${property}\` in the artifact tag does nothing in Publisher` + (ARTIFACT_PROPERTY_REPLACEMENTS[property] ? `. ${ARTIFACT_PROPERTY_REPLACEMENTS[property]}` : property === "tiles" ? `: a dashboard whose tag sits on a query is that one query's ` + `result. Move the tag to model level (\`## artifact\`) to ` + `combine separate queries into a grid.` : `.`));
   }
+  const textBlocks = readTextBlocks(facts.modelAnnotations);
   for (const entry of artifactTag?.array("tiles") ?? []) {
     const kind = tagText(entry, "kind");
-    if (kind === "text") {
-      add(`\`${tagText(entry) ?? "a tile"}\` in \`tiles=[…]\` is a text tile, ` + `but Publisher does not render text tiles yet, so it is left out ` + `of the page.`);
+    const label = tagText(entry) ?? "a tile";
+    const read = kind === "text" ? ["kind", "colspan", "break"] : [];
+    const carried = Object.keys(entry.dict ?? {}).filter((property) => !read.includes(property) && !(property === "kind" && kind === "query"));
+    if (carried.length > 0) {
+      const named = carried.map((property) => `\`${property}\``).join(", ");
+      add(kind === "text" ? `\`${label}\` in \`tiles=[…]\` carries ${named}, which Publisher does not read: a text tile's entry reads only \`colspan\` and \`break\`.` : `\`${label}\` in \`tiles=[…]\` carries ${named}, ` + `which Publisher does not read: a tile entry is the run expression ` + `alone. Per-tile presentation goes on the view the tile names ` + `(\`# colspan\`, \`# break\`, \`# label\`, \`# subtitle\`, ` + `\`# borderless\`).`);
+    }
+    if (kind !== "text")
       continue;
+    if (!textBlocks.some((block) => block.name === label)) {
+      add(`text tile \`${label}\` has no \`##|(markdown) ${label}\` block, so it shows nothing. ` + `Fix: write that block, with the text on the lines below its opener and \`|##\` after.`);
     }
-    const carried = Object.keys(entry.dict ?? {}).filter((property) => !(property === "kind" && kind === "query"));
-    if (carried.length === 0)
-      continue;
-    const named = carried.map((property) => `\`${property}\``).join(", ");
-    add(`\`${tagText(entry) ?? "a tile"}\` in \`tiles=[…]\` carries ${named}, ` + `which Publisher does not read: a tile entry is the run expression ` + `alone. Per-tile presentation goes on the view the tile names ` + `(\`# colspan\`, \`# break\`, \`# label\`, \`# subtitle\`, ` + `\`# borderless\`).`);
+    if (entry.has("colspan") && positiveInteger(tagNumeric(entry, "colspan")) === undefined) {
+      add(`colspan on text tile \`${label}\` must be a positive integer. The tile spans one column.`);
+    }
+  }
+  const seenBlocks = new Set;
+  for (const { name } of textBlocks) {
+    if (seenBlocks.has(name)) {
+      add(`the \`##|(markdown) ${name}\` block is written twice, and only the first is shown. Fix: rename or delete the second.`);
+    }
+    seenBlocks.add(name);
   }
-  for (const { query: tile } of manifest.tiles ?? []) {
+  for (const { query: tile } of manifest.kind === "notebook" ? [] : queryTiles(manifest.tiles)) {
     const tag = motlyTag(tileAnnotations(tile, facts) ?? []);
     if (!tag?.has("colspan"))
       continue;
@@ -23402,21 +23836,28 @@ function lintDashboard(facts, manifest) {
   }
   const columnsTag = ownTag?.tag("dashboard");
   if (columnsTag?.has("columns")) {
-    if (positiveInteger(tagNumeric(columnsTag, "columns")) === undefined) {
+    const width = positiveInteger(tagNumeric(columnsTag, "columns"));
+    if (width === undefined) {
       const raw = tagText(columnsTag, "columns");
       add(`# dashboard { columns=… } must be a positive integer, got ` + `${raw === undefined ? "a value that could not be read" : JSON.stringify(raw)}. ` + `The grid falls back to the default.`);
+    } else if (width > MAX_BUILDER_COLUMNS) {
+      add(`# dashboard { columns=${width} } is wider than the builder offers (${MAX_BUILDER_COLUMNS}); the reader still renders it.`);
     }
   }
-  for (const { query: tile } of manifest.tiles ?? []) {
+  for (const { query: tile } of queryTiles(manifest.tiles)) {
     const parts = plainTileParts(tile);
     if (!parts)
       continue;
     const resolved = referencedTileGivens(tile, facts) !== undefined || "name" in parts && facts.queries.some((query) => query.name === parts.name);
+    if (!resolved && "name" in parts && textBlocks.some((block) => block.name === parts.name)) {
+      add(`tile "${tile}" names the ##|(markdown) ${tile} block but has no kind=text, so it is read as a query. Fix: write ${tile} { kind=text }`, "error");
+      continue;
+    }
     if (!resolved) {
       const detail = "source" in parts ? facts.sourceFields.has(parts.source) ? `source "${parts.source}" has no view "${parts.view}"` : `no source "${parts.source}" in this file` : `no query named "${parts.name}" in this file`;
       add(`tile "${tile}" does not resolve: ${detail}.`, "error");
     }
-    const unbindable = (referencedTileGivens(tile, facts) ?? []).filter((name) => !facts.givens.has(name));
+    const unbindable = Array.from(new Set(referencedTileGivens(tile, facts) ?? [])).filter((name) => !facts.givens.has(name));
     for (const name of unbindable) {
       add(`tile "${tile}" filters by given "${name}", which this file does ` + `not import, so no control is shown for it and it stays at its ` + `default. Add it to an import in ${facts.modelPath}.`);
     }
@@ -23579,9 +24020,40 @@ function lintSelfDrills(facts) {
   }
   return Array.from(findings.values());
 }
-var DASHBOARDS_DIR = "dashboards", COMPONENT_FILE_SUFFIXES, MODEL_FILE_SUFFIX2 = ".malloy", PUBLISHER_OWNED_TAGS, RENDERER_CHILD_TAGS, UNKNOWN_RENDER_TAG, COMPOSITE_ARTIFACT_PROPERTIES, QUERY_ARTIFACT_PROPERTIES, ARTIFACT_PROPERTY_REPLACEMENTS, NAME_THE_TILES = `Name the views to show: tiles=["source -> view", …].`;
+async function compileTileGivens(tiles, materializer, fallbackRegistry, gatesOf, skip, onPrepared, onCompileError) {
+  const compiled = new Map;
+  for (const tile of tiles) {
+    const key = normalizeTileExpression(tile);
+    if (compiled.has(key) || skip?.has(key))
+      continue;
+    let prepared;
+    try {
+      prepared = await materializer.loadQuery(`run: ${tile}`).getPreparedQuery();
+    } catch (error) {
+      onCompileError?.(tile, error);
+      continue;
+    }
+    await onPrepared?.(tile, prepared);
+    const usage = prepared._query?.givenUsage;
+    if (!usage)
+      continue;
+    const registry = prepared._modelDef?.givens ?? fallbackRegistry ?? {};
+    const reads = usage.map((given) => registry[given.id]?.name).filter((name) => name !== undefined);
+    const target = prepared._query?.structRef;
+    const sourceName = typeof target === "string" ? target : target?.as || target?.name;
+    const gateReads = new Set;
+    for (const expr of sourceName ? gatesOf(sourceName) ?? [] : []) {
+      for (const name of referencedGivenNames(expr))
+        gateReads.add(name);
+    }
+    compiled.set(key, { reads, gateReads: Array.from(gateReads) });
+  }
+  return compiled;
+}
+var MAX_BUILDER_COLUMNS = 24, COMPONENT_FILE_SUFFIXES, MODEL_FILE_SUFFIX2 = ".malloy", PUBLISHER_OWNED_TAGS, RENDERER_CHILD_TAGS, UNKNOWN_RENDER_TAG, COMPOSITE_ARTIFACT_PROPERTIES, QUERY_ARTIFACT_PROPERTIES, ARTIFACT_PROPERTY_REPLACEMENTS, NAME_THE_TILES = `Name the views to show: tiles=["source -> view", …].`;
 var init_dashboard = __esm(() => {
   init_annotations();
+  init_authorize();
   init_notebook();
   init_given();
   init_motly();
@@ -23800,9 +24272,8 @@ var init_filter = __esm(() => {
 
 // src/service/source_extraction.ts
 import {
-  isJoined as isJoined3,
-  isSourceDef as isSourceDef6,
-  modelDefToModelInfo
+  isJoined as isJoined4,
+  isSourceDef as isSourceDef6
 } from "@malloydata/malloy";
 function joinFieldNamesUnresolvableDeclaration(field, modelDef) {
   const ids = [field.referenceID, field.sourceID].filter((id) => !!id);
@@ -23833,7 +24304,7 @@ function considerNoteOwner(declaredBy, note, candidate) {
 function collectSourceInfos(modelDef) {
   const names = Object.keys(modelDef.contents).filter((name) => isSourceDef6(modelDef.contents[name]));
   const exports = [...new Set([...modelDef.exports, ...names])];
-  return modelDefToModelInfo({ ...modelDef, exports }).entries.filter((entry) => entry.kind === "source");
+  return modelInfoOf({ ...modelDef, exports }).entries.filter((entry) => entry.kind === "source");
 }
 function extractSourcesFromModelDef(modelDef, givens, onParseError) {
   const filterMap = new Map;
@@ -23971,7 +24442,7 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
         continue;
       }
       const fieldName = field.as || field.name;
-      if (isJoined3(field) && isSourceDef6(field)) {
+      if (isJoined4(field) && isSourceDef6(field)) {
         const joinedStruct = field;
         if (joinFieldNamesUnresolvableDeclaration(joinedStruct, modelDef)) {
           continue;
@@ -24029,16 +24500,17 @@ var init_source_extraction = __esm(() => {
   init_annotations();
   init_authorize();
   init_authorize_routes();
+  init_model_info();
   init_filter();
   init_gate_registry_walk();
 });
 
 // src/utils.ts
-import * as fs from "fs";
-import * as path from "path";
+import * as fs2 from "fs";
+import * as path2 from "path";
 import { fileURLToPath } from "url";
 function ignoreDotfiles(file) {
-  return path.basename(file).startsWith(".");
+  return path2.basename(file).startsWith(".");
 }
 function errMessage(err) {
   return err instanceof Error ? err.message : String(err);
@@ -24047,11 +24519,11 @@ var URL_READER;
 var init_utils = __esm(() => {
   URL_READER = {
     readURL: (url) => {
-      let path2 = url.toString();
+      let path3 = url.toString();
       if (url.protocol == "file:") {
-        path2 = fileURLToPath(url);
+        path3 = fileURLToPath(url);
       }
-      return fs.promises.readFile(path2, "utf8");
+      return fs2.promises.readFile(path3, "utf8");
     }
   };
 });
@@ -24065,20 +24537,20 @@ init_error_wire();
 init_authorize();
 init_gate_classification();
 init_gate_dimension();
+init_model_info();
 var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
 import {
   contextOverlay,
   isSourceDef as isSourceDef7,
   MalloyConfig,
-  modelDefToModelInfo as modelDefToModelInfo2,
   Runtime
 } from "@malloydata/malloy";
 import {
   MalloySQLParser,
   MalloySQLStatementType
 } from "@malloydata/malloy-sql";
-import * as fs2 from "fs";
-import * as path2 from "path";
+import * as fs3 from "fs";
+import * as path3 from "path";
 import { parentPort, threadId } from "node:worker_threads";
 import { fileURLToPath as fileURLToPath2, pathToFileURL } from "url";
 
@@ -24265,6 +24737,254 @@ var EXPLORES_EMPTY_SUPPRESSES_CONVENTION = `"explores" in publisher.json is depr
 var EXPLORES_EMPTY_IS_UNCURATED = `"explores": [] in publisher.json does nothing. Fix: delete it.`;
 var QUERYABLE_SOURCES_DEPRECATION = `"queryableSources" in publisher.json does nothing. Fix: delete it.`;
 
+// src/service/package_retrieval.ts
+init_errors();
+import { createHash } from "crypto";
+import * as fs from "fs";
+import * as path from "path";
+var PACKAGE_REPRESENTATIONS = ["single", "facets"];
+var KEYPHRASE_MODES = ["auto", "never", "always"];
+var REFINE_LEVEL_NAMES = ["LOW", "MEDIUM", "HIGH"];
+var DEFAULT_RERANK_TOP_SOURCES = 8;
+var DEFAULT_REFINE_MIN_LEVEL = "MEDIUM";
+var RETRIEVAL_KEYS = [
+  "representation",
+  "keyphrases",
+  "refine",
+  "rerank",
+  "sourceMatch",
+  "sourceSummary",
+  "prompts"
+];
+var PROMPT_KEYS = [
+  "keyphrase",
+  "refine",
+  "rerank",
+  "sourceMatch",
+  "sourceSummary"
+];
+var REFINE_KEYS = ["enabled", "minLevel"];
+var RERANK_KEYS = ["enabled", "topSources"];
+var SOURCE_MATCH_KEYS = ["enabled"];
+var SOURCE_SUMMARY_KEYS = ["enabled"];
+var DEFAULT_PACKAGE_RETRIEVAL = {
+  representation: "single",
+  keyphrases: "auto",
+  prompts: {}
+};
+function refineSettingsOf(retrieval) {
+  return retrieval.refine ?? {
+    enabled: "auto",
+    minLevel: DEFAULT_REFINE_MIN_LEVEL
+  };
+}
+function rerankSettingsOf(retrieval) {
+  return retrieval.rerank ?? {
+    enabled: "auto",
+    topSources: DEFAULT_RERANK_TOP_SOURCES
+  };
+}
+function sourceMatchSettingsOf(retrieval) {
+  return retrieval.sourceMatch ?? { enabled: "auto" };
+}
+function sourceSummarySettingsOf(retrieval) {
+  return retrieval.sourceSummary ?? { enabled: "auto" };
+}
+function fail(message) {
+  throw new PackageManifestError(message);
+}
+function describe(value) {
+  return JSON.stringify(value) ?? "nothing";
+}
+function stepsOutside(within) {
+  return within === ".." || within.startsWith(`..${path.sep}`) || path.isAbsolute(within);
+}
+function resolvePromptPath(packageRoot, rel, promptKey = "keyphrase") {
+  const key = `retrieval.prompts.${promptKey}`;
+  const fix = `Fix: use a path inside the package, e.g. "prompts/${promptKey}.md".`;
+  if (rel.includes("\x00") || path.isAbsolute(rel) || /^[A-Za-z]:[\\/]/.test(rel) || rel.startsWith("\\")) {
+    fail(`Invalid publisher.json ${key}: ${describe(rel)} is an absolute path. ${fix}`);
+  }
+  const root = path.resolve(packageRoot);
+  const resolved = path.resolve(root, rel);
+  const within = path.relative(root, resolved);
+  if (within === "" || stepsOutside(within)) {
+    fail(`Invalid publisher.json ${key}: ${describe(rel)} resolves outside the package directory. ${fix}`);
+  }
+  return resolved;
+}
+function objectBlock(raw, key, valid, fix) {
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    fail(`Invalid publisher.json ${key}: expected an object, got ${describe(raw)}. Fix: ${fix}`);
+  }
+  const obj = raw;
+  const unknown = Object.keys(obj).filter((k) => !valid.includes(k));
+  if (unknown.length > 0) {
+    fail(`Invalid publisher.json ${key}: unknown key ${unknown.map((k) => `'${k}'`).join(", ")}. ` + `Valid keys: ${valid.join(", ")}.`);
+  }
+  return obj;
+}
+function parseEnabled(value, key) {
+  if (value === undefined || value === null)
+    return "auto";
+  if (value === "auto" || value === true || value === false)
+    return value;
+  return fail(`Invalid publisher.json ${key}: expected "auto", true or false, got ${describe(value)}. ` + `Fix: "enabled": "auto" (on when the server has an LLM), true (require one) or false.`);
+}
+function parseRefine(raw) {
+  const obj = objectBlock(raw, "retrieval.refine", REFINE_KEYS, `"refine": { "enabled": "auto", "minLevel": "MEDIUM" }`);
+  const minLevel = obj.minLevel ?? DEFAULT_REFINE_MIN_LEVEL;
+  if (typeof minLevel !== "string" || !REFINE_LEVEL_NAMES.includes(minLevel)) {
+    fail(`Invalid publisher.json retrieval.refine.minLevel: expected one of ${REFINE_LEVEL_NAMES.join(", ")}, got ${describe(obj.minLevel)}. ` + `Fix: "minLevel": "MEDIUM" (drop candidates the model rates below it).`);
+  }
+  return {
+    enabled: parseEnabled(obj.enabled, "retrieval.refine.enabled"),
+    minLevel
+  };
+}
+function parseRerank(raw) {
+  const obj = objectBlock(raw, "retrieval.rerank", RERANK_KEYS, `"rerank": { "enabled": "auto", "topSources": ${DEFAULT_RERANK_TOP_SOURCES} }`);
+  const top = obj.topSources ?? DEFAULT_RERANK_TOP_SOURCES;
+  if (typeof top !== "number" || !Number.isSafeInteger(top) || top <= 0) {
+    fail(`Invalid publisher.json retrieval.rerank.topSources: expected a positive integer, got ${describe(obj.topSources)}. ` + `Fix: "topSources": ${DEFAULT_RERANK_TOP_SOURCES}`);
+  }
+  return {
+    enabled: parseEnabled(obj.enabled, "retrieval.rerank.enabled"),
+    topSources: top
+  };
+}
+function parseSourceMatch(raw) {
+  const obj = objectBlock(raw, "retrieval.sourceMatch", SOURCE_MATCH_KEYS, `"sourceMatch": { "enabled": "auto" }`);
+  return {
+    enabled: parseEnabled(obj.enabled, "retrieval.sourceMatch.enabled")
+  };
+}
+function parseSourceSummary(raw) {
+  const obj = objectBlock(raw, "retrieval.sourceSummary", SOURCE_SUMMARY_KEYS, `"sourceSummary": { "enabled": "auto" }`);
+  return {
+    enabled: parseEnabled(obj.enabled, "retrieval.sourceSummary.enabled")
+  };
+}
+function parsePackageRetrieval(raw) {
+  if (raw === undefined || raw === null) {
+    return {
+      representation: DEFAULT_PACKAGE_RETRIEVAL.representation,
+      keyphrases: DEFAULT_PACKAGE_RETRIEVAL.keyphrases,
+      promptPaths: {}
+    };
+  }
+  if (typeof raw !== "object" || Array.isArray(raw)) {
+    fail(`Invalid publisher.json retrieval: expected an object, got ${describe(raw)}. ` + `Fix: "retrieval": { "representation": "single" }`);
+  }
+  const obj = raw;
+  const unknown = Object.keys(obj).filter((k) => !RETRIEVAL_KEYS.includes(k));
+  if (unknown.length > 0) {
+    fail(`Invalid publisher.json retrieval: unknown key ${unknown.map((k) => `'${k}'`).join(", ")}. ` + `Valid keys: ${RETRIEVAL_KEYS.join(", ")}.`);
+  }
+  const representation = obj.representation ?? "single";
+  if (typeof representation !== "string" || !PACKAGE_REPRESENTATIONS.includes(representation)) {
+    fail(`Invalid publisher.json retrieval.representation: expected one of ${PACKAGE_REPRESENTATIONS.join(", ")}, got ${describe(obj.representation)}. ` + `Fix: "representation": "single" (one row per entity) or "facets" (a name row plus doc chunks).`);
+  }
+  const keyphrases = obj.keyphrases ?? "auto";
+  if (typeof keyphrases !== "string" || !KEYPHRASE_MODES.includes(keyphrases)) {
+    fail(`Invalid publisher.json retrieval.keyphrases: expected one of ${KEYPHRASE_MODES.join(", ")}, got ${describe(obj.keyphrases)}. ` + `Fix: "keyphrases": "auto".`);
+  }
+  const refine = obj.refine === undefined || obj.refine === null ? undefined : parseRefine(obj.refine);
+  const rerank = obj.rerank === undefined || obj.rerank === null ? undefined : parseRerank(obj.rerank);
+  const sourceMatch = obj.sourceMatch === undefined || obj.sourceMatch === null ? undefined : parseSourceMatch(obj.sourceMatch);
+  const sourceSummary = obj.sourceSummary === undefined || obj.sourceSummary === null ? undefined : parseSourceSummary(obj.sourceSummary);
+  const promptPaths = {};
+  if (obj.prompts !== undefined && obj.prompts !== null) {
+    if (typeof obj.prompts !== "object" || Array.isArray(obj.prompts)) {
+      fail(`Invalid publisher.json retrieval.prompts: expected an object, got ${describe(obj.prompts)}. ` + `Fix: "prompts": { "keyphrase": "prompts/keyphrase.md" }`);
+    }
+    const prompts = obj.prompts;
+    const badPrompt = Object.keys(prompts).filter((k) => !PROMPT_KEYS.includes(k));
+    if (badPrompt.length > 0) {
+      fail(`Invalid publisher.json retrieval.prompts: unknown key ${badPrompt.map((k) => `'${k}'`).join(", ")}. ` + `Valid keys: ${PROMPT_KEYS.join(", ")}.`);
+    }
+    for (const key of PROMPT_KEYS) {
+      const value = prompts[key];
+      if (value === undefined || value === null)
+        continue;
+      if (typeof value !== "string" || value.trim() === "") {
+        fail(`Invalid publisher.json retrieval.prompts.${key}: expected a file path inside the package, got ${describe(value)}. ` + `Fix: "${key}": "prompts/${key}.md"`);
+      }
+      promptPaths[key] = value;
+    }
+  }
+  return {
+    representation,
+    keyphrases,
+    ...refine ? { refine } : {},
+    ...rerank ? { rerank } : {},
+    ...sourceMatch ? { sourceMatch } : {},
+    ...sourceSummary ? { sourceSummary } : {},
+    promptPaths
+  };
+}
+async function readPromptFile(packageRoot, key, rel) {
+  const file = resolvePromptPath(packageRoot, rel, key);
+  const at = `retrieval.prompts.${key}`;
+  let text;
+  try {
+    const [realRoot, realFile] = await Promise.all([
+      fs.promises.realpath(packageRoot),
+      fs.promises.realpath(file)
+    ]);
+    const within = path.relative(realRoot, realFile);
+    if (stepsOutside(within)) {
+      fail(`Invalid publisher.json ${at}: ${describe(rel)} resolves outside the package directory through a link. ` + `Fix: copy the file into the package.`);
+    }
+    text = await fs.promises.readFile(realFile, "utf8");
+  } catch (error) {
+    if (error instanceof PackageManifestError)
+      throw error;
+    return fail(`Invalid publisher.json ${at}: cannot read ${describe(rel)} (${error.message}). ` + `Fix: create the file inside the package or remove the key.`);
+  }
+  if (text.trim() === "") {
+    fail(`Invalid publisher.json ${at}: ${describe(rel)} is empty. ` + `Fix: write the prompt, or remove the key to use the built-in one.`);
+  }
+  return {
+    path: rel,
+    text,
+    hash: createHash("sha256").update(text).digest("hex")
+  };
+}
+async function readPackageRetrieval(packageRoot, raw) {
+  const parsed = parsePackageRetrieval(raw);
+  const prompts = {};
+  for (const key of PROMPT_KEYS) {
+    const rel = parsed.promptPaths[key];
+    if (rel !== undefined) {
+      prompts[key] = await readPromptFile(packageRoot, key, rel);
+    }
+  }
+  return {
+    representation: parsed.representation,
+    keyphrases: parsed.keyphrases,
+    ...parsed.refine ? { refine: parsed.refine } : {},
+    ...parsed.rerank ? { rerank: parsed.rerank } : {},
+    ...parsed.sourceMatch ? { sourceMatch: parsed.sourceMatch } : {},
+    ...parsed.sourceSummary ? { sourceSummary: parsed.sourceSummary } : {},
+    prompts
+  };
+}
+function assertRequiredStagesAvailable(retrieval, llmConfigured) {
+  if (llmConfigured)
+    return;
+  for (const [key, stage] of [
+    ["refine", retrieval.refine],
+    ["rerank", retrieval.rerank],
+    ["sourceMatch", retrieval.sourceMatch],
+    ["sourceSummary", retrieval.sourceSummary]
+  ]) {
+    if (stage?.enabled === true) {
+      fail(`Invalid publisher.json retrieval.${key}.enabled: true needs an LLM, and this server has none configured. ` + `Fix: set retrieval.llm in publisher.config.json and LLM_API_KEY in the server's environment, ` + `or set "enabled": "auto" (on only when an LLM is configured) or false.`);
+    }
+  }
+}
+
 // src/package_load/package_load_worker.ts
 init_source_extraction();
 init_given();
@@ -24324,9 +25044,9 @@ function newRpcId() {
 }
 function callMain(send) {
   const requestId = newRpcId();
-  return new Promise((resolve2, reject2) => {
+  return new Promise((resolve3, reject2) => {
     pendingRpc.set(requestId, {
-      resolve: (value) => resolve2(value),
+      resolve: (value) => resolve3(value),
       reject: reject2
     });
     send(requestId);
@@ -24446,10 +25166,10 @@ function makeWorkerUrlReader(job) {
     readURL: async (url) => {
       if (url.protocol === "file:") {
         const filePath = fileURLToPath2(url);
-        if (job.replacement && path2.resolve(filePath) === path2.resolve(job.packagePath, job.replacement.modelPath)) {
+        if (job.replacement && path3.resolve(filePath) === path3.resolve(job.packagePath, job.replacement.modelPath)) {
           return job.replacement.source;
         }
-        return fs2.promises.readFile(filePath, "utf8");
+        return fs3.promises.readFile(filePath, "utf8");
       }
       const response = await callMain((requestId) => {
         const req = {
@@ -24499,8 +25219,8 @@ function buildWorkerMalloyConfig(job) {
   return config;
 }
 async function readPackageMetadata(packagePath, modelPaths) {
-  const manifestPath = path2.join(packagePath, PACKAGE_MANIFEST_NAME);
-  const contents = await fs2.promises.readFile(manifestPath, "utf8");
+  const manifestPath = path3.join(packagePath, PACKAGE_MANIFEST_NAME);
+  const contents = await fs3.promises.readFile(manifestPath, "utf8");
   let parsed;
   try {
     parsed = JSON.parse(contents);
@@ -24531,12 +25251,13 @@ async function readPackageMetadata(packagePath, modelPaths) {
     manifestLocation: typeof parsed.manifestLocation === "string" ? parsed.manifestLocation : null,
     materialization: materializationWithQueryMetadata(parsePackageMaterialization(parsed.materialization), queryMetadata.queryMetadata),
     scope: scope.scope,
-    manifestWarnings: manifestWarnings.length > 0 ? manifestWarnings : undefined
+    manifestWarnings: manifestWarnings.length > 0 ? manifestWarnings : undefined,
+    retrieval: await readPackageRetrieval(packagePath, parsed.retrieval)
   };
 }
 async function listPackageFiles(packagePath) {
   const files = await import_recursive_readdir.default(packagePath, [ignoreDotfiles]);
-  return files.map((full) => path2.relative(packagePath, full).replace(/\\/g, "/")).sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
+  return files.map((full) => path3.relative(packagePath, full).replace(/\\/g, "/")).sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
 }
 function filterModelPaths(allRelative) {
   return allRelative.filter((p) => p.endsWith(MODEL_FILE_SUFFIX) || p.endsWith(NOTEBOOK_FILE_SUFFIX));
@@ -24605,7 +25326,7 @@ function buildRuntimeForModel(job, malloyConfig) {
 }
 async function compileMalloyModel(job, malloyConfig, modelPath) {
   const compileStart = performance.now();
-  const fullPath = path2.join(job.packagePath, modelPath);
+  const fullPath = path3.join(job.packagePath, modelPath);
   const modelURL = pathToFileURL(fullPath);
   const importBaseURL = new URL(".", modelURL);
   const { runtime, urlReader, textFor } = buildRuntimeForModel(job, malloyConfig);
@@ -24655,7 +25376,7 @@ async function compileMalloyModel(job, malloyConfig, modelPath) {
     modelPath,
     modelType: "model",
     modelDef,
-    modelInfo: modelDefToModelInfo2(modelDef),
+    modelInfo: modelInfoOf(modelDef),
     sourceInfos,
     sources,
     queries,
@@ -24670,12 +25391,12 @@ async function compileMalloyModel(job, malloyConfig, modelPath) {
 }
 async function compileNotebookModel(job, malloyConfig, modelPath) {
   const compileStart = performance.now();
-  const fullPath = path2.join(job.packagePath, modelPath);
+  const fullPath = path3.join(job.packagePath, modelPath);
   const modelURL = pathToFileURL(fullPath);
   const importBaseURL = new URL(".", modelURL);
   const { runtime, urlReader } = buildRuntimeForModel(job, malloyConfig);
   const authorizeWarningCollection = authorizeWarningCollector();
-  const fileContents = await fs2.promises.readFile(modelURL, "utf8");
+  const fileContents = await fs3.promises.readFile(modelURL, "utf8");
   const parse = MalloySQLParser.parse(fileContents, modelPath);
   let mm;
   const perCellMM = parse.statements.map((stmt) => {
@@ -24699,7 +25420,7 @@ async function compileNotebookModel(job, malloyConfig, modelPath) {
       continue;
     }
     const currentModelDef = (await localMM.getModel())._modelDef;
-    const currentInfo = modelDefToModelInfo2(currentModelDef);
+    const currentInfo = modelInfoOf(currentModelDef);
     const newSources = collectSourceInfos(currentModelDef).filter((s) => !(s.name in oldSources));
     for (const s of newSources)
       oldSources[s.name] = s;
@@ -24781,7 +25502,7 @@ async function compileNotebookModel(job, malloyConfig, modelPath) {
     modelPath,
     modelType: "notebook",
     modelDef: finalModelDef,
-    modelInfo: finalModelDef ? modelDefToModelInfo2(finalModelDef) : undefined,
+    modelInfo: finalModelDef ? modelInfoOf(finalModelDef) : undefined,
     sourceInfos: finalSourceInfos,
     sources: finalSources,
     queries: finalQueries,