@malloy-publisher/server 0.9.2 → 0.9.4

package/dist/app/index.html

@@ -16,7 +16,7 @@ SPDX-License-Identifier: MIT
          href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap"
       />
       <title>Malloy Publisher</title>
-     <script type="module" crossorigin src="/assets/index-CGdwupPP.js"></script>
+     <script type="module" crossorigin src="/assets/index-BZEvIUf-.js"></script>
      <link rel="stylesheet" crossorigin href="/assets/index-DUqldpo0.css">
    </head>
    <body>

package/dist/package_load_worker.mjs

@@ -13195,7 +13195,7 @@ function toQueryTextProblem(problem) {
     ...problem.at ? { at: { range: problem.at.range } } : {}
   };
 }
-var GENERIC_INTERNAL_MESSAGE = "Internal server error.", GENERIC_UPSTREAM_MESSAGE = "Upstream connection error.", MAX_LOGGED_DETAIL_CHARS = 2000, FILESYSTEM_ACCESS_DESCRIPTIONS, NotImplementedError, BadRequestError, InvalidArgumentError, CompileRefusedError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, DashboardNotFoundError, ConnectionNotFoundError, TableNotFoundError, ConnectionError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, PackageManifestError, MaterializationEligibilityError, PublisherConfigError, ServerConfigurationError, FrozenConfigError, AccessDeniedError, QueryCompileError, NotQueryableError, OffSurfaceError, MaterializationNotFoundError, MaterializationConflictError, WriteConflictError, WriteRolledBackError, WriteVerifyError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
+var GENERIC_INTERNAL_MESSAGE = "Internal server error.", GENERIC_UPSTREAM_MESSAGE = "Upstream connection error.", MAX_LOGGED_DETAIL_CHARS = 2000, FILESYSTEM_ACCESS_DESCRIPTIONS, NotImplementedError, BadRequestError, InvalidArgumentError, CompileRefusedError, RenderTagRefusedError, UnparseableTextError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, DashboardNotFoundError, ConnectionNotFoundError, TableNotFoundError, ConnectionError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, PackageManifestError, MaterializationEligibilityError, PublisherConfigError, ServerConfigurationError, FrozenConfigError, AccessDeniedError, QueryCompileError, NotQueryableError, OffSurfaceError, MaterializationNotFoundError, MaterializationConflictError, WriteConflictError, WriteRolledBackError, WriteVerifyError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
 var init_errors = __esm(() => {
   init_constants();
   init_logger();
@@ -13218,6 +13218,15 @@ var init_errors = __esm(() => {
   };
   CompileRefusedError = class CompileRefusedError extends BadRequestError {
   };
+  RenderTagRefusedError = class RenderTagRefusedError extends CompileRefusedError {
+  };
+  UnparseableTextError = class UnparseableTextError extends CompileRefusedError {
+    detail;
+    constructor(message, detail) {
+      super(message);
+      this.detail = detail;
+    }
+  };
   EnvironmentNotFoundError = class EnvironmentNotFoundError extends Error {
     constructor(message) {
       super(message);
@@ -16021,6 +16030,39 @@ function findArtifactLineOutsideBlocks(source, artifactLine) {
     if (artifactLine.test(trimmed))
       return trimmed;
   }
+  return lexedArtifactNote(source, lines, artifactLine);
+}
+function lexedArtifactNote(source, lines, artifactLine) {
+  if (!lines.some((line) => /\S.*##/.test(line) && !/^\s*##/.test(line))) {
+    return;
+  }
+  const translator = new MalloyTranslator(NOTEBOOK_PARSE_URL, null, {
+    urls: { [NOTEBOOK_PARSE_URL]: source }
+  });
+  const parse = translator.parseStep.step(translator).parse;
+  const stream = parse?.tokenStream;
+  const tokens = stream?.getTokens?.() ?? [];
+  const name = (type) => stream?.tokenSource?.vocabulary?.getSymbolicName(type);
+  const text = (i) => source.slice(tokens[i].startIndex, tokens[i].stopIndex + 1);
+  for (let i = 0;i < tokens.length; i++) {
+    const kind = name(tokens[i].type);
+    if (kind === "DOC_ANNOTATION" && artifactLine.test(text(i))) {
+      return text(i).trim();
+    }
+    if (kind !== "DOC_BLOCK_ANNOTATION_BEGIN")
+      continue;
+    let body = "";
+    let j = i;
+    while (name(tokens[j].type) !== "BLOCK_ANNOTATION_END") {
+      body += text(j);
+      if (++j >= tokens.length || name(tokens[j].type) === "EOF") {
+        return;
+      }
+    }
+    if (artifactLine.test(body))
+      return body;
+    i = j;
+  }
   return;
 }
 function artifactKindOfNotes(notes) {
@@ -16296,7 +16338,9 @@ ${body}`;
         codeLine: (codeLineOf(span.startCp, span.stopCp) ?? span.startLine) - span.startLine,
         ...caption !== undefined && { caption },
         startLine: span.startLine,
-        endLine: span.endLine
+        endLine: span.endLine,
+        start: span.start,
+        end: span.end
       });
       continue;
     }
@@ -16350,7 +16394,7 @@ ${body}`;
     });
   }
   const runCount = items.filter((item) => item.kind === "statement" && item.run).length;
-  if (runCount !== modelDef.queryList.length) {
+  if (modelDef && runCount !== modelDef.queryList.length) {
     return refuse({
       line: 1,
       message: `Line 1: the notebook reader found ${runCount} run: statements where Malloy compiled ${modelDef.queryList.length}, so its query cells cannot be matched to their results. Fix: none in the file; report it, since the reader and the compiler disagree.`
@@ -16365,7 +16409,7 @@ ${body}`;
     if (item.kind === "statement") {
       lineRun = undefined;
       const queryIndex = item.run ? runsSeen++ : undefined;
-      cells.push(item.run && belowTag ? {
+      const pushed = item.run && belowTag ? {
         kind: "query",
         type: "code",
         text: item.text,
@@ -16394,7 +16438,9 @@ ${body}`;
         },
         startLine: item.startLine,
         endLine: item.endLine
-      });
+      };
+      cellOffsets.set(pushed, [item.start, item.end]);
+      cells.push(pushed);
       return;
     }
     const { note } = item;
@@ -16498,7 +16544,7 @@ function notebookReaderProblem(modelPath, text, modelDef, url) {
     }
   };
 }
-var NOTEBOOKS_DIR = "notebooks", DASHBOARDS_DIR = "dashboards", ARTIFACT_NOTE, ANY_ARTIFACT_NOTE, MARKDOWN_ROUTE = "markdown", attachedNowhereFix = (block) => block ? "write it as a floating `##|(markdown)` block closed by `|##` for prose that stands on its own, or move it directly above the statement it describes." : "write it as a floating `##(markdown)` line for prose that stands on its own, or move it directly above the statement it describes.", isRuleNode = (node) => node !== undefined && node.ruleIndex !== undefined && (node.childCount ?? 0) > 0, callAccessor = (node, name) => {
+var NOTEBOOKS_DIR = "notebooks", DASHBOARDS_DIR = "dashboards", ARTIFACT_NOTE, ANY_ARTIFACT_NOTE, MARKDOWN_ROUTE = "markdown", attachedNowhereFix = (block) => block ? "write it as a floating `##|(markdown)` block closed by `|##` for prose that stands on its own, or move it directly above the statement it describes." : "write it as a floating `##(markdown)` line for prose that stands on its own, or move it directly above the statement it describes.", cellOffsets, isRuleNode = (node) => node !== undefined && node.ruleIndex !== undefined && (node.childCount ?? 0) > 0, callAccessor = (node, name) => {
   const accessor = node[name];
   return typeof accessor === "function" ? accessor.call(node) : undefined;
 }, STATEMENT_ACCESSORS, NOTEBOOK_PARSE_URL = "file:///publisher-notebook-reader/notebook.malloy", NOTE_TOKENS, TEXT_BLOCK_NAME;
@@ -16563,6 +16609,7 @@ var init_notebook = __esm(() => {
     }
   };
   ANY_ARTIFACT_NOTE = /^#{1,2}\|?(?=[ \t\r\n])(?:[^"'\\]|\\.|"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')*?(?<![\w\u00C0-\u024F\u1E00-\u1EFF.$-])artifact(?![\w\u00C0-\u024F\u1E00-\u1EFF])/;
+  cellOffsets = new WeakMap;
   STATEMENT_ACCESSORS = [
     ["runStatement", "run"],
     ["docAnnotations", "notes"],
@@ -23267,6 +23314,7 @@ function malloyGivenToApi(given) {
     type: renderedType,
     annotations: allNotes.filter((note) => !isReservedRoute(note.route) && note.route !== MARKDOWN_ROUTE).map((note) => note.text),
     ...readGivenControlSpec(allNotes.map((note) => note.text)),
+    ...allNotes.some((note) => note.route === "secure") && { secure: true },
     default: given._internal?.defaultText
   };
 }
@@ -23443,6 +23491,13 @@ function readArtifactTag(tag, declaredType) {
     autorun
   };
 }
+function readDocumentTileExpressions(notes) {
+  const tag = motlyTag(notes);
+  const composite = tag ? readArtifactTag(tag, () => {
+    return;
+  }) : undefined;
+  return (composite?.tiles ?? []).flatMap((entry) => entry.kind === "text" ? [] : [entry.expression]);
+}
 function readTextBlocks(notes) {
   return notes.flatMap((note) => {
     const newline = note.indexOf(`
@@ -23626,6 +23681,9 @@ function givenSpec(declaration, facts) {
     type: declaration.type,
     default: declaration.type.startsWith("filter<") && declaration.default !== undefined ? unwrapFilterLiteral(declaration.default) : declaration.default,
     annotations: declaration.annotations.filter((text) => /^##?\(/.test(text) && !isMarkdownNote(text)),
+    ...declaration.annotations.some((text) => /^#\(secure\)/.test(text)) && {
+      secure: true
+    },
     ...control
   };
 }
@@ -23634,7 +23692,7 @@ function factsCarryArtifactTag(facts) {
     return true;
   return facts.queries.some((query) => Boolean(motlyTag(query.annotations)?.tag("artifact")));
 }
-function buildDashboardManifest(facts) {
+function buildDashboardManifest(facts, options) {
   const name = dashboardSlug(facts.modelPath);
   const base = {
     name,
@@ -23657,6 +23715,9 @@ function buildDashboardManifest(facts) {
         };
       }
       const query = entry.expression;
+      if (options?.restrictedTiles?.has(normalizeTileExpression(query))) {
+        return { kind: "query", query, restricted: true };
+      }
       const givenNames = resolveTileGivens(query, facts);
       const layout = resolveTileLayout(query, facts);
       return {
@@ -23666,7 +23727,7 @@ function buildDashboardManifest(facts) {
         ...layout
       };
     });
-    const queries = queryTiles(tiles);
+    const queries = queryTiles(tiles).filter((tile) => !tile.restricted);
     const doc = docCommentTitleAndDescription(facts.descriptionNotes ?? facts.modelAnnotations, composite.title);
     return {
       ...base,
@@ -23959,9 +24020,40 @@ function lintSelfDrills(facts) {
   }
   return Array.from(findings.values());
 }
+async function compileTileGivens(tiles, materializer, fallbackRegistry, gatesOf, skip, onPrepared, onCompileError) {
+  const compiled = new Map;
+  for (const tile of tiles) {
+    const key = normalizeTileExpression(tile);
+    if (compiled.has(key) || skip?.has(key))
+      continue;
+    let prepared;
+    try {
+      prepared = await materializer.loadQuery(`run: ${tile}`).getPreparedQuery();
+    } catch (error) {
+      onCompileError?.(tile, error);
+      continue;
+    }
+    await onPrepared?.(tile, prepared);
+    const usage = prepared._query?.givenUsage;
+    if (!usage)
+      continue;
+    const registry = prepared._modelDef?.givens ?? fallbackRegistry ?? {};
+    const reads = usage.map((given) => registry[given.id]?.name).filter((name) => name !== undefined);
+    const target = prepared._query?.structRef;
+    const sourceName = typeof target === "string" ? target : target?.as || target?.name;
+    const gateReads = new Set;
+    for (const expr of sourceName ? gatesOf(sourceName) ?? [] : []) {
+      for (const name of referencedGivenNames(expr))
+        gateReads.add(name);
+    }
+    compiled.set(key, { reads, gateReads: Array.from(gateReads) });
+  }
+  return compiled;
+}
 var MAX_BUILDER_COLUMNS = 24, COMPONENT_FILE_SUFFIXES, MODEL_FILE_SUFFIX2 = ".malloy", PUBLISHER_OWNED_TAGS, RENDERER_CHILD_TAGS, UNKNOWN_RENDER_TAG, COMPOSITE_ARTIFACT_PROPERTIES, QUERY_ARTIFACT_PROPERTIES, ARTIFACT_PROPERTY_REPLACEMENTS, NAME_THE_TILES = `Name the views to show: tiles=["source -> view", …].`;
 var init_dashboard = __esm(() => {
   init_annotations();
+  init_authorize();
   init_notebook();
   init_given();
   init_motly();
@@ -24414,11 +24506,11 @@ var init_source_extraction = __esm(() => {
 });
 
 // src/utils.ts
-import * as fs from "fs";
-import * as path from "path";
+import * as fs2 from "fs";
+import * as path2 from "path";
 import { fileURLToPath } from "url";
 function ignoreDotfiles(file) {
-  return path.basename(file).startsWith(".");
+  return path2.basename(file).startsWith(".");
 }
 function errMessage(err) {
   return err instanceof Error ? err.message : String(err);
@@ -24427,11 +24519,11 @@ var URL_READER;
 var init_utils = __esm(() => {
   URL_READER = {
     readURL: (url) => {
-      let path2 = url.toString();
+      let path3 = url.toString();
       if (url.protocol == "file:") {
-        path2 = fileURLToPath(url);
+        path3 = fileURLToPath(url);
       }
-      return fs.promises.readFile(path2, "utf8");
+      return fs2.promises.readFile(path3, "utf8");
     }
   };
 });
@@ -24457,8 +24549,8 @@ import {
   MalloySQLParser,
   MalloySQLStatementType
 } from "@malloydata/malloy-sql";
-import * as fs2 from "fs";
-import * as path2 from "path";
+import * as fs3 from "fs";
+import * as path3 from "path";
 import { parentPort, threadId } from "node:worker_threads";
 import { fileURLToPath as fileURLToPath2, pathToFileURL } from "url";
 
@@ -24645,6 +24737,254 @@ var EXPLORES_EMPTY_SUPPRESSES_CONVENTION = `"explores" in publisher.json is depr
 var EXPLORES_EMPTY_IS_UNCURATED = `"explores": [] in publisher.json does nothing. Fix: delete it.`;
 var QUERYABLE_SOURCES_DEPRECATION = `"queryableSources" in publisher.json does nothing. Fix: delete it.`;
 
+// src/service/package_retrieval.ts
+init_errors();
+import { createHash } from "crypto";
+import * as fs from "fs";
+import * as path from "path";
+var PACKAGE_REPRESENTATIONS = ["single", "facets"];
+var KEYPHRASE_MODES = ["auto", "never", "always"];
+var REFINE_LEVEL_NAMES = ["LOW", "MEDIUM", "HIGH"];
+var DEFAULT_RERANK_TOP_SOURCES = 8;
+var DEFAULT_REFINE_MIN_LEVEL = "MEDIUM";
+var RETRIEVAL_KEYS = [
+  "representation",
+  "keyphrases",
+  "refine",
+  "rerank",
+  "sourceMatch",
+  "sourceSummary",
+  "prompts"
+];
+var PROMPT_KEYS = [
+  "keyphrase",
+  "refine",
+  "rerank",
+  "sourceMatch",
+  "sourceSummary"
+];
+var REFINE_KEYS = ["enabled", "minLevel"];
+var RERANK_KEYS = ["enabled", "topSources"];
+var SOURCE_MATCH_KEYS = ["enabled"];
+var SOURCE_SUMMARY_KEYS = ["enabled"];
+var DEFAULT_PACKAGE_RETRIEVAL = {
+  representation: "single",
+  keyphrases: "auto",
+  prompts: {}
+};
+function refineSettingsOf(retrieval) {
+  return retrieval.refine ?? {
+    enabled: "auto",
+    minLevel: DEFAULT_REFINE_MIN_LEVEL
+  };
+}
+function rerankSettingsOf(retrieval) {
+  return retrieval.rerank ?? {
+    enabled: "auto",
+    topSources: DEFAULT_RERANK_TOP_SOURCES
+  };
+}
+function sourceMatchSettingsOf(retrieval) {
+  return retrieval.sourceMatch ?? { enabled: "auto" };
+}
+function sourceSummarySettingsOf(retrieval) {
+  return retrieval.sourceSummary ?? { enabled: "auto" };
+}
+function fail(message) {
+  throw new PackageManifestError(message);
+}
+function describe(value) {
+  return JSON.stringify(value) ?? "nothing";
+}
+function stepsOutside(within) {
+  return within === ".." || within.startsWith(`..${path.sep}`) || path.isAbsolute(within);
+}
+function resolvePromptPath(packageRoot, rel, promptKey = "keyphrase") {
+  const key = `retrieval.prompts.${promptKey}`;
+  const fix = `Fix: use a path inside the package, e.g. "prompts/${promptKey}.md".`;
+  if (rel.includes("\x00") || path.isAbsolute(rel) || /^[A-Za-z]:[\\/]/.test(rel) || rel.startsWith("\\")) {
+    fail(`Invalid publisher.json ${key}: ${describe(rel)} is an absolute path. ${fix}`);
+  }
+  const root = path.resolve(packageRoot);
+  const resolved = path.resolve(root, rel);
+  const within = path.relative(root, resolved);
+  if (within === "" || stepsOutside(within)) {
+    fail(`Invalid publisher.json ${key}: ${describe(rel)} resolves outside the package directory. ${fix}`);
+  }
+  return resolved;
+}
+function objectBlock(raw, key, valid, fix) {
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    fail(`Invalid publisher.json ${key}: expected an object, got ${describe(raw)}. Fix: ${fix}`);
+  }
+  const obj = raw;
+  const unknown = Object.keys(obj).filter((k) => !valid.includes(k));
+  if (unknown.length > 0) {
+    fail(`Invalid publisher.json ${key}: unknown key ${unknown.map((k) => `'${k}'`).join(", ")}. ` + `Valid keys: ${valid.join(", ")}.`);
+  }
+  return obj;
+}
+function parseEnabled(value, key) {
+  if (value === undefined || value === null)
+    return "auto";
+  if (value === "auto" || value === true || value === false)
+    return value;
+  return fail(`Invalid publisher.json ${key}: expected "auto", true or false, got ${describe(value)}. ` + `Fix: "enabled": "auto" (on when the server has an LLM), true (require one) or false.`);
+}
+function parseRefine(raw) {
+  const obj = objectBlock(raw, "retrieval.refine", REFINE_KEYS, `"refine": { "enabled": "auto", "minLevel": "MEDIUM" }`);
+  const minLevel = obj.minLevel ?? DEFAULT_REFINE_MIN_LEVEL;
+  if (typeof minLevel !== "string" || !REFINE_LEVEL_NAMES.includes(minLevel)) {
+    fail(`Invalid publisher.json retrieval.refine.minLevel: expected one of ${REFINE_LEVEL_NAMES.join(", ")}, got ${describe(obj.minLevel)}. ` + `Fix: "minLevel": "MEDIUM" (drop candidates the model rates below it).`);
+  }
+  return {
+    enabled: parseEnabled(obj.enabled, "retrieval.refine.enabled"),
+    minLevel
+  };
+}
+function parseRerank(raw) {
+  const obj = objectBlock(raw, "retrieval.rerank", RERANK_KEYS, `"rerank": { "enabled": "auto", "topSources": ${DEFAULT_RERANK_TOP_SOURCES} }`);
+  const top = obj.topSources ?? DEFAULT_RERANK_TOP_SOURCES;
+  if (typeof top !== "number" || !Number.isSafeInteger(top) || top <= 0) {
+    fail(`Invalid publisher.json retrieval.rerank.topSources: expected a positive integer, got ${describe(obj.topSources)}. ` + `Fix: "topSources": ${DEFAULT_RERANK_TOP_SOURCES}`);
+  }
+  return {
+    enabled: parseEnabled(obj.enabled, "retrieval.rerank.enabled"),
+    topSources: top
+  };
+}
+function parseSourceMatch(raw) {
+  const obj = objectBlock(raw, "retrieval.sourceMatch", SOURCE_MATCH_KEYS, `"sourceMatch": { "enabled": "auto" }`);
+  return {
+    enabled: parseEnabled(obj.enabled, "retrieval.sourceMatch.enabled")
+  };
+}
+function parseSourceSummary(raw) {
+  const obj = objectBlock(raw, "retrieval.sourceSummary", SOURCE_SUMMARY_KEYS, `"sourceSummary": { "enabled": "auto" }`);
+  return {
+    enabled: parseEnabled(obj.enabled, "retrieval.sourceSummary.enabled")
+  };
+}
+function parsePackageRetrieval(raw) {
+  if (raw === undefined || raw === null) {
+    return {
+      representation: DEFAULT_PACKAGE_RETRIEVAL.representation,
+      keyphrases: DEFAULT_PACKAGE_RETRIEVAL.keyphrases,
+      promptPaths: {}
+    };
+  }
+  if (typeof raw !== "object" || Array.isArray(raw)) {
+    fail(`Invalid publisher.json retrieval: expected an object, got ${describe(raw)}. ` + `Fix: "retrieval": { "representation": "single" }`);
+  }
+  const obj = raw;
+  const unknown = Object.keys(obj).filter((k) => !RETRIEVAL_KEYS.includes(k));
+  if (unknown.length > 0) {
+    fail(`Invalid publisher.json retrieval: unknown key ${unknown.map((k) => `'${k}'`).join(", ")}. ` + `Valid keys: ${RETRIEVAL_KEYS.join(", ")}.`);
+  }
+  const representation = obj.representation ?? "single";
+  if (typeof representation !== "string" || !PACKAGE_REPRESENTATIONS.includes(representation)) {
+    fail(`Invalid publisher.json retrieval.representation: expected one of ${PACKAGE_REPRESENTATIONS.join(", ")}, got ${describe(obj.representation)}. ` + `Fix: "representation": "single" (one row per entity) or "facets" (a name row plus doc chunks).`);
+  }
+  const keyphrases = obj.keyphrases ?? "auto";
+  if (typeof keyphrases !== "string" || !KEYPHRASE_MODES.includes(keyphrases)) {
+    fail(`Invalid publisher.json retrieval.keyphrases: expected one of ${KEYPHRASE_MODES.join(", ")}, got ${describe(obj.keyphrases)}. ` + `Fix: "keyphrases": "auto".`);
+  }
+  const refine = obj.refine === undefined || obj.refine === null ? undefined : parseRefine(obj.refine);
+  const rerank = obj.rerank === undefined || obj.rerank === null ? undefined : parseRerank(obj.rerank);
+  const sourceMatch = obj.sourceMatch === undefined || obj.sourceMatch === null ? undefined : parseSourceMatch(obj.sourceMatch);
+  const sourceSummary = obj.sourceSummary === undefined || obj.sourceSummary === null ? undefined : parseSourceSummary(obj.sourceSummary);
+  const promptPaths = {};
+  if (obj.prompts !== undefined && obj.prompts !== null) {
+    if (typeof obj.prompts !== "object" || Array.isArray(obj.prompts)) {
+      fail(`Invalid publisher.json retrieval.prompts: expected an object, got ${describe(obj.prompts)}. ` + `Fix: "prompts": { "keyphrase": "prompts/keyphrase.md" }`);
+    }
+    const prompts = obj.prompts;
+    const badPrompt = Object.keys(prompts).filter((k) => !PROMPT_KEYS.includes(k));
+    if (badPrompt.length > 0) {
+      fail(`Invalid publisher.json retrieval.prompts: unknown key ${badPrompt.map((k) => `'${k}'`).join(", ")}. ` + `Valid keys: ${PROMPT_KEYS.join(", ")}.`);
+    }
+    for (const key of PROMPT_KEYS) {
+      const value = prompts[key];
+      if (value === undefined || value === null)
+        continue;
+      if (typeof value !== "string" || value.trim() === "") {
+        fail(`Invalid publisher.json retrieval.prompts.${key}: expected a file path inside the package, got ${describe(value)}. ` + `Fix: "${key}": "prompts/${key}.md"`);
+      }
+      promptPaths[key] = value;
+    }
+  }
+  return {
+    representation,
+    keyphrases,
+    ...refine ? { refine } : {},
+    ...rerank ? { rerank } : {},
+    ...sourceMatch ? { sourceMatch } : {},
+    ...sourceSummary ? { sourceSummary } : {},
+    promptPaths
+  };
+}
+async function readPromptFile(packageRoot, key, rel) {
+  const file = resolvePromptPath(packageRoot, rel, key);
+  const at = `retrieval.prompts.${key}`;
+  let text;
+  try {
+    const [realRoot, realFile] = await Promise.all([
+      fs.promises.realpath(packageRoot),
+      fs.promises.realpath(file)
+    ]);
+    const within = path.relative(realRoot, realFile);
+    if (stepsOutside(within)) {
+      fail(`Invalid publisher.json ${at}: ${describe(rel)} resolves outside the package directory through a link. ` + `Fix: copy the file into the package.`);
+    }
+    text = await fs.promises.readFile(realFile, "utf8");
+  } catch (error) {
+    if (error instanceof PackageManifestError)
+      throw error;
+    return fail(`Invalid publisher.json ${at}: cannot read ${describe(rel)} (${error.message}). ` + `Fix: create the file inside the package or remove the key.`);
+  }
+  if (text.trim() === "") {
+    fail(`Invalid publisher.json ${at}: ${describe(rel)} is empty. ` + `Fix: write the prompt, or remove the key to use the built-in one.`);
+  }
+  return {
+    path: rel,
+    text,
+    hash: createHash("sha256").update(text).digest("hex")
+  };
+}
+async function readPackageRetrieval(packageRoot, raw) {
+  const parsed = parsePackageRetrieval(raw);
+  const prompts = {};
+  for (const key of PROMPT_KEYS) {
+    const rel = parsed.promptPaths[key];
+    if (rel !== undefined) {
+      prompts[key] = await readPromptFile(packageRoot, key, rel);
+    }
+  }
+  return {
+    representation: parsed.representation,
+    keyphrases: parsed.keyphrases,
+    ...parsed.refine ? { refine: parsed.refine } : {},
+    ...parsed.rerank ? { rerank: parsed.rerank } : {},
+    ...parsed.sourceMatch ? { sourceMatch: parsed.sourceMatch } : {},
+    ...parsed.sourceSummary ? { sourceSummary: parsed.sourceSummary } : {},
+    prompts
+  };
+}
+function assertRequiredStagesAvailable(retrieval, llmConfigured) {
+  if (llmConfigured)
+    return;
+  for (const [key, stage] of [
+    ["refine", retrieval.refine],
+    ["rerank", retrieval.rerank],
+    ["sourceMatch", retrieval.sourceMatch],
+    ["sourceSummary", retrieval.sourceSummary]
+  ]) {
+    if (stage?.enabled === true) {
+      fail(`Invalid publisher.json retrieval.${key}.enabled: true needs an LLM, and this server has none configured. ` + `Fix: set retrieval.llm in publisher.config.json and LLM_API_KEY in the server's environment, ` + `or set "enabled": "auto" (on only when an LLM is configured) or false.`);
+    }
+  }
+}
+
 // src/package_load/package_load_worker.ts
 init_source_extraction();
 init_given();
@@ -24704,9 +25044,9 @@ function newRpcId() {
 }
 function callMain(send) {
   const requestId = newRpcId();
-  return new Promise((resolve2, reject2) => {
+  return new Promise((resolve3, reject2) => {
     pendingRpc.set(requestId, {
-      resolve: (value) => resolve2(value),
+      resolve: (value) => resolve3(value),
       reject: reject2
     });
     send(requestId);
@@ -24826,10 +25166,10 @@ function makeWorkerUrlReader(job) {
     readURL: async (url) => {
       if (url.protocol === "file:") {
         const filePath = fileURLToPath2(url);
-        if (job.replacement && path2.resolve(filePath) === path2.resolve(job.packagePath, job.replacement.modelPath)) {
+        if (job.replacement && path3.resolve(filePath) === path3.resolve(job.packagePath, job.replacement.modelPath)) {
           return job.replacement.source;
         }
-        return fs2.promises.readFile(filePath, "utf8");
+        return fs3.promises.readFile(filePath, "utf8");
       }
       const response = await callMain((requestId) => {
         const req = {
@@ -24879,8 +25219,8 @@ function buildWorkerMalloyConfig(job) {
   return config;
 }
 async function readPackageMetadata(packagePath, modelPaths) {
-  const manifestPath = path2.join(packagePath, PACKAGE_MANIFEST_NAME);
-  const contents = await fs2.promises.readFile(manifestPath, "utf8");
+  const manifestPath = path3.join(packagePath, PACKAGE_MANIFEST_NAME);
+  const contents = await fs3.promises.readFile(manifestPath, "utf8");
   let parsed;
   try {
     parsed = JSON.parse(contents);
@@ -24911,12 +25251,13 @@ async function readPackageMetadata(packagePath, modelPaths) {
     manifestLocation: typeof parsed.manifestLocation === "string" ? parsed.manifestLocation : null,
     materialization: materializationWithQueryMetadata(parsePackageMaterialization(parsed.materialization), queryMetadata.queryMetadata),
     scope: scope.scope,
-    manifestWarnings: manifestWarnings.length > 0 ? manifestWarnings : undefined
+    manifestWarnings: manifestWarnings.length > 0 ? manifestWarnings : undefined,
+    retrieval: await readPackageRetrieval(packagePath, parsed.retrieval)
   };
 }
 async function listPackageFiles(packagePath) {
   const files = await import_recursive_readdir.default(packagePath, [ignoreDotfiles]);
-  return files.map((full) => path2.relative(packagePath, full).replace(/\\/g, "/")).sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
+  return files.map((full) => path3.relative(packagePath, full).replace(/\\/g, "/")).sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
 }
 function filterModelPaths(allRelative) {
   return allRelative.filter((p) => p.endsWith(MODEL_FILE_SUFFIX) || p.endsWith(NOTEBOOK_FILE_SUFFIX));
@@ -24985,7 +25326,7 @@ function buildRuntimeForModel(job, malloyConfig) {
 }
 async function compileMalloyModel(job, malloyConfig, modelPath) {
   const compileStart = performance.now();
-  const fullPath = path2.join(job.packagePath, modelPath);
+  const fullPath = path3.join(job.packagePath, modelPath);
   const modelURL = pathToFileURL(fullPath);
   const importBaseURL = new URL(".", modelURL);
   const { runtime, urlReader, textFor } = buildRuntimeForModel(job, malloyConfig);
@@ -25050,12 +25391,12 @@ async function compileMalloyModel(job, malloyConfig, modelPath) {
 }
 async function compileNotebookModel(job, malloyConfig, modelPath) {
   const compileStart = performance.now();
-  const fullPath = path2.join(job.packagePath, modelPath);
+  const fullPath = path3.join(job.packagePath, modelPath);
   const modelURL = pathToFileURL(fullPath);
   const importBaseURL = new URL(".", modelURL);
   const { runtime, urlReader } = buildRuntimeForModel(job, malloyConfig);
   const authorizeWarningCollection = authorizeWarningCollector();
-  const fileContents = await fs2.promises.readFile(modelURL, "utf8");
+  const fileContents = await fs3.promises.readFile(modelURL, "utf8");
   const parse = MalloySQLParser.parse(fileContents, modelPath);
   let mm;
   const perCellMM = parse.statements.map((stmt) => {