blume 2.0.3 → 2.1.0

package/src/ai/agent-readability.ts

@@ -8,6 +8,7 @@ import { buildRssFeeds } from "../deploy/rss.ts";
 import { AI_CATALOG_PATH, hasAiCatalog } from "./ai-catalog.ts";
 import { hasApiCatalog } from "./api-catalog.ts";
 import { API_PAGES_PATH, API_SEARCH_PATH, OPENAPI_PATH } from "./api/paths.ts";
+import { servesSkillsIndex } from "./skills.ts";
 
 /** Token map for the machine-readable content-usage echo. */
 const USAGE_TOKENS: [keyof ContentSignalPolicy, string][] = [
@@ -145,7 +146,7 @@ const wellKnownArtifacts = (
   if (hasApiCatalog(config)) {
     artifacts.apiCatalog = abs("/.well-known/api-catalog");
   }
-  if (config.agents.skills) {
+  if (servesSkillsIndex(config)) {
     artifacts.agentSkills = abs("/.well-known/agent-skills/index.json");
   }
   return artifacts;

package/src/ai/agent-surface.ts

@@ -11,7 +11,8 @@ export interface EmittedAgentSurface {
   mcp: boolean;
   /**
    * A skills discovery index is served: generated from `agents.skills` (at
-   * least one valid skill), or shipped by the user in `public/`.
+   * least one valid skill) or the site skill (`agents.skillMd`), or shipped
+   * by the user in `public/`.
    */
   skills: boolean;
 }
@@ -33,12 +34,12 @@ export const servesMcp = (
   );
 
 /**
- * The config the discovery documents are built from: `agents.mcp` and
- * `agents.skills` switched off when the build didn't emit them, so llms.txt,
- * agent-readability.json, the catalogs, and the header rules only point at
- * what is there. A config flag says what was asked for; the MCP server is
- * skipped when a page owns its route, and skills publish nothing when their
- * directory is missing or holds no valid skill.
+ * The config the discovery documents are built from: `agents.mcp`,
+ * `agents.skills`, and `agents.skillMd` switched off when the build didn't
+ * emit them, so llms.txt, agent-readability.json, the catalogs, and the
+ * header rules only point at what is there. A config flag says what was asked
+ * for; the MCP server is skipped when a page owns its route, and skills
+ * publish nothing when their directory is missing or holds no valid skill.
  */
 export const advertisedConfig = (
   config: ResolvedConfig,
@@ -51,6 +52,7 @@ export const advertisedConfig = (
       ...config.agents.mcp,
       enabled: config.agents.mcp.enabled && emitted.mcp,
     },
+    skillMd: config.agents.skillMd && emitted.skills,
     skills: emitted.skills ? config.agents.skills : undefined,
   },
 });

package/src/ai/ai-catalog.ts

@@ -10,7 +10,7 @@ import { resolveReferences } from "../openapi/references.ts";
 import { API_CATALOG_PATH, hasApiCatalog } from "./api-catalog.ts";
 import { OPENAPI_PATH } from "./api/paths.ts";
 import { asciiSlugify } from "./mcp/discovery.ts";
-import { AGENT_SKILLS_DIR } from "./skills.ts";
+import { AGENT_SKILLS_DIR, servesSkillsIndex } from "./skills.ts";
 import type { SkillArtifact } from "./skills.ts";
 
 /**
@@ -86,7 +86,7 @@ export const hasAiCatalog = (config: ResolvedConfig): boolean =>
   (config.agents.mcp.enabled ||
     config.agents.api ||
     config.agents.llmsTxt.enabled ||
-    Boolean(config.agents.skills) ||
+    servesSkillsIndex(config) ||
     resolveReferences(config).length > 0);
 
 /**
@@ -118,7 +118,10 @@ const NAMES_DOCS = /\b(?:docs|documentation)$/iu;
  * reads "the Acme documentation" and "Acme Docs" reads "the Acme Docs", not
  * "the Acme Docs docs".
  */
-const docsPhrase = (title: string, noun: "docs" | "documentation"): string =>
+export const docsPhrase = (
+  title: string,
+  noun: "docs" | "documentation"
+): string =>
   NAMES_DOCS.test(title.trim()) ? `the ${title}` : `the ${title} ${noun}`;
 
 const entrySeeds = (

package/src/ai/ask-context.ts

@@ -228,6 +228,17 @@ const queryTerms = (query: string): string[] =>
 const BASE_INSTRUCTION =
   "You are a helpful documentation assistant for this project. Answer the user's question using ONLY the documentation excerpts below. Each excerpt is headed by its page as `## Page Title (/route)`. If the answer is not covered by the excerpts, say you don't know and suggest where in the docs to look — do not invent details. Always cite the pages you drew from, and write every citation as a Markdown link to that page using its route, e.g. [Page Title](/route).";
 
+/**
+ * Appended when the endpoint gives the model the docs tools (see
+ * `ask-tools.ts`): the excerpts are a head start, not the whole of the docs.
+ */
+const TOOL_INSTRUCTION =
+  "You can also search the documentation yourself. When the excerpts don't fully answer the question, call search_docs with a focused query to find other pages, and read_page to read a page in full (one an excerpt cuts off, say) before you answer. Treat tool results exactly like the excerpts: answer only from them, and cite the pages you drew from the same way.";
+
+/** Said instead of the excerpts when retrieval found none but tools can look. */
+const NO_EXCERPTS_WITH_TOOLS =
+  "No documentation excerpts matched this question up front. Use search_docs to look before you answer.";
+
 /** The non-empty user turns, oldest first. Assistant turns never seed retrieval. */
 const userTurns = (messages: AskMessage[]): string[] =>
   messages
@@ -591,6 +602,48 @@ export const sectionExcerpt = (
   max: number
 ): string => excerptPage(parsePage(content), query, max);
 
+// One Orama index per snapshot, shared by the grounding function and the
+// docs tools so a cold endpoint builds it once.
+const indexes = new WeakMap<AskData, ReturnType<typeof buildOramaIndex>>();
+
+/** The search index over an assistant snapshot, built on first use. */
+export const askIndex = (data: AskData): ReturnType<typeof buildOramaIndex> => {
+  let index = indexes.get(data);
+  if (!index) {
+    index = buildOramaIndex(data.documents, data.defaultLocale);
+    indexes.set(data, index);
+  }
+  return index;
+};
+
+/** The page the reader is on, and the search filters it implies. */
+export interface ReaderScope {
+  current: OramaDoc | undefined;
+  filters: { locale: string | undefined; version: string | undefined };
+}
+
+/**
+ * Where the reader is: their page, when it's in the snapshot, and the locale
+ * and docs version retrieval keeps to. Without a page, a versioned site
+ * grounds in the current docs rather than every archived copy of each page.
+ */
+export const readerScope = (
+  data: AskData,
+  byRoute: ReadonlyMap<string, OramaDoc>,
+  page?: AskPage
+): ReaderScope => {
+  const current = page?.path
+    ? byRoute.get(normalizeRoute(page.path))
+    : undefined;
+  return {
+    current,
+    filters: {
+      locale: current?.locale || undefined,
+      version: data.versioned ? (current?.version ?? "") : undefined,
+    },
+  };
+};
+
 /**
  * Build the request-time grounding function for the assistant endpoint.
  *
@@ -607,20 +660,22 @@ export const sectionExcerpt = (
  *
  * `options.retrieval` (the `ai.assistant.retrieval` config) sizes how much
  * documentation each question carries; omitted fields keep today's defaults.
+ *
+ * `options.tools` says the endpoint also hands the model the docs tools: the
+ * prompt then says so, and a question retrieval found nothing for still gets
+ * a grounded prompt, since the model can go and search.
  */
 export const createAskContext = (
   data: AskData,
-  options?: { instructions?: string; retrieval?: AskRetrievalOptions }
+  options?: {
+    instructions?: string;
+    retrieval?: AskRetrievalOptions;
+    tools?: boolean;
+  }
 ): ((
   messages: AskMessage[],
   page?: AskPage
 ) => Promise<string | undefined>) => {
-  let dbPromise: Promise<Awaited<ReturnType<typeof buildOramaIndex>>> | null =
-    null;
-  const index = () => {
-    dbPromise ??= buildOramaIndex(data.documents, data.defaultLocale);
-    return dbPromise;
-  };
   const byRoute = new Map(data.documents.map((doc) => [doc.route, doc]));
   // Section splitting and tokenizing are per page, not per question, so each
   // page is parsed on first use and reused for the life of the endpoint.
@@ -633,9 +688,12 @@ export const createAskContext = (
     }
     return page;
   };
-  const instruction = options?.instructions
-    ? `${BASE_INSTRUCTION}\n\n${options.instructions}`
+  const base = options?.tools
+    ? `${BASE_INSTRUCTION} ${TOOL_INSTRUCTION}`
     : BASE_INSTRUCTION;
+  const instruction = options?.instructions
+    ? `${base}\n\n${options.instructions}`
+    : base;
   const maxResults = options?.retrieval?.maxResults ?? MAX_RESULTS;
   const excerptChars = options?.retrieval?.excerptChars ?? EXCERPT_CHARS;
   const contextBudget = options?.retrieval?.contextBudget ?? CONTEXT_BUDGET;
@@ -652,16 +710,9 @@ export const createAskContext = (
     const [query = ""] = queries;
 
     // The current page anchors retrieval to its locale and docs version, and
-    // is injected first. Without one, a versioned site grounds in the
-    // current docs rather than every archived copy of each page.
-    const current = page?.path
-      ? byRoute.get(normalizeRoute(page.path))
-      : undefined;
-    const db = await index();
-    const filters = {
-      locale: current?.locale || undefined,
-      version: data.versioned ? (current?.version ?? "") : undefined,
-    };
+    // is injected first.
+    const { current, filters } = readerScope(data, byRoute, page);
+    const db = await askIndex(data);
     const hits = interleave(
       await Promise.all(
         queries.map((text) => queryOramaIndex(db, text, maxResults, filters))
@@ -700,7 +751,9 @@ export const createAskContext = (
     }
 
     if (sections.length === 0) {
-      return;
+      return options?.tools
+        ? `${instruction}\n\n${NO_EXCERPTS_WITH_TOOLS}`
+        : undefined;
     }
     return `${instruction}\n\n<docs>\n${sections.join("\n\n")}\n</docs>`;
   };

package/src/ai/ask-data.ts

@@ -36,6 +36,12 @@ export const buildAskData = async (project: BlumeProject): Promise<AskData> => {
       if (versioned) {
         document.version = doc.version;
       }
+      if (doc.boost !== undefined) {
+        document.boost = doc.boost;
+      }
+      if (doc.keywords) {
+        document.keywords = doc.keywords;
+      }
       return document;
     }),
     site: project.config.deployment.options.site ?? null,

package/src/ai/ask-tools.ts

@@ -0,0 +1,142 @@
+import { jsonSchema, tool } from "ai";
+
+import { normalizeRoute } from "../core/base-path.ts";
+import { queryOramaIndex } from "../search/orama-index.ts";
+import type { OramaDoc } from "../search/orama-index.ts";
+import { askIndex, readerScope, sectionExcerpt } from "./ask-context.ts";
+import type { AskData, AskPage, ReaderScope } from "./ask-context.ts";
+
+/**
+ * The assistant's docs tools: what lets it look beyond the excerpts the
+ * endpoint injects up front. `search_docs` runs the same Orama search the
+ * grounding does, scoped to the reader's language and docs version;
+ * `read_page` returns a whole page, for when an excerpt stops short of the
+ * answer. Both read the assistant snapshot the endpoint already imports, so
+ * they add no build output and no network call.
+ *
+ * The generated `/api/ask` route passes them to `streamText` with a step cap
+ * ({@link ASK_MAX_STEPS}), so the model can search, read, and then answer
+ * within one request.
+ */
+
+/**
+ * The most model steps one answer may take: up to four rounds of tool calls,
+ * then the answer. Enough to search, read a page or two, and search again,
+ * while bounding what one question can spend.
+ */
+export const ASK_MAX_STEPS = 5;
+
+/** Pages one search returns. */
+const SEARCH_RESULTS = 5;
+
+/** Characters of each search result's most relevant excerpt. */
+const SEARCH_EXCERPT_CHARS = 700;
+
+/** Characters of a page `read_page` returns; a longer page is cut here. */
+const PAGE_CHARS = 20_000;
+
+/** A result block, headed the way the injected excerpts are, for citing. */
+const block = (doc: OramaDoc, body: string): string =>
+  `## ${doc.title} (${doc.route})\n${body}`;
+
+/**
+ * Search the docs for `query` within the reader's scope: the most relevant
+ * excerpt of each of the top pages, or a note that nothing matched.
+ */
+export const searchDocs = async (
+  data: AskData,
+  scope: ReaderScope,
+  query: string
+): Promise<string> => {
+  const hits = await queryOramaIndex(
+    await askIndex(data),
+    query,
+    SEARCH_RESULTS,
+    scope.filters
+  );
+  if (hits.length === 0) {
+    return `No pages matched "${query}". Try other words, or a broader query.`;
+  }
+  return hits
+    .map((doc) =>
+      block(doc, sectionExcerpt(doc.content, query, SEARCH_EXCERPT_CHARS))
+    )
+    .join("\n\n");
+};
+
+/**
+ * The route a model asked for, as the snapshot keys it: a full URL on the
+ * site becomes its path, and a query string or fragment is dropped.
+ */
+const routeOf = (data: AskData, requested: string): string => {
+  let path = requested.trim();
+  if (data.site && path.startsWith(data.site)) {
+    path = path.slice(data.site.replace(/\/$/u, "").length) || "/";
+  }
+  path = path.replace(/[?#].*$/u, "");
+  return normalizeRoute(path.startsWith("/") ? path : `/${path}`);
+};
+
+/** A whole page by route, cut at {@link PAGE_CHARS}, or a note that it doesn't exist. */
+export const readPage = (
+  data: AskData,
+  byRoute: ReadonlyMap<string, OramaDoc>,
+  requested: string
+): string => {
+  const route = routeOf(data, requested);
+  const doc = byRoute.get(route);
+  if (!doc) {
+    return `There is no page at ${route}. Use search_docs to find the right one.`;
+  }
+  const text =
+    doc.content.length > PAGE_CHARS
+      ? `${doc.content.slice(0, PAGE_CHARS)}\n\n[The page continues; this is its first ${PAGE_CHARS.toLocaleString("en-US")} characters.]`
+      : doc.content;
+  return block(doc, text);
+};
+
+/**
+ * The tools for one request, given the page the reader is on. Built per
+ * request because the search scope follows the reader's locale and version.
+ */
+export const createAskTools = (data: AskData) => {
+  const byRoute = new Map(data.documents.map((doc) => [doc.route, doc]));
+  return (page?: AskPage) => {
+    const scope = readerScope(data, byRoute, page);
+    return {
+      read_page: tool({
+        description:
+          "Read one documentation page in full, by its route (the path in parentheses after a page title, such as /docs/getting-started).",
+        execute: ({ route }) => Promise.resolve(readPage(data, byRoute, route)),
+        inputSchema: jsonSchema<{ route: string }>({
+          additionalProperties: false,
+          properties: {
+            route: {
+              description: "The page's route, such as /docs/getting-started.",
+              type: "string",
+            },
+          },
+          required: ["route"],
+          type: "object",
+        }),
+      }),
+      search_docs: tool({
+        description:
+          "Search this project's documentation. Returns the most relevant excerpt of each matching page, headed by its title and route.",
+        execute: ({ query }) => searchDocs(data, scope, query),
+        inputSchema: jsonSchema<{ query: string }>({
+          additionalProperties: false,
+          properties: {
+            query: {
+              description:
+                "A few focused keywords, such as the feature or option the question is about.",
+              type: "string",
+            },
+          },
+          required: ["query"],
+          type: "object",
+        }),
+      }),
+    };
+  };
+};