@hevmind/ask 0.3.0 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hevmind/ask",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "type": "module",
   "description": "hev ask: a heading-anchored, agentic search overlay for Astro docs sites.",
   "keywords": [
@@ -28,11 +28,11 @@
     "ask": "./bin/ask.mjs"
   },
   "optionalDependencies": {
-    "@hevmind/ask-win32-x64": "0.3.0",
-    "@hevmind/ask-linux-arm64": "0.3.0",
-    "@hevmind/ask-darwin-arm64": "0.3.0",
-    "@hevmind/ask-linux-x64": "0.3.0",
-    "@hevmind/ask-darwin-x64": "0.3.0"
+    "@hevmind/ask-darwin-arm64": "0.3.2",
+    "@hevmind/ask-linux-x64": "0.3.2",
+    "@hevmind/ask-linux-arm64": "0.3.2",
+    "@hevmind/ask-darwin-x64": "0.3.2",
+    "@hevmind/ask-win32-x64": "0.3.2"
   },
   "exports": {
     ".": "./src/index.ts",

package/src/search/loop.ts

@@ -363,13 +363,13 @@ function routedDigestSystemPrompt(digest: Digest): AnthropicTextBlock[] {
       type: 'text',
       text: `You are the documentation assistant for this site. Answer the user's question using ONLY documentation sections you retrieve.
 
-The documentation is large, so it is not all shown here. Use search_sections to find sections relevant to the question, then read the ones you need with open_section for their summary and exact facts. Run a few searches with varied terms if the first does not surface what you need. Open every section your answer draws on — you may only link to sections you opened.
+The documentation is large, so it is not all shown here. Use search_sections to find relevant sections — each result includes a short summary you can answer from directly. When you need a section's exact facts (flags, commands, identifiers), open_section it. One or two focused searches is plenty: once the results cover the question, STOP searching and answer. Do not keep searching for a perfect match.
 
 Write a short, direct answer in Markdown:
 - Start IMMEDIATELY with the substance. Your first sentence must answer the question. Never open with "Based on…", "Here is…", "Sure", a restatement of the question, or any summary/preamble.
 - Keep it tight: one or two short paragraphs, plus a short bullet list only if it genuinely helps. This renders in a small search popover, so do NOT use headings (#, ##) or horizontal rules (---).
-- For exact strings (flags, commands, identifiers, versions), quote the section's \`facts\` verbatim — never reword them.
-- When you reference a section, link to it inline using its exact \`url\`, e.g. [autoscaling](/docs/concepts#kubernetes-autoscaling). Never invent a URL or anchor.
+- For exact strings (flags, commands, identifiers, versions), quote a section's \`facts\` verbatim — never reword them.
+- When you reference a section, link to it inline using its exact \`url\` from your search results or open_section, e.g. [autoscaling](/docs/concepts#kubernetes-autoscaling). Never invent a URL or anchor.
 - If the documentation does not cover the question, say so plainly in one sentence and do not fabricate an answer.`,
     },
     {
@@ -415,6 +415,7 @@ async function* routedDigestAnswerLoop({
   const byId = new Map(chunks.map((chunk) => [chunk.id, chunk]));
   const nodesById = new Map(digest.nodes.map((node) => [node.id, node]));
   const opened = new Map<string, DigestNode>();
+  const seen = new Map<string, DigestNode>(); // sections surfaced by search, in rank order
   const messages: AnthropicMessage[] = [{ role: 'user', content: `Query: ${query}` }];
   const system = routedDigestSystemPrompt(digest);
 
@@ -450,10 +451,15 @@ async function* routedDigestAnswerLoop({
       if (block.name === 'search_sections') {
         const searchQuery = normalizeToolQuery(block.input) || query;
         yield { type: 'search', query: searchQuery };
+        const results = searchSections(searchQuery, chunks, nodesById, digest, config);
+        for (const result of results) {
+          const node = nodesById.get(result.id);
+          if (node && !seen.has(node.id)) seen.set(node.id, node);
+        }
         toolResults.push({
           type: 'tool_result',
           tool_use_id: block.id,
-          content: JSON.stringify(searchSections(searchQuery, chunks, nodesById, digest, config)),
+          content: JSON.stringify(results),
         });
       } else if (block.name === 'open_section') {
         const id = normalizeId(block.input);
@@ -472,38 +478,39 @@ async function* routedDigestAnswerLoop({
     messages.push({ role: 'user', content: toolResults });
   }
 
-  // Fallback: ground the answer even if the model opened nothing, by opening the
-  // best keyword matches for the original query.
-  if (!opened.size) {
+  // Fallback: if the model never searched or opened anything, ground on the best
+  // keyword matches for the original query so the answer isn't empty.
+  if (!opened.size && !seen.size) {
     for (const candidate of prefilter(chunks, query, digest.glossary, config.maxResults, config.perDocCap, digest.nodes)) {
-      const node = open(candidate.id);
-      if (node) yield { type: 'search', query: node.heading ?? node.title };
-    }
-    if (opened.size && lastRole(messages) !== 'user') {
-      const sections = [...opened.values()].map((node) => openSectionResult(node, byId));
-      messages.push({ role: 'user', content: `Opened sections:\n${JSON.stringify(sections)}` });
+      const node = nodesById.get(candidate.id);
+      if (node && !seen.has(node.id)) seen.set(node.id, node);
     }
   }
 
-  if (lastRole(messages) === 'assistant') {
-    messages.push({
-      role: 'user',
-      content:
-        'Write the answer now. Begin directly with the answer itself — no preamble, no "based on…" opener, no headings. Link only to sections you opened, using their exact url.',
-    });
-  }
-
-  const sources = sourcesFromNodes(opened, config.maxResults);
+  // Ground the answer in everything surfaced: opened sections (full facts) first,
+  // then searched summaries, capped to maxResults.
+  const grounded = [...new Map<string, DigestNode>([...opened, ...seen]).values()].slice(0, config.maxResults);
+  const sources = sourcesFromNodes(new Map(grounded.map((node) => [node.id, node])), config.maxResults);
   yield { type: 'sources', sources };
 
-  // Phase 2: streamed answer turn — no tools, so the model can only answer.
+  // Phase 2: a clean answer turn. Replaying the tool transcript keeps the model in
+  // "let me search more" mode (and it tries to call tools that no longer exist), so
+  // instead hand it just the question and the gathered sections — now it can only
+  // write the final prose answer.
+  const answerContext = grounded.map((node) => openSectionResult(node, byId));
+  const answerMessages: AnthropicMessage[] = [
+    {
+      role: 'user',
+      content: `Question: ${query}\n\nAnswer using only these documentation sections:\n${JSON.stringify(answerContext)}`,
+    },
+  ];
   for await (const event of tracedStream(
     stream,
     {
       apiKey,
       model: config.model,
-      system: answerSystem(system, sources),
-      messages,
+      system: routedAnswerSystemPrompt(digest),
+      messages: answerMessages,
       maxTokens: config.answerMaxTokens,
       signal,
     },
@@ -515,6 +522,27 @@ async function* routedDigestAnswerLoop({
   yield { type: 'done' };
 }
 
+/** Answer-only system prompt for the routed loop's final turn (no tools). */
+function routedAnswerSystemPrompt(digest: Digest): AnthropicTextBlock[] {
+  return [
+    {
+      type: 'text',
+      text: `You are the documentation assistant for this site. Write the answer to the user's question using ONLY the documentation sections provided in the next message. You have no tools — produce the final prose answer now.
+
+- Start IMMEDIATELY with the substance. Your first sentence must answer the question. Never open with "Based on…", "Here is…", "Sure", "Let me…", or any preamble or statement about searching, opening, or checking further.
+- Keep it tight: one or two short paragraphs, plus a short bullet list only if it genuinely helps. This renders in a small search popover, so do NOT use headings (#, ##) or horizontal rules (---).
+- For exact strings (flags, commands, identifiers, versions), quote a section's \`facts\` verbatim — never reword them.
+- Link to sections inline using their exact \`url\` from the provided sections, e.g. [autoscaling](/docs/concepts#kubernetes-autoscaling). Never invent a URL or anchor.
+- If the provided sections do not answer the question, say so plainly in one sentence and do not fabricate an answer.`,
+    },
+    {
+      type: 'text',
+      text: `<domain_context>\n${digest.context || ''}\n</domain_context>`,
+      cache_control: { type: 'ephemeral' },
+    },
+  ];
+}
+
 function sourcesFromNodes(opened: Map<string, DigestNode>, maxResults: number): Source[] {
   const sources: Source[] = [];
   const urls = new Set<string>();

package/src/search/prefilter.ts

@@ -11,6 +11,16 @@ export interface Candidate {
   snippet: string;
 }
 
+/** Grammatical/generic words that carry no topical signal in a docs query. */
+const STOPWORDS = new Set<string>([
+  'the', 'a', 'an', 'and', 'or', 'but', 'of', 'to', 'in', 'on', 'at', 'by', 'for', 'with', 'from', 'into', 'as',
+  'is', 'are', 'be', 'was', 'were', 'been', 'being', 'do', 'does', 'did', 'how', 'what', 'when', 'where', 'which',
+  'who', 'whom', 'why', 'can', 'could', 'should', 'would', 'shall', 'may', 'might', 'my', 'your', 'yours', 'you',
+  'i', 'it', 'its', 'this', 'that', 'these', 'those', 'there', 'here', 'we', 'our', 'us', 'me', 'if', 'then',
+  'else', 'not', 'no', 'so', 'than', 'too', 'very', 'just', 'also', 'only', 'out', 'about', 'up', 'set', 'get',
+  'use', 'using', 'used', 'via', 'want', 'need', 'make', 'have', 'has', 'will',
+]);
+
 /**
  * Distinctive tokens the digest carries for a section: its `terms`, the
  * tokens of its distilled `summary`, and the tokens of its verbatim `facts`. A
@@ -44,22 +54,47 @@ export function prefilter(
   perDocCap: number,
   nodes?: DigestNode[],
 ): Candidate[] {
-  const terms = expandQueryTerms(query, glossary);
-  if (!terms.length) return [];
+  const expanded = expandQueryTerms(query, glossary);
+  if (!expanded.length) return [];
+  // Drop grammatical/generic stopwords so a few ubiquitous words ("how", "set
+  // up", "with") don't dominate ranking. Fall back to the raw terms if the query
+  // is nothing but stopwords.
+  const filtered = expanded.filter((term) => term.length > 1 && !STOPWORDS.has(term));
+  const terms = filtered.length ? filtered : expanded;
 
   const signal = nodeSignal(nodes);
+  // Inverse document frequency: down-weight terms common across the corpus
+  // (ubiquitous words like "authentication") so a rare, on-topic term ("oidc")
+  // dominates. Without it, plain overlap buries the specific section under pages
+  // that merely share several common words — which degrades as the corpus grows.
+  const df = new Map<string, number>();
+  let lengthSum = 0;
+  for (const chunk of chunks) {
+    lengthSum += chunk.tokens.size;
+    for (const token of chunk.tokens) df.set(token, (df.get(token) ?? 0) + 1);
+  }
+  const total = chunks.length;
+  const avgLen = total ? lengthSum / total : 1;
+  const weights = new Map(terms.map((term) => [term, Math.log(1 + total / (1 + (df.get(term) ?? 0)))]));
+  const b = 0.75; // BM25 length-normalization strength
+
   const scored = chunks
     .map((chunk) => {
       const boost = signal.get(chunk.id);
-      let score = 0;
+      let raw = 0;
       for (const term of terms) {
-        if (chunk.tokens.has(term)) score += 1;
-        if (boost?.has(term)) score += 1;
+        const weight = weights.get(term) ?? 0;
+        if (chunk.tokens.has(term)) raw += weight;
+        if (boost?.has(term)) raw += weight;
       }
-      return { chunk, score };
+      // Length-penalize (not -reward): a huge page (e.g. an autogenerated CLI flag
+      // dump that mentions nearly every term) is divided down, but short sections
+      // are not boosted — the floor at 1 avoids over-rewarding tiny reference stubs.
+      const norm = Math.max(1, 1 - b + (b * chunk.tokens.size) / (avgLen || 1));
+      return { chunk, score: raw / norm };
     })
     .filter((candidate) => candidate.score > 0)
-    .sort((a, b) => b.score - a.score || a.chunk.id.localeCompare(b.chunk.id));
+    .sort((a, b2) => b2.score - a.score || a.chunk.id.localeCompare(b2.chunk.id));
 
   const perDoc = new Map<string, number>();
   const capped = [];