@hevmind/ask 0.2.0 → 0.3.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hevmind/ask",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "type": "module",
   "description": "hev ask: a heading-anchored, agentic search overlay for Astro docs sites.",
   "keywords": [
@@ -28,11 +28,11 @@
     "ask": "./bin/ask.mjs"
   },
   "optionalDependencies": {
-    "@hevmind/ask-darwin-arm64": "0.2.0",
-    "@hevmind/ask-linux-x64": "0.2.0",
-    "@hevmind/ask-win32-x64": "0.2.0",
-    "@hevmind/ask-linux-arm64": "0.2.0",
-    "@hevmind/ask-darwin-x64": "0.2.0"
+    "@hevmind/ask-darwin-arm64": "0.3.1",
+    "@hevmind/ask-darwin-x64": "0.3.1",
+    "@hevmind/ask-linux-arm64": "0.3.1",
+    "@hevmind/ask-win32-x64": "0.3.1",
+    "@hevmind/ask-linux-x64": "0.3.1"
   },
   "exports": {
     ".": "./src/index.ts",

package/skills/build-digest/SKILL.md

@@ -1,120 +1,84 @@
 ---
 name: build-digest
 description: >-
-  Build the @hevmind/ask ask digest (.hev-ask/ markdown tree) for an Astro docs site
-  using your Claude Code subscription instead of an ANTHROPIC_API_KEY. Use when
-  the user asks to build, rebuild, or refresh the hev ask digest, knowledge
-  graph, KG, or search index, or after docs content changes. Runs `ask digest
-  corpus --shards-dir`, distils each shard in a fresh context, and runs
-  `ask digest assemble --input-dir`.
+  Build the @hevmind/ask ask digest (the committed .hev-ask/ markdown tree) for a
+  docs site using your own agent subscription instead of a provider API key. Use
+  when asked to build, rebuild, or refresh the hev ask digest, search index, or
+  knowledge graph (KG), or after docs content changes. Shards the corpus, distils
+  each shard in a fresh context, then assembles and verifies.
 ---
 
 # Build the hev ask digest
 
-`@hevmind/ask` searches an Astro docs site. Its agentic loop, keyword ranking, and
-suggested questions are powered by a committed ask digest tree at
-`.hev-ask/`. Only the **distillation** needs a model — the node
-structure, verbatim facts, overview map, and content hash are computed
-deterministically by the CLI. This skill performs that distillation here, in
-the user's subscription, so it costs **no API tokens on their own key**.
-
-The corpus is split into **shards** (~200KB of text each, along slug-prefix
-boundaries) and each shard is distilled in its own fresh context. Corpus size
-is therefore never a context-limit problem — a bigger site just means more
-shards. All state lives on disk in `.hev-ask/shards/`, so the build can be
-stopped, resumed, and incrementally refreshed: after content edits, only the
-shards whose content changed need re-distilling.
-
-Run every command from the **site root** (the directory whose `astro.config.*`
-registers `hevAsk()`). Prefer `pnpm exec ask digest …`; fall back to
-`npx -p @hevmind/ask ask digest …` if pnpm isn't used. Pass the same content flags
-the site's integration uses if they differ from the defaults (`--collection`,
-`--base-path`, `--chunk-heading-depth`, `--content-glob`, `--digest-dir`);
-they must match across `corpus` and `assemble`.
-
-**Never read a shard input file into the orchestrating context** (they hold
-the full corpus text). The orchestrator works only from command output,
-`manifest.json`, and `status`; shard contents are read by the per-shard
-distillation agents.
+`@hevmind/ask`'s agentic loop, keyword ranking, and suggested questions run off a
+committed digest tree at `.hev-ask/`. Only the **distillation** needs a model —
+the CLI computes the node structure, verbatim facts, overview map, and content
+hashes deterministically. This skill does that distillation here in your
+subscription, so it costs **no provider API tokens**.
 
-## Steps
+`ask` is the `@hevmind/ask` binary: install it on PATH, or it resolves via the
+package bin / `HEV_ASK_BINARY` (see `api/cli.mdx`). Run every command from the
+**site root** (the dir whose config registers `hevAsk()`). If the integration
+overrides any content flags (`--collection`, `--base-path`,
+`--chunk-heading-depth`, `--content-glob`, `--digest-dir`), pass the same ones to
+`corpus` and `assemble` — they must match.
 
-1. **Shard the corpus.**
+The corpus splits into ~200KB **shards** (`--shard-bytes` tunes it), each
+distilled in its own context, so corpus size is never a context limit. State
+lives in `.hev-ask/shards/`, so the build resumes and refreshes incrementally:
+after edits, only changed shards re-distil.
 
-   ```sh
-   pnpm exec ask digest corpus --shards-dir .hev-ask/shards
-   ```
+**Never read a shard input file yourself** — they hold the full corpus text.
+Work from command output and `status`; the per-shard agents read the shards.
 
-   Deterministic and keyless. Writes one `input-<shard-id>.json` per shard
-   plus `manifest.json`, and reports `(N sections, M shards, P pending,
-   up-to-date|needs-rebuild)`. Re-running after content edits is safe and is
-   the refresh mechanism: unchanged shards keep their distillations; changed
-   ones are marked pending again.
-
-2. **Check state.** If the corpus reported `up-to-date` AND `0 pending`, the
-   committed digest already matches the content — **stop here** and tell the
-   user nothing needs rebuilding. Otherwise:
+## Steps
 
-   ```sh
-   pnpm exec ask digest status --shards-dir .hev-ask/shards
-   ```
+1. **Shard.** `ask digest corpus --shards-dir .hev-ask/shards`
+   Deterministic and keyless. Reports `(N sections, M shards, P pending,
+   up-to-date|needs-rebuild)`. Safe to re-run; this is the refresh mechanism.
 
-   lists which shards are `pending` or `stale` (distilled against older
-   content). Both need distilling.
+2. **Check.** If corpus said `up-to-date` AND `0 pending`, the digest already
+   matches the content — **stop and tell the user nothing needs rebuilding.**
+   Otherwise `ask digest status --shards-dir .hev-ask/shards` lists the
+   `pending`/`stale` shards (both need distilling).
 
 3. **Distil each pending/stale shard in a fresh context.** Spawn one agent per
-   shard (run a few in parallel; don't read shard contents yourself). Each
-   agent gets this prompt, with `<id>` filled in:
+   shard (a few in parallel; don't read shards yourself). Give each this prompt
+   with `<id>` filled in:
 
-   > Read ONLY `.hev-ask/shards/input-<id>.json` (from the site root). It has
-   > `shardId`, `shardHash`, and a `sections` array of `{ id, url, title,
-   > text }`. Write `.hev-ask/shards/distill-<id>.json` with exactly this
-   > shape:
+   > Read ONLY `.hev-ask/shards/input-<id>.json` (from the site root): it has
+   > `shardId`, `shardHash`, and a `sections` array of `{ id, url, title, text }`.
+   > Write `.hev-ask/shards/distill-<id>.json` with exactly this shape:
    >
    > ```json
    > {
-   >   "shardHash": "<copy the shardHash from the input file verbatim>",
-   >   "notes": "5-10 lines: what this shard covers, its key concepts, and how users talk about them.",
-   >   "glossary": [
-   >     { "term": "ask digest", "aliases": ["digest", "kg"], "definition": "One-line definition." }
-   >   ],
-   >   "summaries": [
-   >     { "id": "<exact section id from sections>", "summary": "1-3 sentence distillation." }
-   >   ]
+   >   "shardHash": "<copy shardHash verbatim>",
+   >   "notes": "5-10 lines: what this shard covers, its key concepts, and how users phrase them.",
+   >   "glossary": [{ "term": "...", "aliases": ["..."], "definition": "One line." }],
+   >   "summaries": [{ "id": "<exact section id>", "summary": "1-3 sentences." }]
    > }
    > ```
    >
-   > Rules:
-   > - Emit **one `summaries` entry for every `id`** in `sections` — no more,
-   >   no fewer. Use the exact id strings.
-   > - Summaries are what the search agent reasons from: faithful,
-   >   self-contained, 1-3 sentences. **Paraphrase prose, but never restate
-   >   code, flags, commands, or exact identifiers** — those are extracted
-   >   verbatim by the CLI and would only drift if you retyped them.
-   > - `glossary`: at most the ~10 terms from this shard a real user would
-   >   type (aliases like `k8s` for `kubernetes`); one-line definitions. The
-   >   CLI dedupes and caps the merged glossary.
-   > - `notes` is NOT user-facing — it feeds a later site-wide synthesis pass.
-   > - Your final message: just the shard id and how many summaries you wrote.
-
-   If the run is interrupted, just re-run from step 1 — disk state is the
-   source of truth and `status` shows what's left.
-
-4. **Synthesize the global context.** Once every shard is distilled, extract
-   only the `notes` fields (small) — never the full distill files:
-
-   ```sh
-   python3 -c "import json,glob; [print('##', f.split('distill-')[1].removesuffix('.json'), '\n' + json.load(open(f)).get('notes','')) for f in sorted(glob.glob('.hev-ask/shards/distill-*.json'))]"
-   ```
-
-   From those notes, write `.hev-ask/shards/global.json`:
+   > - One `summaries` entry for every `id` in `sections` — exact ids, no more, no fewer.
+   > - Summaries are what the search agent reasons from: faithful, self-contained.
+   >   Paraphrase prose; **never restate code, flags, commands, or identifiers** —
+   >   the CLI extracts those verbatim, and they'd only drift if retyped.
+   > - `glossary`: ≤10 terms a real user would actually type (aliases like
+   >   `k8s`→`kubernetes`); one-line definitions. The CLI dedupes and caps them.
+   > - `notes` is not user-facing; it feeds the global synthesis pass.
+   > - Reply with just the shard id and how many summaries you wrote.
+
+   Interrupted? Re-run from step 1 — disk is the source of truth and `status`
+   shows what's left.
+
+4. **Synthesize the global context.** Once every shard is distilled, read the
+   `notes` field from each `.hev-ask/shards/distill-*.json` (small — never the
+   full files) and write `.hev-ask/shards/global.json`:
 
    ```json
    {
      "context": "Compact markdown orientation: what the product/site is, its core concepts and feature areas, and how users talk about them.",
-     "suggestions": [
-       "A natural question a reader might type that these docs answer."
-     ],
+     "suggestions": ["A natural question a reader might type that these docs answer."],
      "glossary": []
    }
    ```
@@ -122,28 +86,17 @@ distillation agents.
    `suggestions`: 3-5 questions phrased the way a user would ask them, each
    genuinely answerable from these docs (they show in the overlay on open).
 
-5. **Assemble.**
-
-   ```sh
-   pnpm exec ask digest assemble --input-dir .hev-ask/shards
-   ```
-
-   Merges every current shard distillation with the global synthesis, derives
-   the deterministic parts, and writes the `.hev-ask/` markdown tree. Sections from
-   undistilled shards fall back to plain excerpts and are reported — the
-   digest stays usable mid-wave, but aim for 0 pending before committing.
-
-6. **Verify.**
-
-   ```sh
-   pnpm exec ask digest verify
-   ```
+5. **Assemble.** `ask digest assemble --input-dir .hev-ask/shards`
+   Merges every shard distillation with the global synthesis, derives the
+   deterministic parts, and writes the `.hev-ask/` tree. Undistilled shards fall
+   back to excerpts and are reported — usable mid-wave, but aim for 0 pending
+   before committing.
 
-   Anchor drift is fatal; coverage/fidelity warnings are informational.
+6. **Verify.** `ask digest verify` — anchor drift is fatal; coverage/fidelity
+   warnings are informational.
 
-7. **Clean up and commit.** The shards directory is a local cache — it is the
-   resume/refresh state, so keep it on disk but out of git, and drop the bulky
-   input files (regenerated by `corpus` any time):
+7. **Commit.** The shards dir is the local resume/refresh cache — keep it on disk
+   but out of git, and drop the bulky input files (`corpus` regenerates them):
 
    ```sh
    rm -f .hev-ask/shards/input-*.json
@@ -151,14 +104,11 @@ distillation agents.
    git add .gitignore .hev-ask
    ```
 
-   Only the `.hev-ask/` tree is committed; `.hev-ask/shards/` remains local.
+   Only the `.hev-ask/` tree is committed; `.hev-ask/shards/` stays local.
 
 ## Notes
 
-- A small site may produce a single shard; the flow is the same (you can
-  distil it yourself instead of spawning an agent — a single shard's input is
-  small enough to read directly).
-- `--shard-bytes` (default 200000) tunes shard size if a site's sections are
-  unusually dense.
-- If `corpus` fails because no content is found, you're likely not in the
-  site root, or the collection name isn't `docs` — pass `--collection <name>`.
+- A small site may produce a single shard — distil it yourself instead of
+  spawning an agent (its input is small enough to read directly).
+- If `corpus` finds no content, you're likely not in the site root, or the
+  collection isn't named `docs` — pass `--collection <name>`.

package/src/search/loop.ts

@@ -112,13 +112,37 @@ async function* tracedStream(
 }
 
 /**
- * Entry point. When the committed digest carries distilled `nodes`, the
- * agent navigates that shadow digest (digest path). A node-less (v1 / degraded)
- * digest falls back to the original keyword-search loop, unchanged.
+ * Cap on the characters the digest path inlines into the system prompt (the
+ * `<map>` + `<summaries>` blocks). Below it, every section summary is inlined so
+ * the agent navigates from a complete map — best for small/medium sites. Above
+ * it (large docs, e.g. a CLI/API reference with thousands of sections), inlining
+ * everything would blow the context window, so the loop switches to search-routed
+ * navigation: a compact page map plus a search tool that surfaces ids on demand.
+ * ~200 KB ≈ ~50k tokens; a ~500-section site stays fully inlined as before.
+ */
+export const INLINE_DIGEST_BUDGET = 200_000;
+
+/** Cheap estimate of what `buildDigestSystemPrompt` would inline, without building it. */
+export function digestInlineSize(digest: Digest): number {
+  let size = digest.overview.length;
+  for (const node of digest.nodes) size += node.id.length + node.summary.length + 24;
+  return size;
+}
+
+/**
+ * Entry point. When the committed digest carries distilled `nodes`, the agent
+ * navigates that shadow digest: small digests are inlined whole (digest path);
+ * digests larger than {@link INLINE_DIGEST_BUDGET} are navigated by search so the
+ * prompt stays bounded (routed path). A node-less (v1 / degraded) digest falls
+ * back to the original keyword-search loop, unchanged.
  */
 export async function* runAgenticAnswerLoop(args: AnswerLoopArgs): AsyncGenerator<AgenticEvent> {
   if (args.digest.nodes && args.digest.nodes.length > 0) {
-    yield* digestAnswerLoop(args);
+    if (digestInlineSize(args.digest) <= INLINE_DIGEST_BUDGET) {
+      yield* digestAnswerLoop(args);
+    } else {
+      yield* routedDigestAnswerLoop(args);
+    }
   } else {
     yield* legacyAnswerLoop(args);
   }
@@ -300,6 +324,208 @@ function renderNodeMap(nodes: DigestNode[]): string {
   return nodes.map((node) => `- ${node.heading ?? node.title} — \`${node.id}\``).join('\n');
 }
 
+// ---------------------------------------------------------------------------
+// Routed path: navigate a large digest by search instead of inlining it whole.
+// ---------------------------------------------------------------------------
+
+const SEARCH_SECTIONS_TOOL: AnthropicTool = {
+  name: 'search_sections',
+  description:
+    'Search the documentation for sections relevant to a focused sub-query. Returns matching section ids with their group, heading, and a one-line summary. Use it to find the ids you then read with open_section.',
+  input_schema: {
+    type: 'object',
+    properties: {
+      query: { type: 'string', description: 'Focused keyword query or synonym expansion to search for.' },
+    },
+    required: ['query'],
+  },
+};
+
+/** Compact group → page map: orientation only, so the prompt stays bounded. */
+function routedDigestMap(nodes: DigestNode[]): string {
+  const byGroup = new Map<string, Set<string>>();
+  for (const node of nodes) {
+    const group = node.group ?? 'Docs';
+    if (!byGroup.has(group)) byGroup.set(group, new Set());
+    byGroup.get(group)!.add(node.title);
+  }
+  const lines: string[] = [];
+  for (const [group, pages] of byGroup) {
+    lines.push(`## ${group}`);
+    for (const page of pages) lines.push(`- ${page}`);
+  }
+  return lines.join('\n');
+}
+
+function routedDigestSystemPrompt(digest: Digest): AnthropicTextBlock[] {
+  return [
+    {
+      type: 'text',
+      text: `You are the documentation assistant for this site. Answer the user's question using ONLY documentation sections you retrieve.
+
+The documentation is large, so it is not all shown here. Use search_sections to find relevant sections — each result includes a short summary you can answer from directly. When you need a section's exact facts (flags, commands, identifiers), open_section it. One or two focused searches is plenty: once the results cover the question, STOP searching and answer. Do not keep searching for a perfect match.
+
+Write a short, direct answer in Markdown:
+- Start IMMEDIATELY with the substance. Your first sentence must answer the question. Never open with "Based on…", "Here is…", "Sure", a restatement of the question, or any summary/preamble.
+- Keep it tight: one or two short paragraphs, plus a short bullet list only if it genuinely helps. This renders in a small search popover, so do NOT use headings (#, ##) or horizontal rules (---).
+- For exact strings (flags, commands, identifiers, versions), quote a section's \`facts\` verbatim — never reword them.
+- When you reference a section, link to it inline using its exact \`url\` from your search results or open_section, e.g. [autoscaling](/docs/concepts#kubernetes-autoscaling). Never invent a URL or anchor.
+- If the documentation does not cover the question, say so plainly in one sentence and do not fabricate an answer.`,
+    },
+    {
+      type: 'text',
+      text: `<domain_context>\n${digest.context || 'No digest context is available.'}\n</domain_context>\n\n<map>\n${routedDigestMap(digest.nodes)}\n</map>`,
+      cache_control: { type: 'ephemeral' },
+    },
+  ];
+}
+
+/** Search the digest's nodes for a sub-query; returns distilled candidates. */
+function searchSections(
+  searchQuery: string,
+  chunks: Chunk[],
+  nodesById: Map<string, DigestNode>,
+  digest: Digest,
+  config: SearchLoopConfig,
+) {
+  return prefilter(chunks, searchQuery, digest.glossary, config.candidatePerSearch, config.perDocCap, digest.nodes)
+    .map((candidate) => nodesById.get(candidate.id))
+    .filter((node): node is DigestNode => node !== undefined)
+    .map((node) => ({
+      id: node.id,
+      url: node.url,
+      group: node.group,
+      heading: node.heading,
+      summary: node.summary,
+      ...(node.mode === 'source-primary' ? { reference: true } : {}),
+    }));
+}
+
+async function* routedDigestAnswerLoop({
+  apiKey,
+  query,
+  chunks,
+  digest,
+  config,
+  signal,
+  call = callClaude,
+  stream = streamClaude,
+  telemetry = makeTelemetry(),
+}: AnswerLoopArgs): AsyncGenerator<AgenticEvent> {
+  const byId = new Map(chunks.map((chunk) => [chunk.id, chunk]));
+  const nodesById = new Map(digest.nodes.map((node) => [node.id, node]));
+  const opened = new Map<string, DigestNode>();
+  const seen = new Map<string, DigestNode>(); // sections surfaced by search, in rank order
+  const messages: AnthropicMessage[] = [{ role: 'user', content: `Query: ${query}` }];
+  const system = routedDigestSystemPrompt(digest);
+
+  const open = (id: string): DigestNode | null => {
+    const node = nodesById.get(id);
+    if (node) opened.set(id, node);
+    return node ?? null;
+  };
+
+  // Phase 1: bounded loop of searches and section opens (non-streaming tool turns).
+  for (let i = 0; i < config.maxIterations; i += 1) {
+    const response = await tracedCall(
+      call,
+      {
+        apiKey,
+        model: config.model,
+        system,
+        messages,
+        tools: [SEARCH_SECTIONS_TOOL, OPEN_SECTION_TOOL],
+        toolChoice: { type: 'auto' },
+        maxTokens: 1024,
+        signal,
+      },
+      telemetry,
+      i,
+    );
+
+    messages.push({ role: 'assistant', content: response.content });
+    const toolResults: AnthropicToolResultBlock[] = [];
+
+    for (const block of response.content) {
+      if (block.type !== 'tool_use') continue;
+      if (block.name === 'search_sections') {
+        const searchQuery = normalizeToolQuery(block.input) || query;
+        yield { type: 'search', query: searchQuery };
+        const results = searchSections(searchQuery, chunks, nodesById, digest, config);
+        for (const result of results) {
+          const node = nodesById.get(result.id);
+          if (node && !seen.has(node.id)) seen.set(node.id, node);
+        }
+        toolResults.push({
+          type: 'tool_result',
+          tool_use_id: block.id,
+          content: JSON.stringify(results),
+        });
+      } else if (block.name === 'open_section') {
+        const id = normalizeId(block.input);
+        const node = open(id);
+        toolResults.push({
+          type: 'tool_result',
+          tool_use_id: block.id,
+          content: node
+            ? JSON.stringify(openSectionResult(node, byId))
+            : JSON.stringify({ error: `No section "${id}". Search first, then open an exact id from the results.` }),
+        });
+      }
+    }
+
+    if (!toolResults.length) break; // model is ready to answer
+    messages.push({ role: 'user', content: toolResults });
+  }
+
+  // Fallback: if the model never searched or opened anything, ground on the best
+  // keyword matches for the original query so the answer isn't empty.
+  if (!opened.size && !seen.size) {
+    for (const candidate of prefilter(chunks, query, digest.glossary, config.maxResults, config.perDocCap, digest.nodes)) {
+      const node = nodesById.get(candidate.id);
+      if (node && !seen.has(node.id)) seen.set(node.id, node);
+    }
+    if (seen.size && lastRole(messages) !== 'user') {
+      const sections = [...seen.values()].map((node) => openSectionResult(node, byId));
+      messages.push({ role: 'user', content: `Relevant sections:\n${JSON.stringify(sections)}` });
+    }
+  }
+
+  // The answer is grounded in everything the model surfaced: sections it opened
+  // (full facts) ranked first, then the summaries from its searches. This lets it
+  // answer from search results without a separate open per section.
+  const grounded = new Map<string, DigestNode>([...opened, ...seen]);
+
+  if (lastRole(messages) === 'assistant') {
+    messages.push({
+      role: 'user',
+      content:
+        'Write the answer now, grounded in the sections from your search results and any you opened. Begin directly with the answer — no preamble. Do NOT say you will search, check, or look further: you cannot, and you already have the context you will get. If the sections do not cover the question, say so in one sentence. Link only to section urls you have seen.',
+    });
+  }
+
+  const sources = sourcesFromNodes(grounded, config.maxResults);
+  yield { type: 'sources', sources };
+
+  // Phase 2: streamed answer turn — no tools, so the model can only answer.
+  for await (const event of tracedStream(
+    stream,
+    {
+      apiKey,
+      model: config.model,
+      system: answerSystem(system, sources),
+      messages,
+      maxTokens: config.answerMaxTokens,
+      signal,
+    },
+    telemetry,
+  )) {
+    if (event.type === 'text' && event.text) yield { type: 'token', text: event.text };
+  }
+
+  yield { type: 'done' };
+}
+
 function sourcesFromNodes(opened: Map<string, DigestNode>, maxResults: number): Source[] {
   const sources: Source[] = [];
   const urls = new Set<string>();

package/src/search/prefilter.ts

@@ -48,13 +48,24 @@ export function prefilter(
   if (!terms.length) return [];
 
   const signal = nodeSignal(nodes);
+  // Inverse document frequency: down-weight terms common across the corpus
+  // (stopwords, ubiquitous words like "authentication" or "setup") so a rare,
+  // on-topic term ("oidc") dominates ranking. Without it, plain overlap buries
+  // the specific section under pages that merely share several common words —
+  // which degrades badly as the corpus grows (hundreds → thousands of sections).
+  const df = new Map<string, number>();
+  for (const chunk of chunks) for (const token of chunk.tokens) df.set(token, (df.get(token) ?? 0) + 1);
+  const total = chunks.length;
+  const weights = new Map(terms.map((term) => [term, Math.log(1 + total / (1 + (df.get(term) ?? 0)))]));
+
   const scored = chunks
     .map((chunk) => {
       const boost = signal.get(chunk.id);
       let score = 0;
       for (const term of terms) {
-        if (chunk.tokens.has(term)) score += 1;
-        if (boost?.has(term)) score += 1;
+        const weight = weights.get(term) ?? 0;
+        if (chunk.tokens.has(term)) score += weight;
+        if (boost?.has(term)) score += weight;
       }
       return { chunk, score };
     })