@agentskit/doc-bridge 1.7.45 → 1.9.0

package/src/query/search.ts

@@ -1,126 +1,323 @@
 import type { DocBridgeIndexV1 } from '../schemas/doc-bridge-index.js'
+import type { Evidence } from '../schemas/knowledge.js'
+import type { Confidence, RetrievalKind } from '../schemas/retrieval-index.js'
+import { rankRetrieval, type RankExplanation, type RankedEntry } from '../retrieval/rank.js'
+import { buildBm25Index, bm25Search, type Bm25Index, type Bm25Input } from '../retrieval/bm25.js'
+import { isProjectedEntry } from '../index-builder/project-corpus.js'
+import { foldAccents, hasSearchToken, searchTokens } from './text.js'
 
 export type SearchMatch = {
+  /** The record kind as consumers have always read it: `ownership`, `intent`, `change`, or the entry kind. */
   readonly type: string
   readonly id: string
   readonly path: string
   readonly summary?: string
   readonly score: number
+  /** The snapshot entity this result stands for, when it stands for one. */
+  readonly entityId?: string
+  readonly kind?: RetrievalKind
+  readonly provenance?: 'observed' | 'declared' | 'proposed'
+  readonly confidence?: Confidence
+  readonly evidence?: readonly Evidence[]
+  readonly explain?: RankExplanation
 }
 
-const tokenize = (value: string): string[] =>
-  value
-    .toLowerCase()
-    .split(/[^a-z0-9@/_-]+/)
-    .filter((t) => t.length >= 2)
+export type SearchOptions = {
+  /** Attach the matched terms and each scoring component to every result. Never changes the ranking. */
+  readonly explain?: boolean
+  /** The `--agent` prior. */
+  readonly agent?: boolean
+}
 
-const PACKAGE_INTENT =
-  /\b(package|module|pkg|edit|change|where|owns?|ownership|handoff|start)\b/i
+/**
+ * Search the index.
+ *
+ * An index that carries the retrieval projection is ranked over it — every snapshot entity, with
+ * evidence, provenance and confidence on each result. An index built before the projection
+ * existed is ranked the way it always was, over its `knowledge[]` and lookup records, so an older
+ * artifact keeps answering until it is rebuilt.
+ */
+export const searchIndex = (index: DocBridgeIndexV1, term: string, limit = 20, options: SearchOptions = {}): SearchMatch[] => {
+  if (index.projection) return projectedSearch(index, term, limit, options)
+  return legacySearch(index, term, limit)
+}
 
-const scoreHay = (tokens: readonly string[], hay: string, weight = 1): number => {
-  let score = 0
-  for (const token of tokens) {
-    if (!hay.includes(token)) continue
-    score += token.length * weight
-    // whole-word-ish bonus
-    if (new RegExp(`(?:^|[^a-z0-9])${token}(?:[^a-z0-9]|$)`).test(hay)) {
-      score += token.length
+/**
+ * How a projected result presents itself to a reader of the old shape.
+ *
+ * An entry that stands for an ownership record is reported as that record — `type: ownership`,
+ * the record's id, its agent document as path — because that is what a routing command expects
+ * to receive and what `ak-docs query ownership <id>` accepts. Everything else is reported by its
+ * entity id and its kind.
+ */
+type Presented = { readonly agentDocs: ReadonlyMap<string, string>; readonly curatedIds: ReadonlyMap<string, string> }
+
+const presentation = (index: DocBridgeIndexV1, known: Presented, item: RankedEntry): Pick<SearchMatch, 'type' | 'id' | 'path'> => {
+  const { entry } = item
+  // The unit an ownership record names, or the agent document that record points at: one answer.
+  const ownershipId = entry.ownershipId ?? (entry.kind === 'document' ? known.agentDocs.get(entry.path) : undefined)
+  if (ownershipId) {
+    const record = index.lookup?.ownership?.[ownershipId]
+    return { type: 'ownership', id: ownershipId, path: record?.agentDoc ?? entry.path }
+  }
+  // A curated sidecar keeps the id its frontmatter gave it: that is the id `doc.get` and llms.txt know.
+  const curatedId = entry.kind === 'document' ? known.curatedIds.get(entry.path) : undefined
+  return { type: entry.kind, id: curatedId ?? entry.id, path: entry.path }
+}
+
+const projectedSearch = (index: DocBridgeIndexV1, term: string, limit: number, options: SearchOptions): SearchMatch[] => {
+  const projection = index.projection
+  if (!projection) return []
+  const ranked = rankRetrieval(projection, term, { limit: Math.max(limit * 2, 40), ...(options.agent ? { agent: true } : {}) })
+  /*
+   * Which ownership record an agent document stands for. A record attached to a projected unit
+   * is authoritative; a corpus-derived seed that points at the same document is the same unit
+   * under another name and must not win the presentation over it.
+   */
+  const agentDocs = new Map<string, string>()
+  for (const entry of projection.entries) {
+    const record = entry.ownershipId ? index.lookup?.ownership?.[entry.ownershipId] : undefined
+    if (record?.agentDoc && !agentDocs.has(record.agentDoc)) agentDocs.set(record.agentDoc, record.id)
+  }
+  for (const record of Object.values(index.lookup?.ownership ?? {})) {
+    if (record.agentDoc && !agentDocs.has(record.agentDoc)) agentDocs.set(record.agentDoc, record.id)
+  }
+  const known: Presented = {
+    agentDocs,
+    curatedIds: new Map(index.knowledge.filter((entry) => !isProjectedEntry(entry)).map((entry) => [entry.path, entry.id])),
+  }
+  const byPath = new Map<string, SearchMatch>()
+
+  for (const item of ranked) {
+    const presented = presentation(index, known, item)
+    const match: SearchMatch = {
+      ...presented,
+      ...(item.entry.summary ? { summary: item.entry.summary } : {}),
+      score: item.score,
+      entityId: item.entry.id,
+      kind: item.entry.kind,
+      provenance: item.entry.provenance,
+      confidence: item.confidence,
+      evidence: [{ source: evidenceSource(item.entry.kind), path: item.entry.path, contentHash: item.entry.contentHash }],
+      ...(options.explain ? { explain: item.explanation } : {}),
+    }
+    const existing = byPath.get(match.path)
+    if (!existing || match.score > existing.score || (match.score === existing.score && match.type === 'ownership' && existing.type !== 'ownership')) {
+      byPath.set(match.path, match)
     }
   }
-  return score
+
+  return [...byPath.values()].sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, limit)
 }
 
-/** Exact / near-exact identity boost so "core" ranks the core package over mentions of @agentskit/core. */
-const identityBoost = (id: string, path: string, tokens: readonly string[], term: string): number => {
-  const idLower = id.toLowerCase()
-  const termLower = term.toLowerCase().trim()
-  const base = path.split('/').pop()?.replace(/\.mdx?$/i, '').toLowerCase() ?? ''
-  let boost = 0
+const evidenceSource = (kind: RetrievalKind): Evidence['source'] =>
+  kind === 'module' ? 'code' : kind === 'document' ? 'documentation' : kind === 'intent' || kind === 'change' ? 'configuration' : 'derived'
 
-  if (idLower === termLower || base === termLower) boost += 200
-  if (tokens.length === 1 && (idLower === tokens[0] || base === tokens[0])) boost += 200
+/*
+ * ---------------------------------------------------------------------------------------------
+ * The pre-projection ranking, kept verbatim for indexes that carry no projection.
+ * ---------------------------------------------------------------------------------------------
+ */
 
-  for (const token of tokens) {
-    if (idLower === token) boost += 120
-    else if (idLower.startsWith(`${token}-`) || idLower.endsWith(`-${token}`)) boost += 40
-    else if (idLower.includes(token) && idLower.length <= token.length + 4) boost += 30
-    if (base === token) boost += 100
-  }
+const BM25_SCALE = 50
+const EXACT_IDENTITY = 240
+const EXACT_SYMBOL = 200
+const DIRECTORY_MATCH = 120
+const TOKEN_ID = 120
+const TOKEN_BASE = 100
+const TOKEN_SYMBOL = 90
+const SHORT_ID_BONUS = 40
+const CURATED_FACTOR = 1.15
+const OWNERSHIP_FACTOR = 1.1
+const ROUTE_TITLE_FACTOR = 1.6
+const CHANGE_INTENT_FACTOR = 1.4
+const CHANGE_WITHOUT_INTENT_FACTOR = 0.15
+const KIND_FACTOR = 1.15
+const PARTIAL_ID_FACTOR = 1.2
+const RELEVANCE_FLOOR = 1 / 3
 
-  // Prefer short ids when term is the id (package name)
-  if (tokens.includes(idLower)) boost += Math.max(0, 40 - idLower.length)
+const LEGACY_WEIGHTS = { id: 8, symbols: 7, title: 6, path: 4, tags: 3, description: 2, body: 1 } as const
 
-  return boost
+const PACKAGE_INTENT =
+  /\b(package|module|pkg|edit|change|where|owns?|ownership|handoff|start)\b|\b(?:pacote|pacotes|modulo|modulos|onde|quem|dono|donos|responsavel|responsaveis|comec\w*|inici\w*|edit\w*|mud\w*|alter\w*)/i
+const CHANGE_INTENT =
+  /\b(change|edit|modify|update|fix|migrate|replace)\b|\b(?:alter\w*|mud\w*|modific\w*|edit\w*|atualiz\w*|corrig\w*|migr\w*|substitu\w*|troc\w*)/i
+const SYMBOL_SHAPED = /[a-z0-9][A-Z]|^[A-Za-z_$][A-Za-z0-9_$]{2,}$/
+const PATH_SHAPED = /\/|\.[A-Za-z]{1,4}$/
+
+type CandidateFields = {
+  readonly id: string
+  readonly title: string
+  readonly path: string
+  readonly description?: string
+  readonly body?: string
+  readonly tags?: readonly string[]
+  readonly symbols?: readonly string[]
 }
 
-const preferOwnership = (term: string): boolean =>
-  PACKAGE_INTENT.test(term) || /^(where|how).*(edit|change|package|module)/i.test(term)
+type Candidate = {
+  readonly ref: string
+  readonly match: Omit<SearchMatch, 'score'>
+  readonly fields: CandidateFields
+  readonly entryType?: string
+}
 
-export const searchIndex = (index: DocBridgeIndexV1, term: string, limit = 20): SearchMatch[] => {
-  const tokens = tokenize(term)
-  if (!tokens.length) return []
+const pathBase = (path: string): string => foldAccents((path.split('/').pop() ?? '').replace(/\.[A-Za-z0-9]+$/, '').toLowerCase())
 
-  const wantOwnership = preferOwnership(term)
-  const byPath = new Map<string, SearchMatch>()
+const preferOwnership = (term: string): boolean => PACKAGE_INTENT.test(term) || /^(where|how).*(edit|change|package|module)/i.test(term)
 
-  const consider = (match: SearchMatch) => {
-    const key = match.path
-    const existing = byPath.get(key)
-    if (!existing) {
-      byPath.set(key, match)
-      return
-    }
-    // Prefer ownership over knowledge for same path; else higher score
-    const prefer =
-      match.score > existing.score ||
-      (match.score === existing.score && match.type === 'ownership' && existing.type !== 'ownership') ||
-      (wantOwnership && match.type === 'ownership' && existing.type !== 'ownership' && match.score >= existing.score - 20)
-    if (prefer) byPath.set(key, match)
+const titleCoversQuery = (title: string, tokens: readonly string[]): boolean =>
+  tokens.length > 1 && tokens.every((token) => hasSearchToken(foldAccents(title.toLowerCase()), token))
+
+const identityBoost = (fields: CandidateFields, tokens: readonly string[], term: string): number => {
+  const idLower = foldAccents(fields.id.toLowerCase())
+  const pathLower = foldAccents(fields.path.toLowerCase())
+  const termLower = foldAccents(term.toLowerCase().trim())
+  const base = pathBase(fields.path)
+  const symbols = new Set((fields.symbols ?? []).map((symbol) => foldAccents(symbol.toLowerCase())))
+  let boost = 0
+  if (idLower === termLower || base === termLower || pathLower === termLower) boost += EXACT_IDENTITY
+  if (symbols.has(termLower)) boost += EXACT_SYMBOL
+  if (termLower && (pathLower.startsWith(`${termLower}/`) || pathLower.includes(`/${termLower}/`))) boost += DIRECTORY_MATCH
+  for (const token of tokens) {
+    if (idLower === token) boost += TOKEN_ID
+    if (base === token) boost += TOKEN_BASE
+    if (symbols.has(token)) boost += TOKEN_SYMBOL
   }
+  if (tokens.includes(idLower)) boost += Math.max(0, SHORT_ID_BONUS - idLower.length)
+  return boost
+}
 
-  for (const entry of index.knowledge) {
-    const body = (entry as { body?: string }).body ?? ''
-    const hay = `${entry.id} ${entry.title} ${entry.description ?? ''} ${entry.path} ${body}`.toLowerCase()
-    let score = scoreHay(tokens, hay, 1)
-    score += identityBoost(entry.id, entry.path, tokens, term)
-    if (score > 0) {
-      consider({
-        type: 'knowledge',
-        id: entry.id,
-        path: entry.path,
-        ...(entry.description ? { summary: entry.description } : {}),
-        score,
-      })
+const candidateRef = (kind: string, position: number, id: string): string => `${kind}#${position}#${id}`
+
+const knowledgeCandidates = (index: DocBridgeIndexV1): Candidate[] =>
+  index.knowledge.map((entry, position) => ({
+    ref: candidateRef('knowledge', position, entry.id),
+    match: { type: 'knowledge', id: entry.id, path: entry.path, ...(entry.description ? { summary: entry.description } : {}) },
+    fields: {
+      id: entry.id,
+      title: entry.title,
+      path: entry.path,
+      ...(entry.description ? { description: entry.description } : {}),
+      ...(entry.body ? { body: entry.body } : {}),
+      ...(entry.tags ? { tags: entry.tags } : {}),
+      ...(entry.symbols ? { symbols: entry.symbols } : {}),
+    },
+    entryType: entry.type,
+  }))
+
+const ownershipCandidates = (index: DocBridgeIndexV1): Candidate[] =>
+  Object.entries(index.lookup?.ownership ?? {}).map(([id, owner], position) => {
+    const agentDoc = owner.agentDoc ? index.knowledge.find((entry) => entry.path === owner.agentDoc) : undefined
+    const description = owner.purpose ?? agentDoc?.description
+    return {
+      ref: candidateRef('ownership', position, id),
+      match: { type: 'ownership', id, path: owner.agentDoc ?? owner.path, ...(owner.purpose ? { summary: owner.purpose } : {}) },
+      fields: {
+        id,
+        title: agentDoc?.title ?? id,
+        path: [owner.path, owner.agentDoc, owner.humanDoc].filter(Boolean).join(' '),
+        ...(description ? { description } : {}),
+        ...(agentDoc?.body ? { body: agentDoc.body } : {}),
+        tags: [owner.group, owner.layer, 'ownership'].filter((value): value is string => Boolean(value)),
+      },
+      entryType: 'ownership',
     }
+  })
+
+const intentCandidates = (index: DocBridgeIndexV1): Candidate[] =>
+  Object.values(index.lookup?.intents ?? {}).map((intent, position) => ({
+    ref: candidateRef('intent', position, intent.id),
+    match: { type: 'intent', id: intent.id, path: intent.paths[0] ?? '', summary: intent.title },
+    fields: { id: intent.id, title: intent.title, path: intent.paths.join(' '), tags: ['intent'] },
+    entryType: 'intent',
+  }))
+
+const changeCandidates = (index: DocBridgeIndexV1): Candidate[] =>
+  Object.values(index.lookup?.changes ?? {}).map((change, position) => ({
+    ref: candidateRef('change', position, change.id),
+    match: { type: 'change', id: change.id, path: change.startHere, summary: change.title },
+    fields: { id: change.id, title: change.title, path: change.startHere, tags: ['change', ...(change.relatedPackages ?? [])] },
+    entryType: 'change',
+  }))
+
+const partialIdMatch = (id: string, tokens: readonly string[]): boolean => {
+  const idLower = foldAccents(id.toLowerCase())
+  return tokens.some(
+    (token) => idLower !== token && (idLower.startsWith(`${token}-`) || idLower.endsWith(`-${token}`) || (idLower.includes(token) && idLower.length <= token.length + 4)),
+  )
+}
+
+const priorFactor = (candidate: Candidate, tokens: readonly string[], term: string, wantOwnership: boolean): number => {
+  const type = candidate.match.type
+  const trimmed = term.trim()
+  const covered = titleCoversQuery(candidate.fields.title, tokens)
+  let factor = 1
+  if (!isProjectedEntry({ type: candidate.entryType ?? '' })) factor *= CURATED_FACTOR
+  if (partialIdMatch(candidate.fields.id, tokens)) factor *= PARTIAL_ID_FACTOR
+  if (type === 'ownership' && wantOwnership) factor *= OWNERSHIP_FACTOR
+  if ((type === 'intent' || type === 'change') && covered) factor *= ROUTE_TITLE_FACTOR
+  if (type === 'change') factor *= CHANGE_INTENT.test(term) ? CHANGE_INTENT_FACTOR : covered ? 1 : CHANGE_WITHOUT_INTENT_FACTOR
+  const oneWord = !/\s/.test(trimmed)
+  const looksLikeSymbol = oneWord && SYMBOL_SHAPED.test(trimmed)
+  const looksLikePath = oneWord && PATH_SHAPED.test(trimmed)
+  if ((looksLikeSymbol || looksLikePath) && candidate.entryType === 'module') factor *= KIND_FACTOR
+  if (!looksLikeSymbol && !looksLikePath && tokens.length >= 3 && candidate.entryType === 'document') factor *= KIND_FACTOR
+  return factor
+}
+
+type PreparedIndex = { readonly byRef: ReadonlyMap<string, Candidate>; readonly bm25: Bm25Index }
+
+const prepared = new WeakMap<DocBridgeIndexV1, PreparedIndex>()
+
+const prepare = (index: DocBridgeIndexV1): PreparedIndex | undefined => {
+  const cached = prepared.get(index)
+  if (cached) return cached
+  const candidates = [...knowledgeCandidates(index), ...ownershipCandidates(index), ...intentCandidates(index), ...changeCandidates(index)]
+  if (!candidates.length) return undefined
+  const inputs: Bm25Input[] = candidates.map((candidate) => ({ ref: candidate.ref, fields: candidate.fields }))
+  const weights = { ...LEGACY_WEIGHTS, ...Object.fromEntries(Object.entries(index.retrieval?.weights ?? {}).filter(([field]) => field in LEGACY_WEIGHTS)) }
+  const value: PreparedIndex = {
+    byRef: new Map(candidates.map((candidate) => [candidate.ref, candidate])),
+    bm25: buildBm25Index(inputs, weights, index.retrieval?.params ?? {}),
   }
+  prepared.set(index, value)
+  return value
+}
 
-  for (const [id, owner] of Object.entries(index.lookup?.ownership ?? {})) {
-    const path = owner.agentDoc ?? owner.path
-    const hay = `${id} ${owner.path} ${owner.purpose ?? ''} ${owner.group ?? ''} ${owner.agentDoc ?? ''} ${owner.humanDoc ?? ''}`.toLowerCase()
-    let score = scoreHay(tokens, hay, 2)
-    score += identityBoost(id, path, tokens, term)
-    // Ownership is primary for routing questions
-    if (wantOwnership) score += 25
-    score += 15 // slight base preference for actionable ownership targets
-    if (score > 0) {
-      consider({
-        type: 'ownership',
-        id,
-        path,
-        ...(owner.purpose ? { summary: owner.purpose } : {}),
-        score,
-      })
+const legacySearch = (index: DocBridgeIndexV1, term: string, limit: number): SearchMatch[] => {
+  const tokens = searchTokens(term)
+  if (!tokens.length) return []
+  const ready = prepare(index)
+  if (!ready) return []
+  const { byRef, bm25 } = ready
+  const hits = bm25Search(bm25, tokens)
+  const wantOwnership = preferOwnership(term)
+  const byPath = new Map<string, SearchMatch>()
+
+  for (const hit of hits) {
+    const candidate = byRef.get(hit.ref)
+    if (!candidate) continue
+    const score = hit.score * BM25_SCALE * priorFactor(candidate, tokens, term, wantOwnership) + identityBoost(candidate.fields, tokens, term)
+    const match: SearchMatch = { ...candidate.match, score }
+    const existing = byPath.get(match.path)
+    if (!existing || match.score > existing.score || (match.score === existing.score && match.type === 'ownership' && existing.type !== 'ownership')) {
+      byPath.set(match.path, match)
     }
   }
 
-  return [...byPath.values()].sort((a, b) => {
-    if (b.score !== a.score) return b.score - a.score
-    // Tie-break: exact id match, then ownership, then shorter id
-    const aExact = tokens.includes(a.id.toLowerCase()) ? 1 : 0
-    const bExact = tokens.includes(b.id.toLowerCase()) ? 1 : 0
-    if (bExact !== aExact) return bExact - aExact
-    if (a.type === 'ownership' && b.type !== 'ownership') return -1
-    if (b.type === 'ownership' && a.type !== 'ownership') return 1
-    return a.id.localeCompare(b.id)
-  }).slice(0, limit)
+  const ranked = [...byPath.values()]
+    .sort((a, b) => {
+      if (b.score !== a.score) return b.score - a.score
+      const aExact = tokens.includes(foldAccents(a.id.toLowerCase())) ? 1 : 0
+      const bExact = tokens.includes(foldAccents(b.id.toLowerCase())) ? 1 : 0
+      if (bExact !== aExact) return bExact - aExact
+      if (a.type === 'ownership' && b.type !== 'ownership') return -1
+      if (b.type === 'ownership' && a.type !== 'ownership') return 1
+      return a.id.localeCompare(b.id)
+    })
+    .slice(0, limit)
+
+  const best = ranked[0]?.score ?? 0
+  return best > 0 ? ranked.filter((match) => match.score >= best * RELEVANCE_FLOOR) : ranked
 }

package/src/query/text.ts

@@ -0,0 +1,155 @@
+const SEARCH_TOKEN_SEPARATOR = /[^\p{L}\p{N}@/_-]+/gu
+
+/**
+ * Bumped whenever a stopword list or the token expansion changes. The index records it, so a
+ * ranking that depended on a different list is visible rather than silently incomparable.
+ */
+export const SEARCH_LEXICON_VERSION = 1 as const
+
+/**
+ * Words that carry no retrieval signal because they appear in nearly every sentence of the
+ * corpus. Without them `search and` scores every record that has a prose summary.
+ *
+ * Portuguese is here for the same reason English is: the repository targets non-English
+ * documentation, and a query in one language must be ranked by the same rules as the other.
+ * Single characters never reach these sets, since tokens shorter than two characters are dropped.
+ */
+const ENGLISH_STOPWORDS = [
+  'about', 'after', 'again', 'all', 'also', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'been',
+  'before', 'being', 'between', 'both', 'but', 'by', 'can', 'could', 'did', 'do', 'does', 'doing',
+  'done', 'during', 'each', 'either', 'else', 'for', 'from', 'had', 'has', 'have', 'he', 'her',
+  'here', 'hers', 'him', 'his', 'how', 'if', 'in', 'into', 'is', 'it', 'its', 'just', 'me', 'more',
+  'most', 'much', 'must', 'my', 'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or',
+  'other', 'our', 'out', 'over', 'own', 'same', 'she', 'should', 'so', 'some', 'such', 'than',
+  'that', 'the', 'their', 'them', 'then', 'there', 'these', 'they', 'this', 'those', 'through',
+  'to', 'too', 'under', 'until', 'up', 'us', 'very', 'was', 'we', 'were', 'what', 'when', 'where',
+  'which', 'while', 'who', 'whom', 'why', 'will', 'with', 'would', 'you', 'your',
+] as const
+
+const PORTUGUESE_STOPWORDS = [
+  'ainda', 'ao', 'aos', 'aquela', 'aquelas', 'aquele', 'aqueles', 'aquilo', 'as', 'até', 'com',
+  'como', 'da', 'das', 'de', 'dela', 'delas', 'dele', 'deles', 'depois', 'do', 'dos', 'ela',
+  'elas', 'ele', 'eles', 'em', 'entre', 'era', 'eram', 'essa', 'essas', 'esse', 'esses', 'esta',
+  'estas', 'este', 'estes', 'está', 'estão', 'eu', 'fica', 'ficam', 'foi', 'foram', 'isso',
+  'isto', 'já', 'lhe', 'lhes', 'mais', 'mas', 'me', 'menos', 'mesmo', 'meu', 'minha', 'muito',
+  'na', 'nas', 'nem', 'no', 'nos', 'nossa', 'nosso', 'num', 'numa', 'não', 'onde', 'os', 'ou',
+  'para', 'pela', 'pelas', 'pelo', 'pelos', 'per', 'por', 'porque', 'pra', 'qual', 'quais',
+  'quando', 'que', 'quem', 'se', 'sem', 'ser', 'seu', 'seus', 'sobre', 'sua', 'suas', 'são',
+  'também', 'tem', 'ter', 'teu', 'toda', 'todas', 'todo', 'todos', 'tua', 'têm', 'um', 'uma',
+  'umas', 'uns', 'vira', 'você', 'vocês',
+] as const
+
+/**
+ * Strip diacritics so `reconciliacao` and `reconciliação` are the same term.
+ *
+ * Non-English documentation is written with accents and queried without them at least as often as
+ * the reverse, and a retrieval path that treats the two as different words fails a question it has
+ * the answer to. Folding happens on both sides — the indexed text and the query — so the two can
+ * never disagree.
+ */
+export const foldAccents = (value: string): string =>
+  value.normalize('NFD').replace(/\p{Diacritic}/gu, '').normalize('NFC')
+
+export const SEARCH_STOPWORDS: ReadonlySet<string> = new Set<string>(
+  [...ENGLISH_STOPWORDS, ...PORTUGUESE_STOPWORDS].map(foldAccents),
+)
+
+export const isSearchStopword = (token: string): boolean => SEARCH_STOPWORDS.has(foldAccents(token))
+
+/** Keep the deterministic search path usable for non-English documentation too. */
+export const tokenizeSearchText = (value: string): string[] =>
+  value
+    .toLowerCase()
+    .split(SEARCH_TOKEN_SEPARATOR)
+    .filter((token) => token.length >= 2)
+
+const CAMEL_BOUNDARY = /(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/g
+
+const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u
+const CJK_RUN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu
+
+/**
+ * Characters and character bigrams of a CJK run.
+ *
+ * CJK text has no spaces, so a whole sentence arrives as one token and an exact-term index would
+ * never match a query for two of its characters. Indexing unigrams and bigrams on both sides is
+ * the standard answer: the bigram carries the signal, the unigram keeps a one-character query
+ * answerable.
+ */
+const cjkGrams = (run: string): string[] => {
+  const characters = [...run]
+  const grams = [...characters]
+  for (let position = 0; position + 1 < characters.length; position += 1) {
+    grams.push(`${characters[position]}${characters[position + 1]}`)
+  }
+  return grams
+}
+
+/**
+ * A code identifier or path, plus the words inside it.
+ *
+ * `reconcileKnowledge` yields the whole identifier and `reconcile` and `knowledge`, so an agent
+ * that types the exact symbol gets an exact match while a human who half-remembers it still
+ * lands nearby. `src/mcp/server.ts` yields the whole path and each segment without its extension.
+ */
+export const expandSearchToken = (token: string): string[] => {
+  const whole = foldAccents(token.toLowerCase())
+  const parts = foldAccents(token)
+    .replace(CAMEL_BOUNDARY, ' ')
+    .split(/[^\p{L}\p{N}]+/u)
+    .map((part) => part.toLowerCase())
+    .filter((part) => part.length >= 2 && part !== whole)
+  const grams = CJK.test(token) ? [...token.matchAll(CJK_RUN)].flatMap(([run]) => cjkGrams(run)) : []
+  return [...new Set([whole, ...parts, ...grams])]
+}
+
+/**
+ * Collapse a plural to its singular so `schema` finds `schemas`.
+ *
+ * The substring matcher this path replaced accepted an optional plural suffix, and losing that
+ * would have made an exact-term index quietly worse at the most ordinary query there is. These
+ * are the safe rules only: applied to both the indexed text and the query, an imperfect stem
+ * still matches itself, so the worst case is a missed relation rather than a wrong one.
+ */
+export const singularizeSearchToken = (token: string): string => {
+  if (token.length <= 3) return token
+  if (token.endsWith('ies')) return `${token.slice(0, -3)}y`
+  if (/(?:sses|xes|ches|shes|zes)$/.test(token)) return token.slice(0, -2)
+  if (/(?:ss|us|is|os)$/.test(token)) return token
+  if (token.endsWith('es') && token.length > 4) return token.slice(0, -1)
+  if (token.endsWith('s')) return token.slice(0, -1)
+  return token
+}
+
+export type SearchTokenizeOptions = {
+  /** Drop stopwords. On for queries and for indexed text; off only when a caller needs raw tokens. */
+  readonly dropStopwords?: boolean
+  /** Also emit the words inside an identifier or path. */
+  readonly expand?: boolean
+}
+
+/**
+ * The tokenizer both indexing and querying use, so a term can never be present on one side and
+ * absent on the other.
+ */
+export const searchTokens = (value: string, options: SearchTokenizeOptions = {}): string[] => {
+  const dropStopwords = options.dropStopwords ?? true
+  const expand = options.expand ?? true
+  const tokens: string[] = []
+  for (const raw of value.split(SEARCH_TOKEN_SEPARATOR)) {
+    if (!raw) continue
+    for (const token of expand ? expandSearchToken(raw) : [raw.toLowerCase()]) {
+      if (token.length < 2 && !CJK.test(token)) continue
+      if (dropStopwords && isSearchStopword(token)) continue
+      tokens.push(CJK.test(token) ? token : singularizeSearchToken(token))
+    }
+  }
+  return tokens
+}
+
+export const hasSearchToken = (hay: string, token: string): boolean => {
+  // CJK text commonly has no whitespace; substring matching is the native word boundary there.
+  if (/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u.test(token)) return hay.includes(token)
+  const escaped = token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  return new RegExp(`(?:^|[^\\p{L}\\p{N}])${escaped}(?:s|es)?(?:[^\\p{L}\\p{N}]|$)`, 'u').test(hay)
+}