@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/search/find-similar.js

@@ -0,0 +1,509 @@
+import { extractKeyTerms, buildFTS5Query } from "../embedding/key-terms.js";
+import { reciprocalRankFusion, sortByRRFScore } from "./rrf.js";
+import { searchCache, getCachedContent, normalizeUrl, getCacheStats } from "../cache/store.js";
+import { filterByDomains } from "./filters.js";
+import { handleSearch } from "../tools/search.js";
+import { getExtractProvider } from "../providers/extract-provider.js";
+import { getEmbeddingService } from "../embedding/embed.js";
+import { createLogger } from "../logger.js";
+import { selectMode } from "./find-similar/mode.js";
+import { crawlRank } from "./find-similar/crawl-rank.js";
+const log = createLogger("search");
+const DEFAULT_MAX_RESULTS = 10;
+const MAX_FTS5_CANDIDATES = 20;
+const MAX_EMBEDDING_CANDIDATES = 20;
+const WEB_SEARCH_QUERY_COUNT = 3;
+async function findSimilar(input, engines, router, backendStatus) {
+  const start = Date.now();
+  const embeddingAvailable = checkEmbeddingAvailable();
+  const initialCacheSize = safeCacheCount();
+  const initialEmbedIndexSize = safeEmbedIndexSize();
+  try {
+    const mode = selectMode(input);
+    if (mode === "crawl-rank") {
+      const seed = input.url?.trim();
+      if (!seed) {
+        return {
+          results: [],
+          method: "fts5",
+          cache_hits: 0,
+          search_hits: 0,
+          embedding_available: embeddingAvailable,
+          error: "crawl-rank mode requires a url",
+          total_time_ms: Date.now() - start
+        };
+      }
+      const cr = await crawlRank(seed, input, router);
+      return { ...cr, total_time_ms: Date.now() - start };
+    }
+    const url = input.url?.trim();
+    const concept = input.concept?.trim();
+    if (!url && !concept) {
+      return {
+        results: [],
+        method: "fts5",
+        cache_hits: 0,
+        search_hits: 0,
+        embedding_available: embeddingAvailable,
+        error: "Either url or concept must be provided",
+        total_time_ms: Date.now() - start
+      };
+    }
+    const maxResults = input.max_results ?? DEFAULT_MAX_RESULTS;
+    const includeCache = input.include_cache ?? true;
+    const includeWeb = input.include_web ?? true;
+    const signal = await prepareSignal(url, concept, router);
+    if (signal.terms.length === 0 && !signal.queryText) {
+      log.warn("no key terms or query text extracted, falling back to web search");
+      if (!includeWeb) {
+        return {
+          results: [],
+          method: "fts5",
+          cache_hits: 0,
+          search_hits: 0,
+          embedding_available: embeddingAvailable,
+          error: "Could not extract key terms from input and web search is disabled",
+          total_time_ms: Date.now() - start
+        };
+      }
+    }
+    let cacheResults = [];
+    const fts5RankMap = /* @__PURE__ */ new Map();
+    let embeddingResults = [];
+    const embeddingRankMap = /* @__PURE__ */ new Map();
+    await Promise.all([
+      (async () => {
+        if (includeCache && signal.terms.length > 0) {
+          cacheResults = runFTS5Search(
+            signal.terms,
+            signal.inputNormalizedUrl,
+            input.include_domains,
+            input.exclude_domains,
+            MAX_FTS5_CANDIDATES,
+            fts5RankMap
+          );
+          log.debug("FTS5 search complete", { hits: cacheResults.length });
+        }
+      })(),
+      (async () => {
+        if (includeCache && embeddingAvailable && signal.queryText) {
+          embeddingResults = await runEmbeddingSearch(
+            signal.queryText,
+            signal.inputNormalizedUrl,
+            input.include_domains,
+            input.exclude_domains,
+            MAX_EMBEDDING_CANDIDATES,
+            embeddingRankMap
+          );
+          log.debug("embedding search complete", { hits: embeddingResults.length });
+        }
+      })()
+    ]);
+    let searchResults = [];
+    const searchRankMap = /* @__PURE__ */ new Map();
+    const combinedLocalHits = /* @__PURE__ */ new Set();
+    for (const r of cacheResults) combinedLocalHits.add(safeNormalize(r.url));
+    for (const r of embeddingResults) combinedLocalHits.add(safeNormalize(r.url));
+    if (combinedLocalHits.size < maxResults && includeWeb) {
+      searchResults = await runWebSearchFallback(
+        signal,
+        engines,
+        router,
+        backendStatus,
+        maxResults,
+        signal.inputNormalizedUrl,
+        input.include_domains,
+        input.exclude_domains,
+        searchRankMap
+      );
+      log.debug("web search fallback complete", { hits: searchResults.length });
+      if (embeddingAvailable && signal.queryText && searchResults.length > 0) {
+        const freshEmbeddingResults = await runEmbeddingSearch(
+          signal.queryText,
+          signal.inputNormalizedUrl,
+          input.include_domains,
+          input.exclude_domains,
+          MAX_EMBEDDING_CANDIDATES,
+          embeddingRankMap
+        );
+        if (freshEmbeddingResults.length > 0) {
+          embeddingResults = freshEmbeddingResults;
+          log.debug("re-ran embedding search after web fallback", { hits: embeddingResults.length });
+        }
+      }
+    }
+    const rankedLists = [];
+    if (fts5RankMap.size > 0) rankedLists.push(fts5RankMap);
+    if (embeddingRankMap.size > 0) rankedLists.push(embeddingRankMap);
+    if (searchRankMap.size > 0) rankedLists.push(searchRankMap);
+    const allResults = mergeResults(cacheResults, embeddingResults, searchResults);
+    let finalResults;
+    if (rankedLists.length >= 1) {
+      finalResults = fuseResults(rankedLists, allResults, maxResults);
+    } else {
+      finalResults = allResults.sort((a, b) => b.relevance_score - a.relevance_score).slice(0, maxResults);
+    }
+    const method = determineMethod(
+      cacheResults.length > 0,
+      embeddingResults.length > 0,
+      searchResults.length > 0
+    );
+    const cacheHits = finalResults.filter((r) => r.source === "cache").length;
+    const searchHits = finalResults.filter((r) => r.source === "search").length;
+    const coldStart = buildColdStartNote(
+      cacheHits,
+      embeddingAvailable,
+      initialCacheSize,
+      initialEmbedIndexSize
+    );
+    return {
+      results: finalResults,
+      method,
+      cache_hits: cacheHits,
+      search_hits: searchHits,
+      embedding_available: embeddingAvailable,
+      ...coldStart ? { cold_start: coldStart } : {},
+      total_time_ms: Date.now() - start
+    };
+  } catch (err) {
+    log.error("findSimilar failed", { error: String(err) });
+    return {
+      results: [],
+      method: "fts5",
+      cache_hits: 0,
+      search_hits: 0,
+      embedding_available: embeddingAvailable,
+      error: `find_similar failed: ${err instanceof Error ? err.message : String(err)}`,
+      total_time_ms: Date.now() - start
+    };
+  }
+}
+function checkEmbeddingAvailable() {
+  try {
+    const svc = getEmbeddingService();
+    return svc.isAvailable() && svc.isSubprocessReady();
+  } catch {
+    return false;
+  }
+}
+function safeCacheCount() {
+  try {
+    return getCacheStats().total_urls;
+  } catch {
+    return 0;
+  }
+}
+function safeEmbedIndexSize() {
+  try {
+    return getEmbeddingService().getIndex().size();
+  } catch {
+    return 0;
+  }
+}
+function buildColdStartNote(cacheHits, embeddingAvailable, initialCacheSize, initialEmbedIndexSize) {
+  if (initialCacheSize === 0) {
+    return "Cache is empty. Results come from live web search only. Use wigolo_fetch / wigolo_crawl to warm the cache, then re-run find_similar for hybrid local+web ranking.";
+  }
+  if (!embeddingAvailable && initialCacheSize > 0) {
+    return "Embeddings unavailable or index empty (cached pages have not been embedded yet). Falling back to FTS5 keyword ranking. Set up sentence-transformers to enable semantic matching.";
+  }
+  if (cacheHits === 0 && initialCacheSize < 20) {
+    return `Cache has only ${initialCacheSize} pages. Add more context by fetching or crawling relevant sites before relying on find_similar for cross-source similarity.`;
+  }
+  if (!embeddingAvailable && initialEmbedIndexSize === 0) {
+    return "Embedding index is empty. Semantic matching disabled until background embedding jobs catch up.";
+  }
+  return void 0;
+}
+function safeNormalize(url) {
+  try {
+    return normalizeUrl(url);
+  } catch {
+    return url;
+  }
+}
+function mergeResults(...lists) {
+  const seen = /* @__PURE__ */ new Map();
+  for (const list of lists) {
+    for (const r of list) {
+      const key = safeNormalize(r.url);
+      const existing = seen.get(key);
+      if (!existing) {
+        seen.set(key, r);
+      } else {
+        existing.match_signals = {
+          ...existing.match_signals,
+          ...r.match_signals,
+          fused_score: existing.match_signals.fused_score
+        };
+      }
+    }
+  }
+  return [...seen.values()];
+}
+async function prepareSignal(url, concept, router) {
+  if (url) {
+    return await prepareSignalFromUrl(url, router);
+  }
+  if (concept) {
+    const terms = extractKeyTerms(concept, "");
+    return { terms, title: concept, queryText: concept };
+  }
+  return { terms: [], title: "" };
+}
+async function prepareSignalFromUrl(url, router) {
+  let normalizedInputUrl;
+  try {
+    normalizedInputUrl = normalizeUrl(url);
+  } catch {
+    normalizedInputUrl = url;
+  }
+  const cached = getCachedContent(url);
+  if (cached) {
+    const terms = extractKeyTerms(cached.markdown, cached.title);
+    return {
+      terms,
+      title: cached.title,
+      inputUrl: url,
+      inputNormalizedUrl: normalizedInputUrl,
+      queryText: cached.markdown
+    };
+  }
+  try {
+    log.info("fetching URL for signal extraction", { url });
+    const raw = await router.fetch(url, { renderJs: "auto" });
+    const extractor = await getExtractProvider();
+    const extraction = await extractor.extract(raw.html, raw.finalUrl, {
+      contentType: raw.contentType
+    });
+    const terms = extractKeyTerms(extraction.markdown, extraction.title);
+    return {
+      terms,
+      title: extraction.title,
+      inputUrl: url,
+      inputNormalizedUrl: normalizedInputUrl,
+      queryText: extraction.markdown
+    };
+  } catch (err) {
+    log.warn("failed to fetch URL for signal extraction", { url, error: String(err) });
+    const urlTerms = extractKeyTerms("", url);
+    return {
+      terms: urlTerms,
+      title: url,
+      inputUrl: url,
+      inputNormalizedUrl: normalizedInputUrl
+    };
+  }
+}
+async function runEmbeddingSearch(queryText, excludeNormalizedUrl, includeDomains, excludeDomains, topK, rankMap) {
+  try {
+    const service = getEmbeddingService();
+    if (!service.isAvailable() || !service.isSubprocessReady()) return [];
+    if (service.getIndex().size() === 0) return [];
+    const excludeUrls = excludeNormalizedUrl ? /* @__PURE__ */ new Set([excludeNormalizedUrl]) : void 0;
+    const similar = await service.findSimilar(queryText, topK, excludeUrls);
+    if (similar.length === 0) return [];
+    const hydrated = [];
+    for (const { url: nUrl, score } of similar) {
+      const cached = getCachedContent(nUrl);
+      hydrated.push({ entry: cached, url: nUrl, score });
+    }
+    const filterableInputs = hydrated.map((h) => ({
+      url: h.entry?.url ?? h.url
+    }));
+    const filtered = filterByDomains(filterableInputs, includeDomains, excludeDomains);
+    const allowedUrls = new Set(filtered.map((f) => f.url));
+    const results = [];
+    let rank = 0;
+    for (const h of hydrated) {
+      const displayUrl = h.entry?.url ?? h.url;
+      if (!allowedUrls.has(displayUrl)) continue;
+      rank++;
+      rankMap.set(safeNormalize(displayUrl), rank);
+      results.push({
+        url: displayUrl,
+        title: h.entry?.title ?? displayUrl,
+        markdown: (h.entry?.markdown ?? "").slice(0, 5e3),
+        relevance_score: h.score,
+        source: "cache",
+        match_signals: {
+          embedding_rank: rank,
+          fused_score: 0
+        }
+      });
+    }
+    return results;
+  } catch (err) {
+    log.warn("embedding search failed", { error: String(err) });
+    return [];
+  }
+}
+function runFTS5Search(terms, excludeNormalizedUrl, includeDomains, excludeDomains, maxCandidates, rankMap) {
+  try {
+    const fts5Query = buildFTS5Query(terms);
+    if (!fts5Query) return [];
+    let cached = searchCache(fts5Query);
+    if (excludeNormalizedUrl) {
+      cached = cached.filter((c) => {
+        try {
+          return normalizeUrl(c.url) !== excludeNormalizedUrl;
+        } catch {
+          return c.url !== excludeNormalizedUrl;
+        }
+      });
+    }
+    cached = filterByDomains(cached, includeDomains, excludeDomains);
+    cached = cached.slice(0, maxCandidates);
+    const results = [];
+    for (let i = 0; i < cached.length; i++) {
+      const entry = cached[i];
+      let nUrl;
+      try {
+        nUrl = normalizeUrl(entry.url);
+      } catch {
+        nUrl = entry.url;
+      }
+      rankMap.set(nUrl, i + 1);
+      results.push({
+        url: entry.url,
+        title: entry.title,
+        markdown: entry.markdown.slice(0, 5e3),
+        relevance_score: 0,
+        source: "cache",
+        match_signals: {
+          fts5_rank: i + 1,
+          fused_score: 0
+        }
+      });
+    }
+    return results;
+  } catch (err) {
+    log.error("FTS5 search failed", { error: String(err) });
+    return [];
+  }
+}
+async function runWebSearchFallback(signal, engines, router, backendStatus, maxResults, excludeNormalizedUrl, includeDomains, excludeDomains, rankMap) {
+  try {
+    const queries = generateSearchQueries(signal.terms, signal.title);
+    if (queries.length === 0) return [];
+    const allResults = [];
+    const seenUrls = /* @__PURE__ */ new Set();
+    if (excludeNormalizedUrl) {
+      seenUrls.add(excludeNormalizedUrl);
+    }
+    for (const query of queries) {
+      try {
+        const searchResult = await handleSearch(
+          {
+            query,
+            max_results: maxResults,
+            include_content: true,
+            include_domains: includeDomains,
+            exclude_domains: excludeDomains
+          },
+          engines,
+          router,
+          backendStatus
+        );
+        if (!searchResult.ok) {
+          log.warn("web search query failed", { query, error: searchResult.error_reason });
+          continue;
+        }
+        const searchOutput = searchResult.data;
+        for (const item of searchOutput.results) {
+          let nUrl;
+          try {
+            nUrl = normalizeUrl(item.url);
+          } catch {
+            nUrl = item.url;
+          }
+          if (seenUrls.has(nUrl)) continue;
+          seenUrls.add(nUrl);
+          const rank = allResults.length + 1;
+          rankMap.set(nUrl, rank);
+          allResults.push({
+            url: item.url,
+            title: item.title,
+            markdown: (item.markdown_content ?? item.snippet).slice(0, 5e3),
+            relevance_score: item.relevance_score,
+            source: "search",
+            match_signals: {
+              fused_score: 0
+            }
+          });
+        }
+      } catch (err) {
+        log.warn("web search query failed", { query, error: String(err) });
+      }
+    }
+    try {
+      const embeddingService = getEmbeddingService();
+      if (embeddingService.isAvailable() && embeddingService.isSubprocessReady()) {
+        const embedPromises = allResults.filter((r) => r.markdown).slice(0, 10).map((r) => embeddingService.embedAndStore(r.url, r.markdown));
+        await Promise.allSettled(embedPromises);
+        log.debug("embedded web fallback results", { count: embedPromises.length });
+      }
+    } catch (err) {
+      log.debug("embedding hook skipped for find_similar results", { error: String(err) });
+    }
+    return allResults;
+  } catch (err) {
+    log.error("web search fallback failed", { error: String(err) });
+    return [];
+  }
+}
+function generateSearchQueries(terms, title) {
+  if (terms.length === 0 && !title) return [];
+  const queries = [];
+  if (title && title.length > 3) {
+    queries.push(title.slice(0, 150));
+  }
+  if (terms.length >= 3) {
+    queries.push(terms.slice(0, 5).join(" "));
+  }
+  if (terms.length >= 2) {
+    queries.push(`${terms.slice(0, 3).join(" ")} tutorial guide`);
+  }
+  const unique = [...new Set(queries)];
+  return unique.slice(0, WEB_SEARCH_QUERY_COUNT);
+}
+function fuseResults(rankedLists, allResults, maxResults) {
+  const scores = reciprocalRankFusion(rankedLists);
+  const sorted = sortByRRFScore(scores);
+  const resultsByNormalizedUrl = /* @__PURE__ */ new Map();
+  for (const r of allResults) {
+    const key = safeNormalize(r.url);
+    if (!resultsByNormalizedUrl.has(key)) {
+      resultsByNormalizedUrl.set(key, r);
+    }
+  }
+  const topScore = sorted.length > 0 ? sorted[0][1] : 0;
+  const fused = [];
+  for (const [nUrl, score] of sorted) {
+    if (fused.length >= maxResults) break;
+    const result = resultsByNormalizedUrl.get(nUrl);
+    if (!result) continue;
+    const normalized = topScore > 0 ? score / topScore : 0;
+    fused.push({
+      ...result,
+      relevance_score: normalized,
+      match_signals: {
+        ...result.match_signals,
+        fused_score: score
+      }
+    });
+  }
+  return fused;
+}
+function determineMethod(hasCache, hasEmbedding, hasSearch) {
+  const sources = [hasCache, hasEmbedding, hasSearch].filter(Boolean).length;
+  if (sources >= 2) return "hybrid";
+  if (hasEmbedding) return "embedding";
+  if (hasCache) return "fts5";
+  if (hasSearch) return "search";
+  return "fts5";
+}
+export {
+  findSimilar
+};
+//# sourceMappingURL=find-similar.js.map

package/dist/search/find-similar.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/search/find-similar.ts"],"sourcesContent":["import type {\n  FindSimilarInput,\n  FindSimilarOutput,\n  FindSimilarResult,\n  SearchEngine,\n  CachedContent,\n} from '../types.js';\nimport type { SmartRouter } from '../fetch/router.js';\nimport type { BackendStatus } from '../server/backend-status.js';\nimport { extractKeyTerms, buildFTS5Query } from '../embedding/key-terms.js';\nimport { reciprocalRankFusion, sortByRRFScore } from './rrf.js';\nimport { searchCache, getCachedContent, normalizeUrl, getCacheStats } from '../cache/store.js';\nimport { filterByDomains } from './filters.js';\nimport { handleSearch } from '../tools/search.js';\nimport { getExtractProvider } from '../providers/extract-provider.js';\nimport { getEmbeddingService } from '../embedding/embed.js';\nimport { createLogger } from '../logger.js';\nimport { selectMode } from './find-similar/mode.js';\nimport { crawlRank } from './find-similar/crawl-rank.js';\n\nconst log = createLogger('search');\n\nconst DEFAULT_MAX_RESULTS = 10;\nconst MAX_FTS5_CANDIDATES = 20;\nconst MAX_EMBEDDING_CANDIDATES = 20;\nconst WEB_SEARCH_QUERY_COUNT = 3;\n\ninterface ResolvedSignal {\n  terms: string[];\n  title: string;\n  inputUrl?: string;\n  inputNormalizedUrl?: string;\n  queryText?: string;\n}\n\nexport async function findSimilar(\n  input: FindSimilarInput,\n  engines: SearchEngine[],\n  router: SmartRouter,\n  backendStatus?: BackendStatus,\n): Promise<FindSimilarOutput> {\n  const start = Date.now();\n\n  // Probe embedding availability once up front for the whole request\n  const embeddingAvailable = checkEmbeddingAvailable();\n\n  // Snapshot cache/embedding posture BEFORE the web fallback writes new\n  // entries into cache, otherwise the cold-start note would wrongly report a\n  // populated cache simply because we just fetched during this call.\n  const initialCacheSize = safeCacheCount();\n  const initialEmbedIndexSize = safeEmbedIndexSize();\n\n  try {\n    // Mode dispatch: only 'crawl-rank' diverts. All other modes (cache,\n    // web-expansion, auto) fall through to the existing hybrid flow.\n    const mode = selectMode(input);\n    if (mode === 'crawl-rank') {\n      const seed = input.url?.trim();\n      if (!seed) {\n        return {\n          results: [],\n          method: 'fts5',\n          cache_hits: 0,\n          search_hits: 0,\n          embedding_available: embeddingAvailable,\n          error: 'crawl-rank mode requires a url',\n          total_time_ms: Date.now() - start,\n        };\n      }\n      const cr = await crawlRank(seed, input, router);\n      return { ...cr, total_time_ms: Date.now() - start };\n    }\n\n    const url = input.url?.trim();\n    const concept = input.concept?.trim();\n\n    if (!url && !concept) {\n      return {\n        results: [],\n        method: 'fts5',\n        cache_hits: 0,\n        search_hits: 0,\n        embedding_available: embeddingAvailable,\n        error: 'Either url or concept must be provided',\n        total_time_ms: Date.now() - start,\n      };\n    }\n\n    const maxResults = input.max_results ?? DEFAULT_MAX_RESULTS;\n    const includeCache = input.include_cache ?? true;\n    const includeWeb = input.include_web ?? true;\n\n    const signal = await prepareSignal(url, concept, router);\n\n    if (signal.terms.length === 0 && !signal.queryText) {\n      log.warn('no key terms or query text extracted, falling back to web search');\n\n      if (!includeWeb) {\n        return {\n          results: [],\n          method: 'fts5',\n          cache_hits: 0,\n          search_hits: 0,\n          embedding_available: embeddingAvailable,\n          error: 'Could not extract key terms from input and web search is disabled',\n          total_time_ms: Date.now() - start,\n        };\n      }\n    }\n\n    // Phase 1: FTS5 + embedding in parallel (both hit local state, cheap)\n    let cacheResults: FindSimilarResult[] = [];\n    const fts5RankMap = new Map<string, number>();\n    let embeddingResults: FindSimilarResult[] = [];\n    const embeddingRankMap = new Map<string, number>();\n\n    await Promise.all([\n      (async () => {\n        if (includeCache && signal.terms.length > 0) {\n          cacheResults = runFTS5Search(\n            signal.terms,\n            signal.inputNormalizedUrl,\n            input.include_domains,\n            input.exclude_domains,\n            MAX_FTS5_CANDIDATES,\n            fts5RankMap,\n          );\n          log.debug('FTS5 search complete', { hits: cacheResults.length });\n        }\n      })(),\n      (async () => {\n        if (includeCache && embeddingAvailable && signal.queryText) {\n          embeddingResults = await runEmbeddingSearch(\n            signal.queryText,\n            signal.inputNormalizedUrl,\n            input.include_domains,\n            input.exclude_domains,\n            MAX_EMBEDDING_CANDIDATES,\n            embeddingRankMap,\n          );\n          log.debug('embedding search complete', { hits: embeddingResults.length });\n        }\n      })(),\n    ]);\n\n    // Phase 2: Web search fallback (only if combined unique local hits < maxResults)\n    let searchResults: FindSimilarResult[] = [];\n    const searchRankMap = new Map<string, number>();\n\n    const combinedLocalHits = new Set<string>();\n    for (const r of cacheResults) combinedLocalHits.add(safeNormalize(r.url));\n    for (const r of embeddingResults) combinedLocalHits.add(safeNormalize(r.url));\n\n    if (combinedLocalHits.size < maxResults && includeWeb) {\n      searchResults = await runWebSearchFallback(\n        signal,\n        engines,\n        router,\n        backendStatus,\n        maxResults,\n        signal.inputNormalizedUrl,\n        input.include_domains,\n        input.exclude_domains,\n        searchRankMap,\n      );\n      log.debug('web search fallback complete', { hits: searchResults.length });\n\n      // After web fallback, re-run embedding search against newly-populated index.\n      // The web fallback used embedAndStore() (awaited) so vectors are already in\n      // the index — no sleep needed, just re-query.\n      if (embeddingAvailable && signal.queryText && searchResults.length > 0) {\n        const freshEmbeddingResults = await runEmbeddingSearch(\n          signal.queryText,\n          signal.inputNormalizedUrl,\n          input.include_domains,\n          input.exclude_domains,\n          MAX_EMBEDDING_CANDIDATES,\n          embeddingRankMap,\n        );\n\n        if (freshEmbeddingResults.length > 0) {\n          embeddingResults = freshEmbeddingResults;\n          log.debug('re-ran embedding search after web fallback', { hits: embeddingResults.length });\n        }\n      }\n    }\n\n    // Phase 3: 3-way RRF fusion\n    const rankedLists: Map<string, number>[] = [];\n    if (fts5RankMap.size > 0) rankedLists.push(fts5RankMap);\n    if (embeddingRankMap.size > 0) rankedLists.push(embeddingRankMap);\n    if (searchRankMap.size > 0) rankedLists.push(searchRankMap);\n\n    const allResults = mergeResults(cacheResults, embeddingResults, searchResults);\n\n    let finalResults: FindSimilarResult[];\n\n    if (rankedLists.length >= 1) {\n      finalResults = fuseResults(rankedLists, allResults, maxResults);\n    } else {\n      finalResults = allResults\n        .sort((a, b) => b.relevance_score - a.relevance_score)\n        .slice(0, maxResults);\n    }\n\n    const method = determineMethod(\n      cacheResults.length > 0,\n      embeddingResults.length > 0,\n      searchResults.length > 0,\n    );\n\n    const cacheHits = finalResults.filter(r => r.source === 'cache').length;\n    const searchHits = finalResults.filter(r => r.source === 'search').length;\n\n    const coldStart = buildColdStartNote(\n      cacheHits,\n      embeddingAvailable,\n      initialCacheSize,\n      initialEmbedIndexSize,\n    );\n\n    return {\n      results: finalResults,\n      method,\n      cache_hits: cacheHits,\n      search_hits: searchHits,\n      embedding_available: embeddingAvailable,\n      ...(coldStart ? { cold_start: coldStart } : {}),\n      total_time_ms: Date.now() - start,\n    };\n  } catch (err) {\n    log.error('findSimilar failed', { error: String(err) });\n    return {\n      results: [],\n      method: 'fts5',\n      cache_hits: 0,\n      search_hits: 0,\n      embedding_available: embeddingAvailable,\n      error: `find_similar failed: ${err instanceof Error ? err.message : String(err)}`,\n      total_time_ms: Date.now() - start,\n    };\n  }\n}\n\nfunction checkEmbeddingAvailable(): boolean {\n  try {\n    const svc = getEmbeddingService();\n    // Available = service initialized + subprocess verified (Python + model work).\n    // We no longer require index.size() > 0 because the embedding path can\n    // generate query embeddings on-the-fly and compare against freshly-embedded\n    // web fallback results within the same request.\n    return svc.isAvailable() && svc.isSubprocessReady();\n  } catch {\n    return false;\n  }\n}\n\nfunction safeCacheCount(): number {\n  try {\n    return getCacheStats().total_urls;\n  } catch {\n    return 0;\n  }\n}\n\nfunction safeEmbedIndexSize(): number {\n  try {\n    return getEmbeddingService().getIndex().size();\n  } catch {\n    return 0;\n  }\n}\n\n// Surface a note when local hybrid signals are weak so host LLMs can\n// explain to users why results are search-heavy. Avoids silent fallbacks.\nfunction buildColdStartNote(\n  cacheHits: number,\n  embeddingAvailable: boolean,\n  initialCacheSize: number,\n  initialEmbedIndexSize: number,\n): string | undefined {\n  if (initialCacheSize === 0) {\n    return 'Cache is empty. Results come from live web search only. Use wigolo_fetch / wigolo_crawl to warm the cache, then re-run find_similar for hybrid local+web ranking.';\n  }\n  if (!embeddingAvailable && initialCacheSize > 0) {\n    return 'Embeddings unavailable or index empty (cached pages have not been embedded yet). Falling back to FTS5 keyword ranking. Set up sentence-transformers to enable semantic matching.';\n  }\n  if (cacheHits === 0 && initialCacheSize < 20) {\n    return `Cache has only ${initialCacheSize} pages. Add more context by fetching or crawling relevant sites before relying on find_similar for cross-source similarity.`;\n  }\n  if (!embeddingAvailable && initialEmbedIndexSize === 0) {\n    return 'Embedding index is empty. Semantic matching disabled until background embedding jobs catch up.';\n  }\n  return undefined;\n}\n\nfunction safeNormalize(url: string): string {\n  try {\n    return normalizeUrl(url);\n  } catch {\n    return url;\n  }\n}\n\nfunction mergeResults(...lists: FindSimilarResult[][]): FindSimilarResult[] {\n  const seen = new Map<string, FindSimilarResult>();\n  for (const list of lists) {\n    for (const r of list) {\n      const key = safeNormalize(r.url);\n      const existing = seen.get(key);\n      if (!existing) {\n        seen.set(key, r);\n      } else {\n        // Merge match_signals so fused result records the most-specific source info\n        existing.match_signals = {\n          ...existing.match_signals,\n          ...r.match_signals,\n          fused_score: existing.match_signals.fused_score,\n        };\n      }\n    }\n  }\n  return [...seen.values()];\n}\n\nasync function prepareSignal(\n  url: string | undefined,\n  concept: string | undefined,\n  router: SmartRouter,\n): Promise<ResolvedSignal> {\n  if (url) {\n    return await prepareSignalFromUrl(url, router);\n  }\n\n  if (concept) {\n    const terms = extractKeyTerms(concept, '');\n    return { terms, title: concept, queryText: concept };\n  }\n\n  return { terms: [], title: '' };\n}\n\nasync function prepareSignalFromUrl(\n  url: string,\n  router: SmartRouter,\n): Promise<ResolvedSignal> {\n  let normalizedInputUrl: string;\n  try {\n    normalizedInputUrl = normalizeUrl(url);\n  } catch {\n    normalizedInputUrl = url;\n  }\n\n  const cached = getCachedContent(url);\n  if (cached) {\n    const terms = extractKeyTerms(cached.markdown, cached.title);\n    return {\n      terms,\n      title: cached.title,\n      inputUrl: url,\n      inputNormalizedUrl: normalizedInputUrl,\n      queryText: cached.markdown,\n    };\n  }\n\n  try {\n    log.info('fetching URL for signal extraction', { url });\n    const raw = await router.fetch(url, { renderJs: 'auto' });\n    const extractor = await getExtractProvider();\n    const extraction = await extractor.extract(raw.html, raw.finalUrl, {\n      contentType: raw.contentType,\n    });\n    const terms = extractKeyTerms(extraction.markdown, extraction.title);\n    return {\n      terms,\n      title: extraction.title,\n      inputUrl: url,\n      inputNormalizedUrl: normalizedInputUrl,\n      queryText: extraction.markdown,\n    };\n  } catch (err) {\n    log.warn('failed to fetch URL for signal extraction', { url, error: String(err) });\n    const urlTerms = extractKeyTerms('', url);\n    return {\n      terms: urlTerms,\n      title: url,\n      inputUrl: url,\n      inputNormalizedUrl: normalizedInputUrl,\n    };\n  }\n}\n\nasync function runEmbeddingSearch(\n  queryText: string,\n  excludeNormalizedUrl: string | undefined,\n  includeDomains: string[] | undefined,\n  excludeDomains: string[] | undefined,\n  topK: number,\n  rankMap: Map<string, number>,\n): Promise<FindSimilarResult[]> {\n  try {\n    const service = getEmbeddingService();\n    if (!service.isAvailable() || !service.isSubprocessReady()) return [];\n    if (service.getIndex().size() === 0) return [];\n\n    const excludeUrls = excludeNormalizedUrl ? new Set([excludeNormalizedUrl]) : undefined;\n    const similar = await service.findSimilar(queryText, topK, excludeUrls);\n    if (similar.length === 0) return [];\n\n    // Hydrate with cached content and apply domain filters on the hydrated pool\n    const hydrated: Array<{ entry: CachedContent | null; url: string; score: number }> = [];\n    for (const { url: nUrl, score } of similar) {\n      const cached = getCachedContent(nUrl);\n      hydrated.push({ entry: cached, url: nUrl, score });\n    }\n\n    const filterableInputs = hydrated.map(h => ({\n      url: h.entry?.url ?? h.url,\n    })) as unknown as CachedContent[];\n    const filtered = filterByDomains(filterableInputs, includeDomains, excludeDomains) as unknown as Array<{\n      url: string;\n    }>;\n    const allowedUrls = new Set(filtered.map(f => f.url));\n\n    const results: FindSimilarResult[] = [];\n    let rank = 0;\n    for (const h of hydrated) {\n      const displayUrl = h.entry?.url ?? h.url;\n      if (!allowedUrls.has(displayUrl)) continue;\n\n      rank++;\n      rankMap.set(safeNormalize(displayUrl), rank);\n\n      results.push({\n        url: displayUrl,\n        title: h.entry?.title ?? displayUrl,\n        markdown: (h.entry?.markdown ?? '').slice(0, 5000),\n        relevance_score: h.score,\n        source: 'cache',\n        match_signals: {\n          embedding_rank: rank,\n          fused_score: 0,\n        },\n      });\n    }\n\n    return results;\n  } catch (err) {\n    log.warn('embedding search failed', { error: String(err) });\n    return [];\n  }\n}\n\nfunction runFTS5Search(\n  terms: string[],\n  excludeNormalizedUrl: string | undefined,\n  includeDomains: string[] | undefined,\n  excludeDomains: string[] | undefined,\n  maxCandidates: number,\n  rankMap: Map<string, number>,\n): FindSimilarResult[] {\n  try {\n    const fts5Query = buildFTS5Query(terms);\n    if (!fts5Query) return [];\n\n    let cached = searchCache(fts5Query);\n\n    if (excludeNormalizedUrl) {\n      cached = cached.filter(c => {\n        try {\n          return normalizeUrl(c.url) !== excludeNormalizedUrl;\n        } catch {\n          return c.url !== excludeNormalizedUrl;\n        }\n      });\n    }\n\n    cached = filterByDomains(cached, includeDomains, excludeDomains) as CachedContent[];\n    cached = cached.slice(0, maxCandidates);\n\n    const results: FindSimilarResult[] = [];\n    for (let i = 0; i < cached.length; i++) {\n      const entry = cached[i];\n      let nUrl: string;\n      try {\n        nUrl = normalizeUrl(entry.url);\n      } catch {\n        nUrl = entry.url;\n      }\n\n      rankMap.set(nUrl, i + 1);\n\n      results.push({\n        url: entry.url,\n        title: entry.title,\n        markdown: entry.markdown.slice(0, 5000),\n        relevance_score: 0,\n        source: 'cache',\n        match_signals: {\n          fts5_rank: i + 1,\n          fused_score: 0,\n        },\n      });\n    }\n\n    return results;\n  } catch (err) {\n    log.error('FTS5 search failed', { error: String(err) });\n    return [];\n  }\n}\n\nasync function runWebSearchFallback(\n  signal: ResolvedSignal,\n  engines: SearchEngine[],\n  router: SmartRouter,\n  backendStatus: BackendStatus | undefined,\n  maxResults: number,\n  excludeNormalizedUrl: string | undefined,\n  includeDomains: string[] | undefined,\n  excludeDomains: string[] | undefined,\n  rankMap: Map<string, number>,\n): Promise<FindSimilarResult[]> {\n  try {\n    const queries = generateSearchQueries(signal.terms, signal.title);\n    if (queries.length === 0) return [];\n\n    const allResults: FindSimilarResult[] = [];\n    const seenUrls = new Set<string>();\n\n    if (excludeNormalizedUrl) {\n      seenUrls.add(excludeNormalizedUrl);\n    }\n\n    for (const query of queries) {\n      try {\n        const searchResult = await handleSearch(\n          {\n            query,\n            max_results: maxResults,\n            include_content: true,\n            include_domains: includeDomains,\n            exclude_domains: excludeDomains,\n          },\n          engines,\n          router,\n          backendStatus,\n        );\n\n        if (!searchResult.ok) {\n          log.warn('web search query failed', { query, error: searchResult.error_reason });\n          continue;\n        }\n        const searchOutput = searchResult.data;\n\n        for (const item of searchOutput.results) {\n          let nUrl: string;\n          try {\n            nUrl = normalizeUrl(item.url);\n          } catch {\n            nUrl = item.url;\n          }\n\n          if (seenUrls.has(nUrl)) continue;\n          seenUrls.add(nUrl);\n\n          const rank = allResults.length + 1;\n          rankMap.set(nUrl, rank);\n\n          allResults.push({\n            url: item.url,\n            title: item.title,\n            markdown: (item.markdown_content ?? item.snippet).slice(0, 5000),\n            relevance_score: item.relevance_score,\n            source: 'search',\n            match_signals: {\n              fused_score: 0,\n            },\n          });\n        }\n      } catch (err) {\n        log.warn('web search query failed', { query, error: String(err) });\n      }\n    }\n\n    // Embed web results synchronously so they're in the index for the\n    // re-query pass that runs after this fallback. Other tools use embedAsync\n    // (fire-and-forget), but find_similar needs embeddings in THIS request.\n    try {\n      const embeddingService = getEmbeddingService();\n      if (embeddingService.isAvailable() && embeddingService.isSubprocessReady()) {\n        const embedPromises = allResults\n          .filter(r => r.markdown)\n          .slice(0, 10) // cap to avoid blocking too long\n          .map(r => embeddingService.embedAndStore(r.url, r.markdown));\n        await Promise.allSettled(embedPromises);\n        log.debug('embedded web fallback results', { count: embedPromises.length });\n      }\n    } catch (err) {\n      log.debug('embedding hook skipped for find_similar results', { error: String(err) });\n    }\n\n    return allResults;\n  } catch (err) {\n    log.error('web search fallback failed', { error: String(err) });\n    return [];\n  }\n}\n\nfunction generateSearchQueries(terms: string[], title: string): string[] {\n  if (terms.length === 0 && !title) return [];\n\n  const queries: string[] = [];\n\n  if (title && title.length > 3) {\n    queries.push(title.slice(0, 150));\n  }\n\n  if (terms.length >= 3) {\n    queries.push(terms.slice(0, 5).join(' '));\n  }\n\n  if (terms.length >= 2) {\n    queries.push(`${terms.slice(0, 3).join(' ')} tutorial guide`);\n  }\n\n  const unique = [...new Set(queries)];\n  return unique.slice(0, WEB_SEARCH_QUERY_COUNT);\n}\n\nfunction fuseResults(\n  rankedLists: Map<string, number>[],\n  allResults: FindSimilarResult[],\n  maxResults: number,\n): FindSimilarResult[] {\n  const scores = reciprocalRankFusion(rankedLists);\n  const sorted = sortByRRFScore(scores);\n\n  const resultsByNormalizedUrl = new Map<string, FindSimilarResult>();\n  for (const r of allResults) {\n    const key = safeNormalize(r.url);\n    if (!resultsByNormalizedUrl.has(key)) {\n      resultsByNormalizedUrl.set(key, r);\n    }\n  }\n\n  // Raw RRF scores cap at ~2/60 ≈ 0.033 which reads as \"low relevance\" to\n  // users. Normalize against the top score so the best match is 1.0 and the\n  // rest are proportional; the absolute RRF value is preserved in\n  // match_signals.fused_score for clients that depend on it.\n  const topScore = sorted.length > 0 ? sorted[0][1] : 0;\n  const fused: FindSimilarResult[] = [];\n  for (const [nUrl, score] of sorted) {\n    if (fused.length >= maxResults) break;\n\n    const result = resultsByNormalizedUrl.get(nUrl);\n    if (!result) continue;\n\n    const normalized = topScore > 0 ? score / topScore : 0;\n    fused.push({\n      ...result,\n      relevance_score: normalized,\n      match_signals: {\n        ...result.match_signals,\n        fused_score: score,\n      },\n    });\n  }\n\n  return fused;\n}\n\nfunction determineMethod(\n  hasCache: boolean,\n  hasEmbedding: boolean,\n  hasSearch: boolean,\n): FindSimilarOutput['method'] {\n  const sources = [hasCache, hasEmbedding, hasSearch].filter(Boolean).length;\n  if (sources >= 2) return 'hybrid';\n  if (hasEmbedding) return 'embedding';\n  if (hasCache) return 'fts5';\n  if (hasSearch) return 'search';\n  return 'fts5';\n}\n"],"mappings":"AASA,SAAS,iBAAiB,sBAAsB;AAChD,SAAS,sBAAsB,sBAAsB;AACrD,SAAS,aAAa,kBAAkB,cAAc,qBAAqB;AAC3E,SAAS,uBAAuB;AAChC,SAAS,oBAAoB;AAC7B,SAAS,0BAA0B;AACnC,SAAS,2BAA2B;AACpC,SAAS,oBAAoB;AAC7B,SAAS,kBAAkB;AAC3B,SAAS,iBAAiB;AAE1B,MAAM,MAAM,aAAa,QAAQ;AAEjC,MAAM,sBAAsB;AAC5B,MAAM,sBAAsB;AAC5B,MAAM,2BAA2B;AACjC,MAAM,yBAAyB;AAU/B,eAAsB,YACpB,OACA,SACA,QACA,eAC4B;AAC5B,QAAM,QAAQ,KAAK,IAAI;AAGvB,QAAM,qBAAqB,wBAAwB;AAKnD,QAAM,mBAAmB,eAAe;AACxC,QAAM,wBAAwB,mBAAmB;AAEjD,MAAI;AAGF,UAAM,OAAO,WAAW,KAAK;AAC7B,QAAI,SAAS,cAAc;AACzB,YAAM,OAAO,MAAM,KAAK,KAAK;AAC7B,UAAI,CAAC,MAAM;AACT,eAAO;AAAA,UACL,SAAS,CAAC;AAAA,UACV,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ,aAAa;AAAA,UACb,qBAAqB;AAAA,UACrB,OAAO;AAAA,UACP,eAAe,KAAK,IAAI,IAAI;AAAA,QAC9B;AAAA,MACF;AACA,YAAM,KAAK,MAAM,UAAU,MAAM,OAAO,MAAM;AAC9C,aAAO,EAAE,GAAG,IAAI,eAAe,KAAK,IAAI,IAAI,MAAM;AAAA,IACpD;AAEA,UAAM,MAAM,MAAM,KAAK,KAAK;AAC5B,UAAM,UAAU,MAAM,SAAS,KAAK;AAEpC,QAAI,CAAC,OAAO,CAAC,SAAS;AACpB,aAAO;AAAA,QACL,SAAS,CAAC;AAAA,QACV,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,aAAa;AAAA,QACb,qBAAqB;AAAA,QACrB,OAAO;AAAA,QACP,eAAe,KAAK,IAAI,IAAI;AAAA,MAC9B;AAAA,IACF;AAEA,UAAM,aAAa,MAAM,eAAe;AACxC,UAAM,eAAe,MAAM,iBAAiB;AAC5C,UAAM,aAAa,MAAM,eAAe;AAExC,UAAM,SAAS,MAAM,cAAc,KAAK,SAAS,MAAM;AAEvD,QAAI,OAAO,MAAM,WAAW,KAAK,CAAC,OAAO,WAAW;AAClD,UAAI,KAAK,kEAAkE;AAE3E,UAAI,CAAC,YAAY;AACf,eAAO;AAAA,UACL,SAAS,CAAC;AAAA,UACV,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ,aAAa;AAAA,UACb,qBAAqB;AAAA,UACrB,OAAO;AAAA,UACP,eAAe,KAAK,IAAI,IAAI;AAAA,QAC9B;AAAA,MACF;AAAA,IACF;AAGA,QAAI,eAAoC,CAAC;AACzC,UAAM,cAAc,oBAAI,IAAoB;AAC5C,QAAI,mBAAwC,CAAC;AAC7C,UAAM,mBAAmB,oBAAI,IAAoB;AAEjD,UAAM,QAAQ,IAAI;AAAA,OACf,YAAY;AACX,YAAI,gBAAgB,OAAO,MAAM,SAAS,GAAG;AAC3C,yBAAe;AAAA,YACb,OAAO;AAAA,YACP,OAAO;AAAA,YACP,MAAM;AAAA,YACN,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,cAAI,MAAM,wBAAwB,EAAE,MAAM,aAAa,OAAO,CAAC;AAAA,QACjE;AAAA,MACF,GAAG;AAAA,OACF,YAAY;AACX,YAAI,gBAAgB,sBAAsB,OAAO,WAAW;AAC1D,6BAAmB,MAAM;AAAA,YACvB,OAAO;AAAA,YACP,OAAO;AAAA,YACP,MAAM;AAAA,YACN,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,cAAI,MAAM,6BAA6B,EAAE,MAAM,iBAAiB,OAAO,CAAC;AAAA,QAC1E;AAAA,MACF,GAAG;AAAA,IACL,CAAC;AAGD,QAAI,gBAAqC,CAAC;AAC1C,UAAM,gBAAgB,oBAAI,IAAoB;AAE9C,UAAM,oBAAoB,oBAAI,IAAY;AAC1C,eAAW,KAAK,aAAc,mBAAkB,IAAI,cAAc,EAAE,GAAG,CAAC;AACxE,eAAW,KAAK,iBAAkB,mBAAkB,IAAI,cAAc,EAAE,GAAG,CAAC;AAE5E,QAAI,kBAAkB,OAAO,cAAc,YAAY;AACrD,sBAAgB,MAAM;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,OAAO;AAAA,QACP,MAAM;AAAA,QACN,MAAM;AAAA,QACN;AAAA,MACF;AACA,UAAI,MAAM,gCAAgC,EAAE,MAAM,cAAc,OAAO,CAAC;AAKxE,UAAI,sBAAsB,OAAO,aAAa,cAAc,SAAS,GAAG;AACtE,cAAM,wBAAwB,MAAM;AAAA,UAClC,OAAO;AAAA,UACP,OAAO;AAAA,UACP,MAAM;AAAA,UACN,MAAM;AAAA,UACN;AAAA,UACA;AAAA,QACF;AAEA,YAAI,sBAAsB,SAAS,GAAG;AACpC,6BAAmB;AACnB,cAAI,MAAM,8CAA8C,EAAE,MAAM,iBAAiB,OAAO,CAAC;AAAA,QAC3F;AAAA,MACF;AAAA,IACF;AAGA,UAAM,cAAqC,CAAC;AAC5C,QAAI,YAAY,OAAO,EAAG,aAAY,KAAK,WAAW;AACtD,QAAI,iBAAiB,OAAO,EAAG,aAAY,KAAK,gBAAgB;AAChE,QAAI,cAAc,OAAO,EAAG,aAAY,KAAK,aAAa;AAE1D,UAAM,aAAa,aAAa,cAAc,kBAAkB,aAAa;AAE7E,QAAI;AAEJ,QAAI,YAAY,UAAU,GAAG;AAC3B,qBAAe,YAAY,aAAa,YAAY,UAAU;AAAA,IAChE,OAAO;AACL,qBAAe,WACZ,KAAK,CAAC,GAAG,MAAM,EAAE,kBAAkB,EAAE,eAAe,EACpD,MAAM,GAAG,UAAU;AAAA,IACxB;AAEA,UAAM,SAAS;AAAA,MACb,aAAa,SAAS;AAAA,MACtB,iBAAiB,SAAS;AAAA,MAC1B,cAAc,SAAS;AAAA,IACzB;AAEA,UAAM,YAAY,aAAa,OAAO,OAAK,EAAE,WAAW,OAAO,EAAE;AACjE,UAAM,aAAa,aAAa,OAAO,OAAK,EAAE,WAAW,QAAQ,EAAE;AAEnE,UAAM,YAAY;AAAA,MAChB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,MACA,YAAY;AAAA,MACZ,aAAa;AAAA,MACb,qBAAqB;AAAA,MACrB,GAAI,YAAY,EAAE,YAAY,UAAU,IAAI,CAAC;AAAA,MAC7C,eAAe,KAAK,IAAI,IAAI;AAAA,IAC9B;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,MAAM,sBAAsB,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACtD,WAAO;AAAA,MACL,SAAS,CAAC;AAAA,MACV,QAAQ;AAAA,MACR,YAAY;AAAA,MACZ,aAAa;AAAA,MACb,qBAAqB;AAAA,MACrB,OAAO,wBAAwB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MAC/E,eAAe,KAAK,IAAI,IAAI;AAAA,IAC9B;AAAA,EACF;AACF;AAEA,SAAS,0BAAmC;AAC1C,MAAI;AACF,UAAM,MAAM,oBAAoB;AAKhC,WAAO,IAAI,YAAY,KAAK,IAAI,kBAAkB;AAAA,EACpD,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,iBAAyB;AAChC,MAAI;AACF,WAAO,cAAc,EAAE;AAAA,EACzB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,qBAA6B;AACpC,MAAI;AACF,WAAO,oBAAoB,EAAE,SAAS,EAAE,KAAK;AAAA,EAC/C,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAIA,SAAS,mBACP,WACA,oBACA,kBACA,uBACoB;AACpB,MAAI,qBAAqB,GAAG;AAC1B,WAAO;AAAA,EACT;AACA,MAAI,CAAC,sBAAsB,mBAAmB,GAAG;AAC/C,WAAO;AAAA,EACT;AACA,MAAI,cAAc,KAAK,mBAAmB,IAAI;AAC5C,WAAO,kBAAkB,gBAAgB;AAAA,EAC3C;AACA,MAAI,CAAC,sBAAsB,0BAA0B,GAAG;AACtD,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,SAAS,cAAc,KAAqB;AAC1C,MAAI;AACF,WAAO,aAAa,GAAG;AAAA,EACzB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,gBAAgB,OAAmD;AAC1E,QAAM,OAAO,oBAAI,IAA+B;AAChD,aAAW,QAAQ,OAAO;AACxB,eAAW,KAAK,MAAM;AACpB,YAAM,MAAM,cAAc,EAAE,GAAG;AAC/B,YAAM,WAAW,KAAK,IAAI,GAAG;AAC7B,UAAI,CAAC,UAAU;AACb,aAAK,IAAI,KAAK,CAAC;AAAA,MACjB,OAAO;AAEL,iBAAS,gBAAgB;AAAA,UACvB,GAAG,SAAS;AAAA,UACZ,GAAG,EAAE;AAAA,UACL,aAAa,SAAS,cAAc;AAAA,QACtC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO,CAAC,GAAG,KAAK,OAAO,CAAC;AAC1B;AAEA,eAAe,cACb,KACA,SACA,QACyB;AACzB,MAAI,KAAK;AACP,WAAO,MAAM,qBAAqB,KAAK,MAAM;AAAA,EAC/C;AAEA,MAAI,SAAS;AACX,UAAM,QAAQ,gBAAgB,SAAS,EAAE;AACzC,WAAO,EAAE,OAAO,OAAO,SAAS,WAAW,QAAQ;AAAA,EACrD;AAEA,SAAO,EAAE,OAAO,CAAC,GAAG,OAAO,GAAG;AAChC;AAEA,eAAe,qBACb,KACA,QACyB;AACzB,MAAI;AACJ,MAAI;AACF,yBAAqB,aAAa,GAAG;AAAA,EACvC,QAAQ;AACN,yBAAqB;AAAA,EACvB;AAEA,QAAM,SAAS,iBAAiB,GAAG;AACnC,MAAI,QAAQ;AACV,UAAM,QAAQ,gBAAgB,OAAO,UAAU,OAAO,KAAK;AAC3D,WAAO;AAAA,MACL;AAAA,MACA,OAAO,OAAO;AAAA,MACd,UAAU;AAAA,MACV,oBAAoB;AAAA,MACpB,WAAW,OAAO;AAAA,IACpB;AAAA,EACF;AAEA,MAAI;AACF,QAAI,KAAK,sCAAsC,EAAE,IAAI,CAAC;AACtD,UAAM,MAAM,MAAM,OAAO,MAAM,KAAK,EAAE,UAAU,OAAO,CAAC;AACxD,UAAM,YAAY,MAAM,mBAAmB;AAC3C,UAAM,aAAa,MAAM,UAAU,QAAQ,IAAI,MAAM,IAAI,UAAU;AAAA,MACjE,aAAa,IAAI;AAAA,IACnB,CAAC;AACD,UAAM,QAAQ,gBAAgB,WAAW,UAAU,WAAW,KAAK;AACnE,WAAO;AAAA,MACL;AAAA,MACA,OAAO,WAAW;AAAA,MAClB,UAAU;AAAA,MACV,oBAAoB;AAAA,MACpB,WAAW,WAAW;AAAA,IACxB;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,KAAK,6CAA6C,EAAE,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AACjF,UAAM,WAAW,gBAAgB,IAAI,GAAG;AACxC,WAAO;AAAA,MACL,OAAO;AAAA,MACP,OAAO;AAAA,MACP,UAAU;AAAA,MACV,oBAAoB;AAAA,IACtB;AAAA,EACF;AACF;AAEA,eAAe,mBACb,WACA,sBACA,gBACA,gBACA,MACA,SAC8B;AAC9B,MAAI;AACF,UAAM,UAAU,oBAAoB;AACpC,QAAI,CAAC,QAAQ,YAAY,KAAK,CAAC,QAAQ,kBAAkB,EAAG,QAAO,CAAC;AACpE,QAAI,QAAQ,SAAS,EAAE,KAAK,MAAM,EAAG,QAAO,CAAC;AAE7C,UAAM,cAAc,uBAAuB,oBAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI;AAC7E,UAAM,UAAU,MAAM,QAAQ,YAAY,WAAW,MAAM,WAAW;AACtE,QAAI,QAAQ,WAAW,EAAG,QAAO,CAAC;AAGlC,UAAM,WAA+E,CAAC;AACtF,eAAW,EAAE,KAAK,MAAM,MAAM,KAAK,SAAS;AAC1C,YAAM,SAAS,iBAAiB,IAAI;AACpC,eAAS,KAAK,EAAE,OAAO,QAAQ,KAAK,MAAM,MAAM,CAAC;AAAA,IACnD;AAEA,UAAM,mBAAmB,SAAS,IAAI,QAAM;AAAA,MAC1C,KAAK,EAAE,OAAO,OAAO,EAAE;AAAA,IACzB,EAAE;AACF,UAAM,WAAW,gBAAgB,kBAAkB,gBAAgB,cAAc;AAGjF,UAAM,cAAc,IAAI,IAAI,SAAS,IAAI,OAAK,EAAE,GAAG,CAAC;AAEpD,UAAM,UAA+B,CAAC;AACtC,QAAI,OAAO;AACX,eAAW,KAAK,UAAU;AACxB,YAAM,aAAa,EAAE,OAAO,OAAO,EAAE;AACrC,UAAI,CAAC,YAAY,IAAI,UAAU,EAAG;AAElC;AACA,cAAQ,IAAI,cAAc,UAAU,GAAG,IAAI;AAE3C,cAAQ,KAAK;AAAA,QACX,KAAK;AAAA,QACL,OAAO,EAAE,OAAO,SAAS;AAAA,QACzB,WAAW,EAAE,OAAO,YAAY,IAAI,MAAM,GAAG,GAAI;AAAA,QACjD,iBAAiB,EAAE;AAAA,QACnB,QAAQ;AAAA,QACR,eAAe;AAAA,UACb,gBAAgB;AAAA,UAChB,aAAa;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,KAAK,2BAA2B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC1D,WAAO,CAAC;AAAA,EACV;AACF;AAEA,SAAS,cACP,OACA,sBACA,gBACA,gBACA,eACA,SACqB;AACrB,MAAI;AACF,UAAM,YAAY,eAAe,KAAK;AACtC,QAAI,CAAC,UAAW,QAAO,CAAC;AAExB,QAAI,SAAS,YAAY,SAAS;AAElC,QAAI,sBAAsB;AACxB,eAAS,OAAO,OAAO,OAAK;AAC1B,YAAI;AACF,iBAAO,aAAa,EAAE,GAAG,MAAM;AAAA,QACjC,QAAQ;AACN,iBAAO,EAAE,QAAQ;AAAA,QACnB;AAAA,MACF,CAAC;AAAA,IACH;AAEA,aAAS,gBAAgB,QAAQ,gBAAgB,cAAc;AAC/D,aAAS,OAAO,MAAM,GAAG,aAAa;AAEtC,UAAM,UAA+B,CAAC;AACtC,aAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,YAAM,QAAQ,OAAO,CAAC;AACtB,UAAI;AACJ,UAAI;AACF,eAAO,aAAa,MAAM,GAAG;AAAA,MAC/B,QAAQ;AACN,eAAO,MAAM;AAAA,MACf;AAEA,cAAQ,IAAI,MAAM,IAAI,CAAC;AAEvB,cAAQ,KAAK;AAAA,QACX,KAAK,MAAM;AAAA,QACX,OAAO,MAAM;AAAA,QACb,UAAU,MAAM,SAAS,MAAM,GAAG,GAAI;AAAA,QACtC,iBAAiB;AAAA,QACjB,QAAQ;AAAA,QACR,eAAe;AAAA,UACb,WAAW,IAAI;AAAA,UACf,aAAa;AAAA,QACf;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,sBAAsB,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACtD,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAe,qBACb,QACA,SACA,QACA,eACA,YACA,sBACA,gBACA,gBACA,SAC8B;AAC9B,MAAI;AACF,UAAM,UAAU,sBAAsB,OAAO,OAAO,OAAO,KAAK;AAChE,QAAI,QAAQ,WAAW,EAAG,QAAO,CAAC;AAElC,UAAM,aAAkC,CAAC;AACzC,UAAM,WAAW,oBAAI,IAAY;AAEjC,QAAI,sBAAsB;AACxB,eAAS,IAAI,oBAAoB;AAAA,IACnC;AAEA,eAAW,SAAS,SAAS;AAC3B,UAAI;AACF,cAAM,eAAe,MAAM;AAAA,UACzB;AAAA,YACE;AAAA,YACA,aAAa;AAAA,YACb,iBAAiB;AAAA,YACjB,iBAAiB;AAAA,YACjB,iBAAiB;AAAA,UACnB;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AAEA,YAAI,CAAC,aAAa,IAAI;AACpB,cAAI,KAAK,2BAA2B,EAAE,OAAO,OAAO,aAAa,aAAa,CAAC;AAC/E;AAAA,QACF;AACA,cAAM,eAAe,aAAa;AAElC,mBAAW,QAAQ,aAAa,SAAS;AACvC,cAAI;AACJ,cAAI;AACF,mBAAO,aAAa,KAAK,GAAG;AAAA,UAC9B,QAAQ;AACN,mBAAO,KAAK;AAAA,UACd;AAEA,cAAI,SAAS,IAAI,IAAI,EAAG;AACxB,mBAAS,IAAI,IAAI;AAEjB,gBAAM,OAAO,WAAW,SAAS;AACjC,kBAAQ,IAAI,MAAM,IAAI;AAEtB,qBAAW,KAAK;AAAA,YACd,KAAK,KAAK;AAAA,YACV,OAAO,KAAK;AAAA,YACZ,WAAW,KAAK,oBAAoB,KAAK,SAAS,MAAM,GAAG,GAAI;AAAA,YAC/D,iBAAiB,KAAK;AAAA,YACtB,QAAQ;AAAA,YACR,eAAe;AAAA,cACb,aAAa;AAAA,YACf;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,KAAK,2BAA2B,EAAE,OAAO,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,MACnE;AAAA,IACF;AAKA,QAAI;AACF,YAAM,mBAAmB,oBAAoB;AAC7C,UAAI,iBAAiB,YAAY,KAAK,iBAAiB,kBAAkB,GAAG;AAC1E,cAAM,gBAAgB,WACnB,OAAO,OAAK,EAAE,QAAQ,EACtB,MAAM,GAAG,EAAE,EACX,IAAI,OAAK,iBAAiB,cAAc,EAAE,KAAK,EAAE,QAAQ,CAAC;AAC7D,cAAM,QAAQ,WAAW,aAAa;AACtC,YAAI,MAAM,iCAAiC,EAAE,OAAO,cAAc,OAAO,CAAC;AAAA,MAC5E;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,MAAM,mDAAmD,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IACrF;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,8BAA8B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC9D,WAAO,CAAC;AAAA,EACV;AACF;AAEA,SAAS,sBAAsB,OAAiB,OAAyB;AACvE,MAAI,MAAM,WAAW,KAAK,CAAC,MAAO,QAAO,CAAC;AAE1C,QAAM,UAAoB,CAAC;AAE3B,MAAI,SAAS,MAAM,SAAS,GAAG;AAC7B,YAAQ,KAAK,MAAM,MAAM,GAAG,GAAG,CAAC;AAAA,EAClC;AAEA,MAAI,MAAM,UAAU,GAAG;AACrB,YAAQ,KAAK,MAAM,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC;AAAA,EAC1C;AAEA,MAAI,MAAM,UAAU,GAAG;AACrB,YAAQ,KAAK,GAAG,MAAM,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,iBAAiB;AAAA,EAC9D;AAEA,QAAM,SAAS,CAAC,GAAG,IAAI,IAAI,OAAO,CAAC;AACnC,SAAO,OAAO,MAAM,GAAG,sBAAsB;AAC/C;AAEA,SAAS,YACP,aACA,YACA,YACqB;AACrB,QAAM,SAAS,qBAAqB,WAAW;AAC/C,QAAM,SAAS,eAAe,MAAM;AAEpC,QAAM,yBAAyB,oBAAI,IAA+B;AAClE,aAAW,KAAK,YAAY;AAC1B,UAAM,MAAM,cAAc,EAAE,GAAG;AAC/B,QAAI,CAAC,uBAAuB,IAAI,GAAG,GAAG;AACpC,6BAAuB,IAAI,KAAK,CAAC;AAAA,IACnC;AAAA,EACF;AAMA,QAAM,WAAW,OAAO,SAAS,IAAI,OAAO,CAAC,EAAE,CAAC,IAAI;AACpD,QAAM,QAA6B,CAAC;AACpC,aAAW,CAAC,MAAM,KAAK,KAAK,QAAQ;AAClC,QAAI,MAAM,UAAU,WAAY;AAEhC,UAAM,SAAS,uBAAuB,IAAI,IAAI;AAC9C,QAAI,CAAC,OAAQ;AAEb,UAAM,aAAa,WAAW,IAAI,QAAQ,WAAW;AACrD,UAAM,KAAK;AAAA,MACT,GAAG;AAAA,MACH,iBAAiB;AAAA,MACjB,eAAe;AAAA,QACb,GAAG,OAAO;AAAA,QACV,aAAa;AAAA,MACf;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,SAAS,gBACP,UACA,cACA,WAC6B;AAC7B,QAAM,UAAU,CAAC,UAAU,cAAc,SAAS,EAAE,OAAO,OAAO,EAAE;AACpE,MAAI,WAAW,EAAG,QAAO;AACzB,MAAI,aAAc,QAAO;AACzB,MAAI,SAAU,QAAO;AACrB,MAAI,UAAW,QAAO;AACtB,SAAO;AACT;","names":[]}

package/dist/search/highlights.d.ts

@@ -0,0 +1,19 @@
+import type { SearchResultItem, Citation, Highlight } from '../types.js';
+export interface HighlightSynthesisResult {
+    highlights: Highlight[];
+    citations: Citation[];
+    reranker_used: boolean;
+}
+export interface Passage {
+    text: string;
+    charStart: number;
+    charEnd: number;
+}
+export declare function splitIntoPassages(markdown: string): Passage[];
+export interface AnnotatedPassage extends Passage {
+    sectionHeading: string | null;
+}
+export declare function mapPassageHeadings(markdown: string, passages: Passage[]): AnnotatedPassage[];
+export declare function extractHighlights(query: string, results: SearchResultItem[], maxHighlights?: number): Promise<HighlightSynthesisResult>;
+export declare function fallbackHighlights(results: SearchResultItem[], maxHighlights: number): Highlight[];
+//# sourceMappingURL=highlights.d.ts.map

package/dist/search/highlights.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"highlights.d.ts","sourceRoot":"","sources":["../../src/search/highlights.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAYzE,MAAM,WAAW,wBAAwB;IACvC,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAwBD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,EAAE,CA6B7D;AAOD,MAAM,WAAW,gBAAiB,SAAQ,OAAO;IAC/C,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAKD,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,OAAO,EAAE,GAClB,gBAAgB,EAAE,CAapB;AAMD,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,gBAAgB,EAAE,EAC3B,aAAa,GAAE,MAA+B,GAC7C,OAAO,CAAC,wBAAwB,CAAC,CAmEnC;AAKD,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,aAAa,EAAE,MAAM,GACpB,SAAS,EAAE,CAkCb"}