@staticn0va/wigolo 0.1.0 → 0.1.2

package/dist/search/find-similar/crawl-rank.js

@@ -0,0 +1,272 @@
+import { getExtractProvider } from "../../providers/extract-provider.js";
+import { getEmbedProvider } from "../../providers/embed-provider.js";
+import { createLogger } from "../../logger.js";
+const log = createLogger("search");
+const DEFAULT_MAX_PAGES = 20;
+const DEFAULT_CONCURRENCY = 4;
+const DEFAULT_FETCH_TIMEOUT_MS = 1e4;
+const DEFAULT_MAX_RESULTS = 10;
+const SEED_TEXT_CHARS = 500;
+async function crawlRank(seedUrl, input, router, options = {}) {
+  const start = Date.now();
+  const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
+  const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
+  const fetchTimeoutMs = options.fetchTimeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
+  const maxResults = input.max_results ?? DEFAULT_MAX_RESULTS;
+  let seedHost;
+  try {
+    seedHost = new URL(seedUrl).hostname;
+  } catch {
+    return emptyOutput({
+      error: "Invalid seed URL",
+      embeddingAvailable: false,
+      elapsed: Date.now() - start
+    });
+  }
+  let seedRaw;
+  try {
+    seedRaw = await router.fetch(seedUrl, { renderJs: "auto" });
+  } catch (err) {
+    return emptyOutput({
+      error: `Seed fetch failed: ${err instanceof Error ? err.message : String(err)}`,
+      embeddingAvailable: false,
+      elapsed: Date.now() - start
+    });
+  }
+  if (!seedRaw || typeof seedRaw.statusCode !== "number" || seedRaw.statusCode < 200 || seedRaw.statusCode >= 300) {
+    return emptyOutput({
+      error: `Seed fetch failed: ${seedRaw?.statusCode ?? "unknown"}`,
+      embeddingAvailable: false,
+      elapsed: Date.now() - start
+    });
+  }
+  const extractor = await getExtractProvider();
+  const seedExtraction = await extractor.extract(seedRaw.html, seedRaw.finalUrl, {
+    contentType: seedRaw.contentType
+  });
+  const includeDomains = input.include_domains;
+  const excludeDomains = input.exclude_domains;
+  const allowedLinks = filterLinks(
+    seedExtraction.links ?? [],
+    seedHost,
+    includeDomains,
+    excludeDomains,
+    maxPages
+  );
+  let embedProvider;
+  let embeddingAvailable = false;
+  try {
+    embedProvider = await getEmbedProvider();
+    embeddingAvailable = true;
+  } catch (err) {
+    log.debug("embedding provider unavailable for crawl-rank", { error: String(err) });
+    embeddingAvailable = false;
+  }
+  if (allowedLinks.length === 0) {
+    return emptyOutput({
+      error: "No same-host links found from seed",
+      embeddingAvailable,
+      elapsed: Date.now() - start
+    });
+  }
+  if (!embeddingAvailable || !embedProvider) {
+    const degraded = degradedResults(allowedLinks, maxResults);
+    return {
+      results: degraded,
+      method: "fts5",
+      cache_hits: 0,
+      search_hits: degraded.length,
+      embedding_available: false,
+      error: "Embedding unavailable \u2014 returned link order",
+      total_time_ms: Date.now() - start
+    };
+  }
+  const fetched = await fetchPagesInChunks(
+    allowedLinks,
+    router,
+    extractor,
+    concurrency,
+    fetchTimeoutMs
+  );
+  if (fetched.length === 0) {
+    return emptyOutput({
+      error: "No linked pages could be fetched",
+      embeddingAvailable: true,
+      elapsed: Date.now() - start
+    });
+  }
+  const seedText = buildPageText(seedExtraction.title, seedExtraction.markdown);
+  const pageTexts = fetched.map((p) => p.text);
+  let vectors;
+  try {
+    vectors = await embedProvider.embed([seedText, ...pageTexts]);
+  } catch (err) {
+    log.warn("embedding batch failed; falling back to link order", { error: String(err) });
+    const degraded = degradedResults(allowedLinks, maxResults);
+    return {
+      results: degraded,
+      method: "fts5",
+      cache_hits: 0,
+      search_hits: degraded.length,
+      embedding_available: true,
+      error: `Embedding batch failed: ${err instanceof Error ? err.message : String(err)}`,
+      total_time_ms: Date.now() - start
+    };
+  }
+  const seedVec = vectors[0];
+  const pageVecs = vectors.slice(1);
+  const scored = fetched.map((page, i) => ({
+    page,
+    score: cosineSimilarity(seedVec, pageVecs[i])
+  }));
+  scored.sort((a, b) => b.score - a.score);
+  const final = scored.slice(0, maxResults).map(({ page, score }) => ({
+    url: page.url,
+    title: page.title,
+    markdown: page.markdown.slice(0, 5e3),
+    relevance_score: score,
+    source: "search",
+    match_signals: {
+      fused_score: score
+    }
+  }));
+  return {
+    results: final,
+    method: "embedding",
+    cache_hits: 0,
+    search_hits: final.length,
+    embedding_available: true,
+    total_time_ms: Date.now() - start
+  };
+}
+function filterLinks(links, seedHost, includeDomains, excludeDomains, maxPages) {
+  const allowOtherHosts = (includeDomains?.length ?? 0) > 0;
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const raw of links) {
+    let parsed;
+    try {
+      parsed = new URL(raw);
+    } catch {
+      continue;
+    }
+    const host = parsed.hostname;
+    if (!allowOtherHosts && host !== seedHost) continue;
+    if (allowOtherHosts && host !== seedHost) {
+      const widened = includeDomains.some((d) => domainMatches(host, d));
+      if (!widened) continue;
+    }
+    if (excludeDomains && excludeDomains.some((d) => domainMatches(host, d))) continue;
+    const normalized = normalize(parsed);
+    if (seen.has(normalized)) continue;
+    seen.add(normalized);
+    out.push(parsed.toString());
+    if (out.length >= maxPages) break;
+  }
+  return out;
+}
+function domainMatches(host, pattern) {
+  const p = pattern.toLowerCase().replace(/^\*\./, "");
+  const h = host.toLowerCase();
+  return h === p || h.endsWith(`.${p}`);
+}
+function normalize(parsed) {
+  const copy = new URL(parsed.toString());
+  copy.hash = "";
+  return copy.toString().replace(/\/$/, "");
+}
+async function fetchPagesInChunks(urls, router, extractor, concurrency, fetchTimeoutMs) {
+  const out = [];
+  for (let i = 0; i < urls.length; i += concurrency) {
+    const chunk = urls.slice(i, i + concurrency);
+    const settled = await Promise.allSettled(
+      chunk.map(async (u) => {
+        const raw = await withTimeout(
+          router.fetch(u, { renderJs: "auto" }),
+          fetchTimeoutMs
+        );
+        if (!raw || typeof raw.statusCode !== "number" || raw.statusCode < 200 || raw.statusCode >= 300) {
+          throw new Error(`bad status: ${raw?.statusCode ?? "unknown"}`);
+        }
+        const ex = await extractor.extract(raw.html, raw.finalUrl, {
+          contentType: raw.contentType
+        });
+        return {
+          url: raw.finalUrl,
+          title: ex.title || raw.finalUrl,
+          markdown: ex.markdown ?? "",
+          text: buildPageText(ex.title, ex.markdown ?? "")
+        };
+      })
+    );
+    for (const r of settled) {
+      if (r.status === "fulfilled") out.push(r.value);
+      else log.debug("crawl-rank page fetch failed", { error: String(r.reason) });
+    }
+  }
+  return out;
+}
+function buildPageText(title, markdown) {
+  const t = (title ?? "").trim();
+  const body = (markdown ?? "").slice(0, SEED_TEXT_CHARS);
+  return t ? `${t}
+${body}` : body;
+}
+function cosineSimilarity(a, b) {
+  if (!a || !b || a.length === 0 || b.length === 0) return 0;
+  const len = Math.min(a.length, b.length);
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < len; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+function degradedResults(links, maxResults) {
+  const limited = links.slice(0, maxResults);
+  const n = limited.length;
+  return limited.map((url, i) => ({
+    url,
+    title: url,
+    markdown: "",
+    relevance_score: n > 0 ? 1 - i / n : 0,
+    source: "search",
+    match_signals: {
+      fused_score: n > 0 ? 1 - i / n : 0
+    }
+  }));
+}
+function withTimeout(p, ms) {
+  return new Promise((resolve, reject) => {
+    const t = setTimeout(() => reject(new Error(`timeout after ${ms}ms`)), ms);
+    p.then(
+      (v) => {
+        clearTimeout(t);
+        resolve(v);
+      },
+      (e) => {
+        clearTimeout(t);
+        reject(e);
+      }
+    );
+  });
+}
+function emptyOutput(args) {
+  return {
+    results: [],
+    method: "fts5",
+    cache_hits: 0,
+    search_hits: 0,
+    embedding_available: args.embeddingAvailable,
+    error: args.error,
+    total_time_ms: args.elapsed
+  };
+}
+export {
+  crawlRank
+};
+//# sourceMappingURL=crawl-rank.js.map

package/dist/search/find-similar/crawl-rank.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/search/find-similar/crawl-rank.ts"],"sourcesContent":["import type {\n  FindSimilarInput,\n  FindSimilarOutput,\n  FindSimilarResult,\n} from '../../types.js';\nimport type { SmartRouter } from '../../fetch/router.js';\nimport { getExtractProvider } from '../../providers/extract-provider.js';\nimport { getEmbedProvider } from '../../providers/embed-provider.js';\nimport { createLogger } from '../../logger.js';\n\nconst log = createLogger('search');\n\nconst DEFAULT_MAX_PAGES = 20;\nconst DEFAULT_CONCURRENCY = 4;\nconst DEFAULT_FETCH_TIMEOUT_MS = 10000;\nconst DEFAULT_MAX_RESULTS = 10;\nconst SEED_TEXT_CHARS = 500;\n\nexport interface CrawlRankOptions {\n  maxPages?: number;\n  concurrency?: number;\n  fetchTimeoutMs?: number;\n}\n\ninterface FetchedPage {\n  url: string;\n  title: string;\n  markdown: string;\n  text: string;\n}\n\nexport async function crawlRank(\n  seedUrl: string,\n  input: FindSimilarInput,\n  router: SmartRouter,\n  options: CrawlRankOptions = {},\n): Promise<FindSimilarOutput> {\n  const start = Date.now();\n  const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;\n  const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;\n  const fetchTimeoutMs = options.fetchTimeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;\n  const maxResults = input.max_results ?? DEFAULT_MAX_RESULTS;\n\n  let seedHost: string;\n  try {\n    seedHost = new URL(seedUrl).hostname;\n  } catch {\n    return emptyOutput({\n      error: 'Invalid seed URL',\n      embeddingAvailable: false,\n      elapsed: Date.now() - start,\n    });\n  }\n\n  // 1. Fetch seed\n  let seedRaw;\n  try {\n    seedRaw = await router.fetch(seedUrl, { renderJs: 'auto' });\n  } catch (err) {\n    return emptyOutput({\n      error: `Seed fetch failed: ${err instanceof Error ? err.message : String(err)}`,\n      embeddingAvailable: false,\n      elapsed: Date.now() - start,\n    });\n  }\n\n  if (!seedRaw || typeof seedRaw.statusCode !== 'number' || seedRaw.statusCode < 200 || seedRaw.statusCode >= 300) {\n    return emptyOutput({\n      error: `Seed fetch failed: ${seedRaw?.statusCode ?? 'unknown'}`,\n      embeddingAvailable: false,\n      elapsed: Date.now() - start,\n    });\n  }\n\n  // 2. Extract seed\n  const extractor = await getExtractProvider();\n  const seedExtraction = await extractor.extract(seedRaw.html, seedRaw.finalUrl, {\n    contentType: seedRaw.contentType,\n  });\n\n  // 3. Filter links: same-host (unless explicitly widened), domain filters, dedup, cap\n  const includeDomains = input.include_domains;\n  const excludeDomains = input.exclude_domains;\n  const allowedLinks = filterLinks(\n    seedExtraction.links ?? [],\n    seedHost,\n    includeDomains,\n    excludeDomains,\n    maxPages,\n  );\n\n  // 6. Probe embedding (before failing on empty links so error messaging reflects\n  // capability). If no links AND no embedding, the empty-links error is still\n  // the more useful signal.\n  let embedProvider;\n  let embeddingAvailable = false;\n  try {\n    embedProvider = await getEmbedProvider();\n    embeddingAvailable = true;\n  } catch (err) {\n    log.debug('embedding provider unavailable for crawl-rank', { error: String(err) });\n    embeddingAvailable = false;\n  }\n\n  if (allowedLinks.length === 0) {\n    return emptyOutput({\n      error: 'No same-host links found from seed',\n      embeddingAvailable,\n      elapsed: Date.now() - start,\n    });\n  }\n\n  // Degraded mode: no embeddings, return link order\n  if (!embeddingAvailable || !embedProvider) {\n    const degraded = degradedResults(allowedLinks, maxResults);\n    return {\n      results: degraded,\n      method: 'fts5',\n      cache_hits: 0,\n      search_hits: degraded.length,\n      embedding_available: false,\n      error: 'Embedding unavailable — returned link order',\n      total_time_ms: Date.now() - start,\n    };\n  }\n\n  // 7. Fetch + extract each link with concurrency cap\n  const fetched = await fetchPagesInChunks(\n    allowedLinks,\n    router,\n    extractor,\n    concurrency,\n    fetchTimeoutMs,\n  );\n\n  if (fetched.length === 0) {\n    return emptyOutput({\n      error: 'No linked pages could be fetched',\n      embeddingAvailable: true,\n      elapsed: Date.now() - start,\n    });\n  }\n\n  // 8. Build text per page\n  const seedText = buildPageText(seedExtraction.title, seedExtraction.markdown);\n  const pageTexts = fetched.map(p => p.text);\n\n  // 9. Embed seed + pages in one batch\n  let vectors: Float32Array[];\n  try {\n    vectors = await embedProvider.embed([seedText, ...pageTexts]);\n  } catch (err) {\n    log.warn('embedding batch failed; falling back to link order', { error: String(err) });\n    const degraded = degradedResults(allowedLinks, maxResults);\n    return {\n      results: degraded,\n      method: 'fts5',\n      cache_hits: 0,\n      search_hits: degraded.length,\n      embedding_available: true,\n      error: `Embedding batch failed: ${err instanceof Error ? err.message : String(err)}`,\n      total_time_ms: Date.now() - start,\n    };\n  }\n\n  const seedVec = vectors[0];\n  const pageVecs = vectors.slice(1);\n\n  // 10. Cosine + sort\n  const scored = fetched.map((page, i) => ({\n    page,\n    score: cosineSimilarity(seedVec, pageVecs[i]),\n  }));\n  scored.sort((a, b) => b.score - a.score);\n\n  // 11. Cap + map\n  const final: FindSimilarResult[] = scored.slice(0, maxResults).map(({ page, score }) => ({\n    url: page.url,\n    title: page.title,\n    markdown: page.markdown.slice(0, 5000),\n    relevance_score: score,\n    source: 'search',\n    match_signals: {\n      fused_score: score,\n    },\n  }));\n\n  return {\n    results: final,\n    method: 'embedding',\n    cache_hits: 0,\n    search_hits: final.length,\n    embedding_available: true,\n    total_time_ms: Date.now() - start,\n  };\n}\n\nfunction filterLinks(\n  links: string[],\n  seedHost: string,\n  includeDomains: string[] | undefined,\n  excludeDomains: string[] | undefined,\n  maxPages: number,\n): string[] {\n  const allowOtherHosts = (includeDomains?.length ?? 0) > 0;\n  const seen = new Set<string>();\n  const out: string[] = [];\n\n  for (const raw of links) {\n    let parsed: URL;\n    try {\n      parsed = new URL(raw);\n    } catch {\n      continue;\n    }\n\n    const host = parsed.hostname;\n\n    if (!allowOtherHosts && host !== seedHost) continue;\n    if (allowOtherHosts && host !== seedHost) {\n      const widened = includeDomains!.some(d => domainMatches(host, d));\n      if (!widened) continue;\n    }\n    if (excludeDomains && excludeDomains.some(d => domainMatches(host, d))) continue;\n\n    const normalized = normalize(parsed);\n    if (seen.has(normalized)) continue;\n    seen.add(normalized);\n    out.push(parsed.toString());\n    if (out.length >= maxPages) break;\n  }\n\n  return out;\n}\n\nfunction domainMatches(host: string, pattern: string): boolean {\n  const p = pattern.toLowerCase().replace(/^\\*\\./, '');\n  const h = host.toLowerCase();\n  return h === p || h.endsWith(`.${p}`);\n}\n\nfunction normalize(parsed: URL): string {\n  const copy = new URL(parsed.toString());\n  copy.hash = '';\n  return copy.toString().replace(/\\/$/, '');\n}\n\nasync function fetchPagesInChunks(\n  urls: string[],\n  router: SmartRouter,\n  extractor: Awaited<ReturnType<typeof getExtractProvider>>,\n  concurrency: number,\n  fetchTimeoutMs: number,\n): Promise<FetchedPage[]> {\n  const out: FetchedPage[] = [];\n  for (let i = 0; i < urls.length; i += concurrency) {\n    const chunk = urls.slice(i, i + concurrency);\n    const settled = await Promise.allSettled(\n      chunk.map(async u => {\n        const raw = await withTimeout(\n          router.fetch(u, { renderJs: 'auto' }),\n          fetchTimeoutMs,\n        );\n        if (\n          !raw ||\n          typeof raw.statusCode !== 'number' ||\n          raw.statusCode < 200 ||\n          raw.statusCode >= 300\n        ) {\n          throw new Error(`bad status: ${raw?.statusCode ?? 'unknown'}`);\n        }\n        const ex = await extractor.extract(raw.html, raw.finalUrl, {\n          contentType: raw.contentType,\n        });\n        return {\n          url: raw.finalUrl,\n          title: ex.title || raw.finalUrl,\n          markdown: ex.markdown ?? '',\n          text: buildPageText(ex.title, ex.markdown ?? ''),\n        } satisfies FetchedPage;\n      }),\n    );\n    for (const r of settled) {\n      if (r.status === 'fulfilled') out.push(r.value);\n      else log.debug('crawl-rank page fetch failed', { error: String(r.reason) });\n    }\n  }\n  return out;\n}\n\nfunction buildPageText(title: string, markdown: string): string {\n  const t = (title ?? '').trim();\n  const body = (markdown ?? '').slice(0, SEED_TEXT_CHARS);\n  return t ? `${t}\\n${body}` : body;\n}\n\nfunction cosineSimilarity(a: Float32Array, b: Float32Array): number {\n  if (!a || !b || a.length === 0 || b.length === 0) return 0;\n  const len = Math.min(a.length, b.length);\n  let dot = 0;\n  let normA = 0;\n  let normB = 0;\n  for (let i = 0; i < len; i++) {\n    dot += a[i] * b[i];\n    normA += a[i] * a[i];\n    normB += b[i] * b[i];\n  }\n  if (normA === 0 || normB === 0) return 0;\n  return dot / (Math.sqrt(normA) * Math.sqrt(normB));\n}\n\nfunction degradedResults(links: string[], maxResults: number): FindSimilarResult[] {\n  const limited = links.slice(0, maxResults);\n  const n = limited.length;\n  return limited.map((url, i) => ({\n    url,\n    title: url,\n    markdown: '',\n    relevance_score: n > 0 ? 1 - i / n : 0,\n    source: 'search',\n    match_signals: {\n      fused_score: n > 0 ? 1 - i / n : 0,\n    },\n  }));\n}\n\nfunction withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {\n  return new Promise<T>((resolve, reject) => {\n    const t = setTimeout(() => reject(new Error(`timeout after ${ms}ms`)), ms);\n    p.then(\n      v => {\n        clearTimeout(t);\n        resolve(v);\n      },\n      e => {\n        clearTimeout(t);\n        reject(e);\n      },\n    );\n  });\n}\n\nfunction emptyOutput(args: {\n  error: string;\n  embeddingAvailable: boolean;\n  elapsed: number;\n}): FindSimilarOutput {\n  return {\n    results: [],\n    method: 'fts5',\n    cache_hits: 0,\n    search_hits: 0,\n    embedding_available: args.embeddingAvailable,\n    error: args.error,\n    total_time_ms: args.elapsed,\n  };\n}\n"],"mappings":"AAMA,SAAS,0BAA0B;AACnC,SAAS,wBAAwB;AACjC,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,QAAQ;AAEjC,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAC5B,MAAM,2BAA2B;AACjC,MAAM,sBAAsB;AAC5B,MAAM,kBAAkB;AAexB,eAAsB,UACpB,SACA,OACA,QACA,UAA4B,CAAC,GACD;AAC5B,QAAM,QAAQ,KAAK,IAAI;AACvB,QAAM,WAAW,QAAQ,YAAY;AACrC,QAAM,cAAc,QAAQ,eAAe;AAC3C,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,aAAa,MAAM,eAAe;AAExC,MAAI;AACJ,MAAI;AACF,eAAW,IAAI,IAAI,OAAO,EAAE;AAAA,EAC9B,QAAQ;AACN,WAAO,YAAY;AAAA,MACjB,OAAO;AAAA,MACP,oBAAoB;AAAA,MACpB,SAAS,KAAK,IAAI,IAAI;AAAA,IACxB,CAAC;AAAA,EACH;AAGA,MAAI;AACJ,MAAI;AACF,cAAU,MAAM,OAAO,MAAM,SAAS,EAAE,UAAU,OAAO,CAAC;AAAA,EAC5D,SAAS,KAAK;AACZ,WAAO,YAAY;AAAA,MACjB,OAAO,sBAAsB,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MAC7E,oBAAoB;AAAA,MACpB,SAAS,KAAK,IAAI,IAAI;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,MAAI,CAAC,WAAW,OAAO,QAAQ,eAAe,YAAY,QAAQ,aAAa,OAAO,QAAQ,cAAc,KAAK;AAC/G,WAAO,YAAY;AAAA,MACjB,OAAO,sBAAsB,SAAS,cAAc,SAAS;AAAA,MAC7D,oBAAoB;AAAA,MACpB,SAAS,KAAK,IAAI,IAAI;AAAA,IACxB,CAAC;AAAA,EACH;AAGA,QAAM,YAAY,MAAM,mBAAmB;AAC3C,QAAM,iBAAiB,MAAM,UAAU,QAAQ,QAAQ,MAAM,QAAQ,UAAU;AAAA,IAC7E,aAAa,QAAQ;AAAA,EACvB,CAAC;AAGD,QAAM,iBAAiB,MAAM;AAC7B,QAAM,iBAAiB,MAAM;AAC7B,QAAM,eAAe;AAAA,IACnB,eAAe,SAAS,CAAC;AAAA,IACzB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAKA,MAAI;AACJ,MAAI,qBAAqB;AACzB,MAAI;AACF,oBAAgB,MAAM,iBAAiB;AACvC,yBAAqB;AAAA,EACvB,SAAS,KAAK;AACZ,QAAI,MAAM,iDAAiD,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACjF,yBAAqB;AAAA,EACvB;AAEA,MAAI,aAAa,WAAW,GAAG;AAC7B,WAAO,YAAY;AAAA,MACjB,OAAO;AAAA,MACP;AAAA,MACA,SAAS,KAAK,IAAI,IAAI;AAAA,IACxB,CAAC;AAAA,EACH;AAGA,MAAI,CAAC,sBAAsB,CAAC,eAAe;AACzC,UAAM,WAAW,gBAAgB,cAAc,UAAU;AACzD,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,YAAY;AAAA,MACZ,aAAa,SAAS;AAAA,MACtB,qBAAqB;AAAA,MACrB,OAAO;AAAA,MACP,eAAe,KAAK,IAAI,IAAI;AAAA,IAC9B;AAAA,EACF;AAGA,QAAM,UAAU,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,WAAO,YAAY;AAAA,MACjB,OAAO;AAAA,MACP,oBAAoB;AAAA,MACpB,SAAS,KAAK,IAAI,IAAI;AAAA,IACxB,CAAC;AAAA,EACH;AAGA,QAAM,WAAW,cAAc,eAAe,OAAO,eAAe,QAAQ;AAC5E,QAAM,YAAY,QAAQ,IAAI,OAAK,EAAE,IAAI;AAGzC,MAAI;AACJ,MAAI;AACF,cAAU,MAAM,cAAc,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;AAAA,EAC9D,SAAS,KAAK;AACZ,QAAI,KAAK,sDAAsD,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACrF,UAAM,WAAW,gBAAgB,cAAc,UAAU;AACzD,WAAO;AAAA,MACL,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,YAAY;AAAA,MACZ,aAAa,SAAS;AAAA,MACtB,qBAAqB;AAAA,MACrB,OAAO,2BAA2B,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MAClF,eAAe,KAAK,IAAI,IAAI;AAAA,IAC9B;AAAA,EACF;AAEA,QAAM,UAAU,QAAQ,CAAC;AACzB,QAAM,WAAW,QAAQ,MAAM,CAAC;AAGhC,QAAM,SAAS,QAAQ,IAAI,CAAC,MAAM,OAAO;AAAA,IACvC;AAAA,IACA,OAAO,iBAAiB,SAAS,SAAS,CAAC,CAAC;AAAA,EAC9C,EAAE;AACF,SAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAGvC,QAAM,QAA6B,OAAO,MAAM,GAAG,UAAU,EAAE,IAAI,CAAC,EAAE,MAAM,MAAM,OAAO;AAAA,IACvF,KAAK,KAAK;AAAA,IACV,OAAO,KAAK;AAAA,IACZ,UAAU,KAAK,SAAS,MAAM,GAAG,GAAI;AAAA,IACrC,iBAAiB;AAAA,IACjB,QAAQ;AAAA,IACR,eAAe;AAAA,MACb,aAAa;AAAA,IACf;AAAA,EACF,EAAE;AAEF,SAAO;AAAA,IACL,SAAS;AAAA,IACT,QAAQ;AAAA,IACR,YAAY;AAAA,IACZ,aAAa,MAAM;AAAA,IACnB,qBAAqB;AAAA,IACrB,eAAe,KAAK,IAAI,IAAI;AAAA,EAC9B;AACF;AAEA,SAAS,YACP,OACA,UACA,gBACA,gBACA,UACU;AACV,QAAM,mBAAmB,gBAAgB,UAAU,KAAK;AACxD,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,MAAgB,CAAC;AAEvB,aAAW,OAAO,OAAO;AACvB,QAAI;AACJ,QAAI;AACF,eAAS,IAAI,IAAI,GAAG;AAAA,IACtB,QAAQ;AACN;AAAA,IACF;AAEA,UAAM,OAAO,OAAO;AAEpB,QAAI,CAAC,mBAAmB,SAAS,SAAU;AAC3C,QAAI,mBAAmB,SAAS,UAAU;AACxC,YAAM,UAAU,eAAgB,KAAK,OAAK,cAAc,MAAM,CAAC,CAAC;AAChE,UAAI,CAAC,QAAS;AAAA,IAChB;AACA,QAAI,kBAAkB,eAAe,KAAK,OAAK,cAAc,MAAM,CAAC,CAAC,EAAG;AAExE,UAAM,aAAa,UAAU,MAAM;AACnC,QAAI,KAAK,IAAI,UAAU,EAAG;AAC1B,SAAK,IAAI,UAAU;AACnB,QAAI,KAAK,OAAO,SAAS,CAAC;AAC1B,QAAI,IAAI,UAAU,SAAU;AAAA,EAC9B;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,MAAc,SAA0B;AAC7D,QAAM,IAAI,QAAQ,YAAY,EAAE,QAAQ,SAAS,EAAE;AACnD,QAAM,IAAI,KAAK,YAAY;AAC3B,SAAO,MAAM,KAAK,EAAE,SAAS,IAAI,CAAC,EAAE;AACtC;AAEA,SAAS,UAAU,QAAqB;AACtC,QAAM,OAAO,IAAI,IAAI,OAAO,SAAS,CAAC;AACtC,OAAK,OAAO;AACZ,SAAO,KAAK,SAAS,EAAE,QAAQ,OAAO,EAAE;AAC1C;AAEA,eAAe,mBACb,MACA,QACA,WACA,aACA,gBACwB;AACxB,QAAM,MAAqB,CAAC;AAC5B,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,aAAa;AACjD,UAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,WAAW;AAC3C,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,MAAM,IAAI,OAAM,MAAK;AACnB,cAAM,MAAM,MAAM;AAAA,UAChB,OAAO,MAAM,GAAG,EAAE,UAAU,OAAO,CAAC;AAAA,UACpC;AAAA,QACF;AACA,YACE,CAAC,OACD,OAAO,IAAI,eAAe,YAC1B,IAAI,aAAa,OACjB,IAAI,cAAc,KAClB;AACA,gBAAM,IAAI,MAAM,eAAe,KAAK,cAAc,SAAS,EAAE;AAAA,QAC/D;AACA,cAAM,KAAK,MAAM,UAAU,QAAQ,IAAI,MAAM,IAAI,UAAU;AAAA,UACzD,aAAa,IAAI;AAAA,QACnB,CAAC;AACD,eAAO;AAAA,UACL,KAAK,IAAI;AAAA,UACT,OAAO,GAAG,SAAS,IAAI;AAAA,UACvB,UAAU,GAAG,YAAY;AAAA,UACzB,MAAM,cAAc,GAAG,OAAO,GAAG,YAAY,EAAE;AAAA,QACjD;AAAA,MACF,CAAC;AAAA,IACH;AACA,eAAW,KAAK,SAAS;AACvB,UAAI,EAAE,WAAW,YAAa,KAAI,KAAK,EAAE,KAAK;AAAA,UACzC,KAAI,MAAM,gCAAgC,EAAE,OAAO,OAAO,EAAE,MAAM,EAAE,CAAC;AAAA,IAC5E;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,OAAe,UAA0B;AAC9D,QAAM,KAAK,SAAS,IAAI,KAAK;AAC7B,QAAM,QAAQ,YAAY,IAAI,MAAM,GAAG,eAAe;AACtD,SAAO,IAAI,GAAG,CAAC;AAAA,EAAK,IAAI,KAAK;AAC/B;AAEA,SAAS,iBAAiB,GAAiB,GAAyB;AAClE,MAAI,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,KAAK,EAAE,WAAW,EAAG,QAAO;AACzD,QAAM,MAAM,KAAK,IAAI,EAAE,QAAQ,EAAE,MAAM;AACvC,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC5B,WAAO,EAAE,CAAC,IAAI,EAAE,CAAC;AACjB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AACnB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACrB;AACA,MAAI,UAAU,KAAK,UAAU,EAAG,QAAO;AACvC,SAAO,OAAO,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;AAClD;AAEA,SAAS,gBAAgB,OAAiB,YAAyC;AACjF,QAAM,UAAU,MAAM,MAAM,GAAG,UAAU;AACzC,QAAM,IAAI,QAAQ;AAClB,SAAO,QAAQ,IAAI,CAAC,KAAK,OAAO;AAAA,IAC9B;AAAA,IACA,OAAO;AAAA,IACP,UAAU;AAAA,IACV,iBAAiB,IAAI,IAAI,IAAI,IAAI,IAAI;AAAA,IACrC,QAAQ;AAAA,IACR,eAAe;AAAA,MACb,aAAa,IAAI,IAAI,IAAI,IAAI,IAAI;AAAA,IACnC;AAAA,EACF,EAAE;AACJ;AAEA,SAAS,YAAe,GAAe,IAAwB;AAC7D,SAAO,IAAI,QAAW,CAAC,SAAS,WAAW;AACzC,UAAM,IAAI,WAAW,MAAM,OAAO,IAAI,MAAM,iBAAiB,EAAE,IAAI,CAAC,GAAG,EAAE;AACzE,MAAE;AAAA,MACA,OAAK;AACH,qBAAa,CAAC;AACd,gBAAQ,CAAC;AAAA,MACX;AAAA,MACA,OAAK;AACH,qBAAa,CAAC;AACd,eAAO,CAAC;AAAA,MACV;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,YAAY,MAIC;AACpB,SAAO;AAAA,IACL,SAAS,CAAC;AAAA,IACV,QAAQ;AAAA,IACR,YAAY;AAAA,IACZ,aAAa;AAAA,IACb,qBAAqB,KAAK;AAAA,IAC1B,OAAO,KAAK;AAAA,IACZ,eAAe,KAAK;AAAA,EACtB;AACF;","names":[]}

package/dist/search/find-similar/mode.d.ts

@@ -0,0 +1,4 @@
+import type { FindSimilarInput } from '../../types.js';
+export type FindSimilarMode = 'auto' | 'cache' | 'web-expansion' | 'crawl-rank';
+export declare function selectMode(input: FindSimilarInput): FindSimilarMode;
+//# sourceMappingURL=mode.d.ts.map

package/dist/search/find-similar/mode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mode.d.ts","sourceRoot":"","sources":["../../../src/search/find-similar/mode.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,GAAG,YAAY,CAAC;AAMhF,wBAAgB,UAAU,CAAC,KAAK,EAAE,gBAAgB,GAAG,eAAe,CAOnE"}

package/dist/search/find-similar/mode.js

@@ -0,0 +1,12 @@
+function selectMode(input) {
+  if (input.mode === "crawl-rank") {
+    if (input.url && input.url.trim().length > 0) return "crawl-rank";
+    return "cache";
+  }
+  if (input.mode === "cache" || input.mode === "web-expansion") return input.mode;
+  return "cache";
+}
+export {
+  selectMode
+};
+//# sourceMappingURL=mode.js.map

package/dist/search/find-similar/mode.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/search/find-similar/mode.ts"],"sourcesContent":["import type { FindSimilarInput } from '../../types.js';\n\nexport type FindSimilarMode = 'auto' | 'cache' | 'web-expansion' | 'crawl-rank';\n\n// Dispatcher only branches on 'crawl-rank' today. 'cache' / 'web-expansion' /\n// 'auto' all fall through to the legacy hybrid flow in find-similar.ts so\n// existing callers keep their behavior. Crawl-rank requires an explicit\n// opt-in and a seed URL — concept-only inputs have nothing to crawl from.\nexport function selectMode(input: FindSimilarInput): FindSimilarMode {\n  if (input.mode === 'crawl-rank') {\n    if (input.url && input.url.trim().length > 0) return 'crawl-rank';\n    return 'cache';\n  }\n  if (input.mode === 'cache' || input.mode === 'web-expansion') return input.mode;\n  return 'cache';\n}\n"],"mappings":"AAQO,SAAS,WAAW,OAA0C;AACnE,MAAI,MAAM,SAAS,cAAc;AAC/B,QAAI,MAAM,OAAO,MAAM,IAAI,KAAK,EAAE,SAAS,EAAG,QAAO;AACrD,WAAO;AAAA,EACT;AACA,MAAI,MAAM,SAAS,WAAW,MAAM,SAAS,gBAAiB,QAAO,MAAM;AAC3E,SAAO;AACT;","names":[]}

package/dist/search/find-similar.d.ts

@@ -0,0 +1,5 @@
+import type { FindSimilarInput, FindSimilarOutput, SearchEngine } from '../types.js';
+import type { SmartRouter } from '../fetch/router.js';
+import type { BackendStatus } from '../server/backend-status.js';
+export declare function findSimilar(input: FindSimilarInput, engines: SearchEngine[], router: SmartRouter, backendStatus?: BackendStatus): Promise<FindSimilarOutput>;
+//# sourceMappingURL=find-similar.d.ts.map

package/dist/search/find-similar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"find-similar.d.ts","sourceRoot":"","sources":["../../src/search/find-similar.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EAEb,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AA2BjE,wBAAsB,WAAW,CAC/B,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CA0M5B"}