@bpinternal/site-scout 0.1.0

package/dist/index.cjs

@@ -0,0 +1,1797 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  cachedDeps: () => cachedDeps,
+  discover: () => discover,
+  fetchJson: () => fetchJson,
+  fetchText: () => fetchText,
+  findKnowledgeUrls: () => findKnowledgeUrls,
+  hostOf: () => hostOf,
+  select: () => select,
+  siteOrigin: () => siteOrigin
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/web-fetch.ts
+var import_promises = require("dns/promises");
+var FETCH_TIMEOUT_MS = 6e3;
+var DNS_TIMEOUT_MS = 2e3;
+var MAX_REDIRECTS = 3;
+var MAX_RESPONSE_BYTES = 2e6;
+var SITE_FETCH_USER_AGENT = "BotpressVibe/1.0 (+https://botpress.com) site-analyzer";
+async function readBodyCapped(res) {
+  const reader = res.body?.getReader?.();
+  if (!reader) {
+    const text2 = await res.text();
+    return text2.length > MAX_RESPONSE_BYTES ? null : text2;
+  }
+  const decoder = new TextDecoder();
+  let out = "";
+  let bytes = 0;
+  try {
+    for (; ; ) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value?.byteLength ?? 0;
+      if (bytes > MAX_RESPONSE_BYTES) {
+        try {
+          await reader.cancel();
+        } catch {
+        }
+        return null;
+      }
+      out += decoder.decode(value, { stream: true });
+    }
+    out += decoder.decode();
+    return out;
+  } catch {
+    return null;
+  }
+}
+__name(readBodyCapped, "readBodyCapped");
+async function fetchText(url, baseHost, timeoutMs = FETCH_TIMEOUT_MS) {
+  const res = await safeFetch(url, baseHost, timeoutMs);
+  if (!res) return null;
+  return readBodyCapped(res);
+}
+__name(fetchText, "fetchText");
+async function fetchJson(url, baseHost, timeoutMs = FETCH_TIMEOUT_MS) {
+  const res = await safeFetch(url, baseHost, timeoutMs, "application/json");
+  if (!res) return null;
+  const text2 = await readBodyCapped(res);
+  if (text2 == null) return null;
+  try {
+    return JSON.parse(text2);
+  } catch {
+    return null;
+  }
+}
+__name(fetchJson, "fetchJson");
+async function safeFetch(url, baseHost, timeoutMs, accept) {
+  if (typeof fetch !== "function") return null;
+  let target = url;
+  for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
+    if (!isFetchableSameSite(target, baseHost)) return null;
+    const host = hostOf(target);
+    if (!host || !await hostResolvesToAllowed(host)) return null;
+    const ctl = new AbortController();
+    const timer = setTimeout(() => ctl.abort(), timeoutMs);
+    let res;
+    try {
+      res = await fetch(target, {
+        signal: ctl.signal,
+        headers: { "user-agent": SITE_FETCH_USER_AGENT, ...accept ? { accept } : {} },
+        redirect: "manual"
+      });
+    } catch {
+      clearTimeout(timer);
+      return null;
+    }
+    clearTimeout(timer);
+    const status = typeof res.status === "number" ? res.status : 200;
+    if (status >= 300 && status < 400) {
+      const loc = res.headers?.get?.("location");
+      if (!loc) return null;
+      try {
+        target = new URL(loc, target).toString();
+      } catch {
+        return null;
+      }
+      continue;
+    }
+    if (!res.ok) return null;
+    const len = Number(res.headers?.get?.("content-length") ?? "");
+    if (Number.isFinite(len) && len > MAX_RESPONSE_BYTES) return null;
+    return res;
+  }
+  return null;
+}
+__name(safeFetch, "safeFetch");
+function siteOrigin(baseUrl) {
+  try {
+    return new URL(baseUrl).origin;
+  } catch {
+    try {
+      return new URL(`https://${baseUrl}`).origin;
+    } catch {
+      return null;
+    }
+  }
+}
+__name(siteOrigin, "siteOrigin");
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase() || null;
+  } catch {
+    try {
+      return new URL(`https://${url}`).hostname.toLowerCase() || null;
+    } catch {
+      return null;
+    }
+  }
+}
+__name(hostOf, "hostOf");
+function registrableDomain(host) {
+  return host.split(".").slice(-2).join(".");
+}
+__name(registrableDomain, "registrableDomain");
+function isIpLiteral(host) {
+  if (host.includes(":")) return true;
+  return /^\d{1,3}(\.\d{1,3}){3}$/.test(host);
+}
+__name(isIpLiteral, "isIpLiteral");
+function isFetchableSameSite(url, baseHost) {
+  let u;
+  try {
+    u = new URL(url);
+  } catch {
+    return false;
+  }
+  if (u.protocol !== "http:" && u.protocol !== "https:") return false;
+  const host = u.hostname.toLowerCase();
+  if (!host || host === "localhost" || !host.includes(".") || isIpLiteral(host)) return false;
+  return registrableDomain(host) === registrableDomain(baseHost);
+}
+__name(isFetchableSameSite, "isFetchableSameSite");
+function isDisallowedIpv4(ip) {
+  const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(ip);
+  if (!m) return false;
+  const a = Number(m[1]);
+  const b = Number(m[2]);
+  const c = Number(m[3]);
+  if (a === 0 || a === 10 || a === 127) return true;
+  if (a === 169 && b === 254) return true;
+  if (a === 172 && b >= 16 && b <= 31) return true;
+  if (a === 192 && b === 168) return true;
+  if (a === 100 && b >= 64 && b <= 127) return true;
+  if (a === 192 && b === 0 && c === 0) return true;
+  if (a === 198 && (b === 18 || b === 19)) return true;
+  if (a >= 224) return true;
+  return false;
+}
+__name(isDisallowedIpv4, "isDisallowedIpv4");
+function isDisallowedIp(ip) {
+  const addr = ip.toLowerCase().replace(/^\[|\]$/g, "").replace(/%.*$/, "");
+  const mapped = /^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(addr);
+  if (mapped) return isDisallowedIpv4(mapped[1]);
+  if (addr.includes(":")) {
+    if (addr === "::1" || addr === "::") return true;
+    if (addr.startsWith("fc") || addr.startsWith("fd")) return true;
+    if (/^fe[89ab]/.test(addr)) return true;
+    if (/^fec/.test(addr)) return true;
+    if (addr.startsWith("ff")) return true;
+    return false;
+  }
+  return isDisallowedIpv4(addr);
+}
+__name(isDisallowedIp, "isDisallowedIp");
+async function hostResolvesToAllowed(host) {
+  if (isDisallowedIp(host)) return false;
+  try {
+    const records = await Promise.race([
+      (0, import_promises.lookup)(host, { all: true }),
+      new Promise((_, reject) => setTimeout(() => reject(new Error("dns timeout")), DNS_TIMEOUT_MS))
+    ]);
+    if (!records.length) return true;
+    return records.every((r) => !isDisallowedIp(r.address));
+  } catch {
+    return true;
+  }
+}
+__name(hostResolvesToAllowed, "hostResolvesToAllowed");
+
+// src/guard.ts
+function registrableDomain2(host) {
+  return host.split(".").slice(-2).join(".");
+}
+__name(registrableDomain2, "registrableDomain");
+var DENY_DOMAINS = /* @__PURE__ */ new Set([
+  // search engines
+  "google.com",
+  "google.ca",
+  "bing.com",
+  "duckduckgo.com",
+  "baidu.com",
+  "yandex.com",
+  // social networks
+  "facebook.com",
+  "instagram.com",
+  "x.com",
+  "twitter.com",
+  "reddit.com",
+  "linkedin.com",
+  "tiktok.com",
+  "pinterest.com",
+  "snapchat.com",
+  "threads.net",
+  "youtube.com",
+  "whatsapp.com",
+  "telegram.org",
+  "discord.com",
+  // knowledge / UGC platforms where a "site" isn't a company KB
+  "wikipedia.org",
+  "wikimedia.org",
+  "quora.com",
+  "tumblr.com"
+]);
+var DENY_HOSTS = /* @__PURE__ */ new Set([
+  "drive.google.com",
+  "docs.google.com",
+  "sites.google.com",
+  "maps.google.com",
+  "play.google.com",
+  "mail.google.com"
+]);
+function isUnsupportedSite(website) {
+  const host = hostOf(website);
+  if (!host) return false;
+  if (DENY_HOSTS.has(host)) return true;
+  return DENY_DOMAINS.has(registrableDomain2(host));
+}
+__name(isUnsupportedSite, "isUnsupportedSite");
+
+// src/discovery/parse-robots.ts
+var import_robots_parser = __toESM(require("robots-parser"), 1);
+function parseRobots(text2, robotsUrl = "https://example.com/robots.txt") {
+  const parser2 = (0, import_robots_parser.default)(robotsUrl, text2);
+  const sitemaps = dedupe(parser2.getSitemaps());
+  const disallow = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const rawLine of text2.split(/\r?\n/)) {
+    const line = rawLine.replace(/^/, "").trim();
+    if (!line || line.startsWith("#")) continue;
+    const colon = line.indexOf(":");
+    if (colon === -1) continue;
+    if (line.slice(0, colon).trim().toLowerCase() !== "disallow") continue;
+    let value = line.slice(colon + 1).trim();
+    const hash = value.indexOf("#");
+    if (hash !== -1) value = value.slice(0, hash).trim();
+    if (value && !seen.has(value)) {
+      seen.add(value);
+      disallow.push(value);
+    }
+  }
+  return { sitemaps, disallow };
+}
+__name(parseRobots, "parseRobots");
+function dedupe(arr) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const v of arr) {
+    if (!seen.has(v)) {
+      seen.add(v);
+      out.push(v);
+    }
+  }
+  return out;
+}
+__name(dedupe, "dedupe");
+
+// src/discovery/parse-sitemap.ts
+var import_fast_xml_parser = require("fast-xml-parser");
+var parser = new import_fast_xml_parser.XMLParser({
+  ignoreAttributes: true,
+  trimValues: true,
+  // <loc> values are returned as strings; numbers stay parseable by us.
+  parseTagValue: false
+});
+function asArray(v) {
+  if (v == null) return [];
+  return Array.isArray(v) ? v : [v];
+}
+__name(asArray, "asArray");
+function text(v) {
+  if (v == null) return void 0;
+  const s = String(v).trim();
+  return s || void 0;
+}
+__name(text, "text");
+function parseSitemap(xml) {
+  let doc;
+  try {
+    doc = parser.parse(xml);
+  } catch {
+    return { kind: "unknown", sitemaps: [], urls: [] };
+  }
+  if (doc?.sitemapindex) {
+    const seen = /* @__PURE__ */ new Set();
+    const sitemaps = [];
+    for (const sm of asArray(doc.sitemapindex.sitemap)) {
+      const loc = text(sm?.loc);
+      if (loc && !seen.has(loc)) {
+        seen.add(loc);
+        sitemaps.push(loc);
+      }
+    }
+    return { kind: "index", sitemaps, urls: [] };
+  }
+  if (doc?.urlset) {
+    const seen = /* @__PURE__ */ new Set();
+    const urls = [];
+    for (const u of asArray(doc.urlset.url)) {
+      const loc = text(u?.loc);
+      if (!loc || seen.has(loc)) continue;
+      seen.add(loc);
+      const lastmod = text(u?.lastmod);
+      const changefreq = text(u?.changefreq)?.toLowerCase();
+      const priorityRaw = text(u?.priority);
+      const priority = priorityRaw != null ? Number(priorityRaw) : void 0;
+      urls.push({
+        loc,
+        ...lastmod ? { lastmod } : {},
+        ...priority != null && Number.isFinite(priority) ? { priority } : {},
+        ...changefreq ? { changefreq } : {}
+      });
+    }
+    return { kind: "urlset", sitemaps: [], urls };
+  }
+  return { kind: "unknown", sitemaps: [], urls: [] };
+}
+__name(parseSitemap, "parseSitemap");
+
+// src/discovery/parse-llms.ts
+var MD_LINK = /\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)/g;
+function parseLlms(text2) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const m of text2.matchAll(MD_LINK)) {
+    const title = m[1]?.trim() || void 0;
+    const url = m[2].trim();
+    if (seen.has(url)) continue;
+    seen.add(url);
+    out.push({ url, ...title ? { title } : {} });
+  }
+  return out;
+}
+__name(parseLlms, "parseLlms");
+
+// src/discovery/normalize.ts
+var KEEP_EXT = /* @__PURE__ */ new Set(["html", "htm", "md", "markdown", "mdx", "pdf"]);
+var DROP_EXT = /* @__PURE__ */ new Set([
+  // machine / feeds
+  "xml",
+  "rss",
+  "atom",
+  "json",
+  "csv",
+  "txt",
+  "yaml",
+  "yml",
+  // images
+  "jpg",
+  "jpeg",
+  "png",
+  "gif",
+  "webp",
+  "svg",
+  "ico",
+  "bmp",
+  "tiff",
+  "avif",
+  // video / audio
+  "mp4",
+  "webm",
+  "mov",
+  "avi",
+  "mkv",
+  "mp3",
+  "wav",
+  "ogg",
+  "m4a",
+  // archives / binaries
+  "zip",
+  "gz",
+  "tar",
+  "rar",
+  "7z",
+  "dmg",
+  "exe",
+  "apk",
+  "pkg",
+  // fonts / code / data
+  "woff",
+  "woff2",
+  "ttf",
+  "otf",
+  "eot",
+  "js",
+  "mjs",
+  "css",
+  "map",
+  "wasm",
+  // office (non-pdf) — usually not worth scraping for a support KB
+  "doc",
+  "docx",
+  "xls",
+  "xlsx",
+  "ppt",
+  "pptx"
+]);
+var LOCALE_SEG = /^(?:[a-z]{2})(?:[-_][a-z]{2})?$/i;
+var KNOWN_LANGS = /* @__PURE__ */ new Set([
+  "en",
+  "fr",
+  "es",
+  "de",
+  "it",
+  "pt",
+  "nl",
+  "pl",
+  "ru",
+  "ja",
+  "zh",
+  "ko",
+  "ar",
+  "tr",
+  "sv",
+  "da",
+  "fi",
+  "no",
+  "cs",
+  "el",
+  "he",
+  "th",
+  "vi",
+  "id",
+  "uk",
+  "ro",
+  "hu",
+  "ms",
+  // Malay
+  "tl",
+  // Tagalog
+  "tw",
+  // locale-style variant (zh-TW shorthand on some sites)
+  "hi",
+  // Hindi
+  "bn",
+  // Bengali
+  "sk",
+  // Slovak
+  "bg",
+  // Bulgarian
+  "sr",
+  "sl",
+  "et",
+  "lv",
+  "lt",
+  "fa",
+  "ur",
+  "ta"
+  // NOTE: deliberately NOT adding 'hr' — too often means human-resources as a
+  // content section, not Croatian. Erring toward keeping content over dropping it.
+]);
+function extOf(pathname) {
+  const last = pathname.split("/").pop() ?? "";
+  const dot = last.lastIndexOf(".");
+  if (dot <= 0) return null;
+  return last.slice(dot + 1).toLowerCase();
+}
+__name(extOf, "extOf");
+function normalizeUrl(raw) {
+  let u;
+  try {
+    u = new URL(raw);
+  } catch {
+    try {
+      u = new URL(`https://${raw}`);
+    } catch {
+      return null;
+    }
+  }
+  if (u.protocol !== "http:" && u.protocol !== "https:") return null;
+  u.hash = "";
+  u.hostname = u.hostname.toLowerCase();
+  const TRACKING = /^(utm_|gclid|fbclid|mc_|ref$|ref_|_hs|igshid|si$)/i;
+  const params = [...u.searchParams.entries()].filter(([k]) => !TRACKING.test(k));
+  params.sort(([a], [b]) => a.localeCompare(b));
+  u.search = "";
+  for (const [k, v] of params) u.searchParams.append(k, v);
+  if (u.pathname.length > 1 && u.pathname.endsWith("/")) {
+    u.pathname = u.pathname.replace(/\/+$/, "");
+  }
+  return u.toString();
+}
+__name(normalizeUrl, "normalizeUrl");
+var ROUTE_TEMPLATE = /(\[[^\]]*\]|\{[^}]*\}|\/:[^/]+|\/\*(?:\/|$))/;
+function isRouteTemplate(url) {
+  try {
+    return ROUTE_TEMPLATE.test(decodeURIComponent(new URL(url).pathname));
+  } catch {
+    return ROUTE_TEMPLATE.test(url);
+  }
+}
+__name(isRouteTemplate, "isRouteTemplate");
+function isAllowedContent(url) {
+  let pathname;
+  try {
+    pathname = new URL(url).pathname;
+  } catch {
+    return false;
+  }
+  if (isRouteTemplate(url)) return false;
+  const ext = extOf(pathname);
+  if (ext == null) return true;
+  if (KEEP_EXT.has(ext)) return true;
+  if (DROP_EXT.has(ext)) return false;
+  return true;
+}
+__name(isAllowedContent, "isAllowedContent");
+function sectionOf(url) {
+  let segs;
+  try {
+    segs = new URL(url).pathname.split("/").filter(Boolean);
+  } catch {
+    return "";
+  }
+  for (const s of segs) {
+    if (isLocaleSegment(s)) continue;
+    return s.toLowerCase();
+  }
+  return "";
+}
+__name(sectionOf, "sectionOf");
+function isLocaleSegment(seg) {
+  if (!LOCALE_SEG.test(seg)) return false;
+  const lang = seg.slice(0, 2).toLowerCase();
+  return KNOWN_LANGS.has(lang);
+}
+__name(isLocaleSegment, "isLocaleSegment");
+function localeAgnosticKey(url) {
+  try {
+    const u = new URL(url);
+    const segs = u.pathname.split("/").filter(Boolean);
+    while (segs.length && isLocaleSegment(segs[0])) segs.shift();
+    return `${u.hostname}/${segs.join("/")}`;
+  } catch {
+    return url;
+  }
+}
+__name(localeAgnosticKey, "localeAgnosticKey");
+function isEnglishOrNeutral(url) {
+  try {
+    const segs = new URL(url).pathname.split("/").filter(Boolean);
+    const first = segs[0];
+    if (!first || !isLocaleSegment(first)) return true;
+    return first.slice(0, 2).toLowerCase() === "en";
+  } catch {
+    return true;
+  }
+}
+__name(isEnglishOrNeutral, "isEnglishOrNeutral");
+
+// src/discovery/detect-site-type.ts
+function matchesGlob(url, glob) {
+  let path2;
+  try {
+    path2 = new URL(url).pathname;
+  } catch {
+    return false;
+  }
+  const re = new RegExp(
+    "^" + glob.split("*").map((s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join(".*") + "$"
+  );
+  return re.test(path2);
+}
+__name(matchesGlob, "matchesGlob");
+function isShopifyProductsPayload(payload) {
+  if (!payload || typeof payload !== "object") return false;
+  const products = payload.products;
+  if (!Array.isArray(products)) return false;
+  if (products.length === 0) return true;
+  const first = products[0];
+  return first != null && typeof first === "object" && "handle" in first && typeof first.handle === "string" && "variants" in first && Array.isArray(first.variants);
+}
+__name(isShopifyProductsPayload, "isShopifyProductsPayload");
+var SHOPIFY_EXCLUDES = [
+  "/products/*",
+  // individual SKU pages
+  "*/products/*",
+  // products nested under a collection
+  "/cart*",
+  "/account*",
+  "/checkout*"
+];
+var SHOPIFY_INCLUDE_HINTS = ["/collections", "/pages", "/policies", "/pages/faq", "/pages/contact"];
+async function detectSiteType(siteUrl, baseHost, fetchJsonFn = fetchJson) {
+  const origin = originOf(siteUrl);
+  if (!origin) return unknownType();
+  const payload = await fetchJsonFn(`${origin}/products.json?limit=1`, baseHost);
+  if (isShopifyProductsPayload(payload)) {
+    return { type: "shopify", excludePatterns: [...SHOPIFY_EXCLUDES], includeHints: [...SHOPIFY_INCLUDE_HINTS] };
+  }
+  return unknownType();
+}
+__name(detectSiteType, "detectSiteType");
+function unknownType() {
+  return { type: "unknown", excludePatterns: [], includeHints: [] };
+}
+__name(unknownType, "unknownType");
+function originOf(url) {
+  try {
+    return new URL(url).origin;
+  } catch {
+    try {
+      return new URL(`https://${url}`).origin;
+    } catch {
+      return null;
+    }
+  }
+}
+__name(originOf, "originOf");
+function applyExcludes(urls, patterns) {
+  if (patterns.length === 0) return urls;
+  return urls.filter((u) => !patterns.some((p) => matchesGlob(u, p)));
+}
+__name(applyExcludes, "applyExcludes");
+
+// src/discovery/score.ts
+var TOP = [
+  "docs",
+  "doc",
+  "documentation",
+  "help",
+  "support",
+  "faq",
+  "faqs",
+  "guide",
+  "guides",
+  "kb",
+  "knowledge-base",
+  "knowledgebase",
+  "pricing",
+  "plans",
+  "integrations",
+  "integration",
+  "api",
+  "apis",
+  "features",
+  "feature",
+  "getting-started",
+  "get-started",
+  "how-to",
+  "howto",
+  "tutorial",
+  "tutorials",
+  "contact",
+  "contact-us"
+];
+var MID = [
+  "product",
+  "products",
+  "solutions",
+  "solution",
+  "use-cases",
+  "use-case",
+  "about",
+  "about-us",
+  "security",
+  "company",
+  "platform",
+  "services",
+  "service"
+];
+var LOW = [
+  "blog",
+  "news",
+  "press",
+  "newsroom",
+  "events",
+  "event",
+  "case-studies",
+  "case-study",
+  "customers",
+  "webinar",
+  "webinars",
+  "resources",
+  "resource",
+  "library",
+  "podcast",
+  "stories"
+];
+var DROP = [
+  "tag",
+  "tags",
+  "author",
+  "authors",
+  "category",
+  "categories",
+  "archive",
+  "archives",
+  "login",
+  "signin",
+  "sign-in",
+  "account",
+  "cart",
+  "checkout",
+  "register",
+  "signup",
+  "sign-up",
+  "wishlist",
+  "search"
+];
+function tierWeights() {
+  const m = {};
+  for (const s of TOP) m[s] = 10;
+  for (const s of MID) m[s] = 5;
+  for (const s of LOW) m[s] = 1.5;
+  for (const s of DROP) m[s] = -8;
+  return m;
+}
+__name(tierWeights, "tierWeights");
+var DEFAULT_WEIGHTS = {
+  sectionWeights: tierWeights(),
+  defaultSectionWeight: 3,
+  // unknown section: neutral-positive (homepage-ish)
+  depthPenalty: 0.6,
+  depthWaivedSections: ["docs", "doc", "documentation", "help", "support", "kb", "guide", "guides", "api"],
+  recencyBonus: 3,
+  priorityBonus: 2,
+  multiSourceBonus: 1.5,
+  llmsBonus: 8,
+  queryParamPenalty: 1.5
+};
+var DATED_PATH = /\/(?:19|20)\d{2}(?:[/-](?:0?[1-9]|1[0-2]))?(?:[/-]\d{1,2})?(?=\/|$)/;
+var SITEMAP_BOOST = ["help", "support", "docs", "doc", "kb", "faq", "guide", "api", "pricing", "product-doc"];
+var SITEMAP_PENALTY = ["pdp", "product", "gridwall", "snkrs", "locator", "article", "blog", "news", "image", "video"];
+function scoreSitemapUrl(sitemapUrl) {
+  const lower = sitemapUrl.toLowerCase();
+  let score = 1;
+  for (const k of SITEMAP_BOOST) if (lower.includes(k)) score += 6;
+  for (const k of SITEMAP_PENALTY) if (lower.includes(k)) score -= 5;
+  if (/\/sitemap(-index)?\.xml$/.test(lower)) score += 2;
+  return score;
+}
+__name(scoreSitemapUrl, "scoreSitemapUrl");
+function pathSegments(url) {
+  try {
+    return new URL(url).pathname.split("/").filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+__name(pathSegments, "pathSegments");
+var HIGH_VALUE_SUB_TOKENS = /* @__PURE__ */ new Set([
+  "help",
+  "support",
+  "docs",
+  "doc",
+  "kb",
+  "faq",
+  "knowledgebase",
+  "developer",
+  "developers"
+]);
+function highValueSubdomain(url) {
+  let host;
+  try {
+    host = new URL(url).hostname.toLowerCase();
+  } catch {
+    return null;
+  }
+  const label = host.split(".")[0];
+  if (!label) return null;
+  for (const tok of label.split(/[-_]/)) {
+    if (HIGH_VALUE_SUB_TOKENS.has(tok)) return tok;
+  }
+  return null;
+}
+__name(highValueSubdomain, "highValueSubdomain");
+function recencyScore(lastmod, now, bonus) {
+  if (!lastmod) return 0;
+  const t = Date.parse(lastmod);
+  if (!Number.isFinite(t)) return 0;
+  const years = (now - t) / (365.25 * 24 * 3600 * 1e3);
+  if (years <= 0) return bonus;
+  return Math.max(0, bonus * (1 - years / 2));
+}
+__name(recencyScore, "recencyScore");
+function scoreUrl(url, meta = { sources: [] }, weights = DEFAULT_WEIGHTS, now = Date.parse("2026-06-30")) {
+  const reasons = [];
+  let score = 0;
+  const section = sectionOf(url);
+  const segs = pathSegments(url);
+  if (section === "") {
+    score += 6;
+    reasons.push("homepage (+6)");
+  } else if (section in weights.sectionWeights) {
+    const w = weights.sectionWeights[section];
+    score += w;
+    reasons.push(`section:${section} (${w >= 0 ? "+" : ""}${w})`);
+  } else {
+    score += weights.defaultSectionWeight;
+    reasons.push(`section:${section} default (+${weights.defaultSectionWeight})`);
+  }
+  const sub = highValueSubdomain(url);
+  if (sub) {
+    score += 8;
+    reasons.push(`subdomain:${sub} (+8)`);
+  }
+  for (const seg of segs) {
+    if (DROP.includes(seg.toLowerCase()) && seg.toLowerCase() !== section) {
+      score += -4;
+      reasons.push(`drop-segment:${seg} (-4)`);
+      break;
+    }
+  }
+  const depth = Math.max(0, segs.length - 1);
+  if (depth > 0) {
+    const waived = weights.depthWaivedSections.includes(section);
+    const perLevel = waived ? weights.depthPenalty * 0.25 : weights.depthPenalty;
+    const pen = -(perLevel * depth);
+    if (pen !== 0) {
+      score += pen;
+      reasons.push(`depth:${depth}${waived ? " (waived section)" : ""} (${pen.toFixed(2)})`);
+    }
+  }
+  try {
+    if (DATED_PATH.test(new URL(url).pathname)) {
+      score += -3;
+      reasons.push("dated-path (-3)");
+    }
+  } catch {
+  }
+  const rec = recencyScore(meta.lastmod, now, weights.recencyBonus);
+  if (rec > 0) {
+    score += rec;
+    reasons.push(`recency (+${rec.toFixed(2)})`);
+  }
+  if (typeof meta.priority === "number") {
+    const p = weights.priorityBonus * meta.priority;
+    score += p;
+    reasons.push(`priority:${meta.priority} (+${p.toFixed(2)})`);
+  }
+  if (meta.sources.includes("llms")) {
+    score += weights.llmsBonus;
+    reasons.push(`llms.txt curated (+${weights.llmsBonus})`);
+  }
+  const extra = meta.sources.length - 1;
+  if (extra > 0) {
+    const b = weights.multiSourceBonus * extra;
+    score += b;
+    reasons.push(`multi-source x${meta.sources.length} (+${b.toFixed(2)})`);
+  }
+  try {
+    const q = new URL(url).searchParams;
+    const n = [...q.keys()].length;
+    if (n > 0) {
+      const pen = -(weights.queryParamPenalty * n);
+      score += pen;
+      reasons.push(`query-params:${n} (${pen.toFixed(2)})`);
+    }
+  } catch {
+  }
+  return { url, score, reasons, section, meta };
+}
+__name(scoreUrl, "scoreUrl");
+
+// src/discovery/build-tree.ts
+function newNode(segment) {
+  return { segment, children: [], childMap: /* @__PURE__ */ new Map(), score: -Infinity, count: 0 };
+}
+__name(newNode, "newNode");
+function buildTree(scored) {
+  const root = newNode("");
+  for (const item of scored) {
+    let segs;
+    try {
+      segs = new URL(item.url).pathname.split("/").filter(Boolean);
+    } catch {
+      continue;
+    }
+    if (segs.length > 1 && isLocaleSegment(segs[0])) segs = segs.slice(1);
+    let node = root;
+    for (const seg of segs) {
+      let child = node.childMap.get(seg);
+      if (!child) {
+        child = newNode(seg);
+        node.childMap.set(seg, child);
+        node.children.push(child);
+      }
+      node = child;
+    }
+    if (node.url == null || item.score > node.score) {
+      node.url = item.url;
+      node.meta = item.meta;
+      node.reasons = item.reasons;
+    }
+    if (item.score > node.score) node.score = item.score;
+  }
+  finalize(root);
+  return strip(root);
+}
+__name(buildTree, "buildTree");
+function finalize(node) {
+  let count = node.url ? 1 : 0;
+  let maxScore = node.url ? node.score : -Infinity;
+  for (const child of node.children) {
+    finalize(child);
+    count += child.count;
+    if (child.score > maxScore) maxScore = child.score;
+  }
+  node.count = count;
+  if (!Number.isFinite(node.score) || maxScore > node.score) node.score = maxScore;
+  if (!Number.isFinite(node.score)) node.score = 0;
+  node.children.sort((a, b) => b.score - a.score || a.segment.localeCompare(b.segment));
+}
+__name(finalize, "finalize");
+function strip(node) {
+  return {
+    segment: node.segment,
+    ...node.url ? { url: node.url } : {},
+    ...node.meta ? { meta: node.meta } : {},
+    ...node.reasons ? { reasons: node.reasons } : {},
+    score: node.score,
+    count: node.count,
+    children: node.children.map((c) => strip(c))
+  };
+}
+__name(strip, "strip");
+function* leafUrls(node) {
+  if (node.url) yield node;
+  for (const c of node.children) yield* leafUrls(c);
+}
+__name(leafUrls, "leafUrls");
+
+// src/discovery/discover.ts
+var DEFAULTS = {
+  budgetMs: 12e3,
+  maxUrls: 1e5,
+  maxSitemapFanout: 4,
+  // extra discoverUrls calls on high-value sitemaps
+  maxChildSitemaps: 12,
+  // child sitemaps to parse for metadata
+  enoughUrls: 200,
+  // re-discover until at least this many (or rounds exhausted)
+  maxRounds: 3
+};
+async function discover(website, opts = {}, deps) {
+  const d = deps;
+  const host = hostOf(website);
+  if (!host) {
+    return emptyMap("unknown");
+  }
+  const origin = siteOrigin(website) ?? `https://${host}`;
+  const cfg = { ...DEFAULTS, ...stripUndefined(opts) };
+  const deadline = Date.now() + cfg.budgetMs;
+  const aborted = /* @__PURE__ */ __name(() => opts.signal?.aborted === true || Date.now() >= deadline, "aborted");
+  const remaining = /* @__PURE__ */ __name(() => Math.max(500, deadline - Date.now()), "remaining");
+  const steps = [];
+  const log = /* @__PURE__ */ __name((s) => {
+    steps.push(s);
+    opts.onActivity?.(s);
+  }, "log");
+  const byUrl = /* @__PURE__ */ new Map();
+  const add = /* @__PURE__ */ __name((rawUrl, source, extra = {}) => {
+    if (byUrl.size >= cfg.maxUrls) return;
+    const norm = normalizeUrl(rawUrl);
+    if (!norm || hostOf(norm) == null) return;
+    if (isNonEnglishLocalePath(norm)) return;
+    const m = byUrl.get(norm);
+    if (m) {
+      if (!m.sources.includes(source)) m.sources.push(source);
+      if (extra.lastmod && !m.lastmod) m.lastmod = extra.lastmod;
+      if (extra.priority != null && m.priority == null) m.priority = extra.priority;
+      if (extra.changefreq && !m.changefreq) m.changefreq = extra.changefreq;
+      if (extra.title && !m.title) m.title = extra.title;
+      if (extra.searchRank != null && m.searchRank == null) m.searchRank = extra.searchRank;
+    } else {
+      byUrl.set(norm, { sources: [source], ...extra });
+    }
+  }, "add");
+  let sitemapUrls = [];
+  if (!aborted()) {
+    const robotsTxt = await d.fetchText(`${origin}/robots.txt`, host, remaining()).catch(() => null);
+    const robots = robotsTxt ? parseRobots(robotsTxt, `${origin}/robots.txt`) : { sitemaps: [], disallow: [] };
+    sitemapUrls = robots.sitemaps.length ? robots.sitemaps : [`${origin}/sitemap.xml`];
+    log({
+      kind: "robots",
+      detail: robotsTxt ? `${robots.sitemaps.length} sitemap(s) declared` : "no robots.txt",
+      count: robots.sitemaps.length
+    });
+  }
+  const rankedSitemaps = rankSitemaps(sitemapUrls);
+  const fanoutTargets = rankedSitemaps.slice(0, cfg.maxSitemapFanout);
+  const branches = [];
+  branches.push(
+    discoverUrlsWithRetry(d, host).then((r) => {
+      r.urls.forEach((u) => add(u, "discoverUrls"));
+      log({ kind: "discoverUrls", detail: `domain ${host}`, count: r.urls.length });
+    }).catch(() => log({ kind: "discoverUrls", detail: `domain ${host} failed`, count: 0 }))
+  );
+  for (const sm of fanoutTargets) {
+    if (aborted()) break;
+    branches.push(
+      d.discoverUrls({ url: sm, count: 1e3, onlyHttps: true }).then((r) => {
+        r.urls.forEach((u) => add(u, "discoverUrls"));
+        log({ kind: "discoverUrls", detail: `sitemap ${sm}`, count: r.urls.length });
+      }).catch(() => log({ kind: "discoverUrls", detail: `sitemap ${sm} failed`, count: 0 }))
+    );
+  }
+  branches.push(
+    enrichFromSitemaps(rankedSitemaps.slice(0, 5), host, d, add, log, {
+      maxChildren: cfg.maxChildSitemaps,
+      remaining,
+      aborted
+    })
+  );
+  branches.push(
+    Promise.all([
+      d.fetchText(`${origin}/llms.txt`, host, remaining()).catch(() => null),
+      d.fetchText(`${origin}/llms-full.txt`, host, remaining()).catch(() => null)
+    ]).then(([a, b]) => {
+      let n = 0;
+      for (const text2 of [a, b]) {
+        if (!text2) continue;
+        for (const link of parseLlms(text2)) {
+          add(link.url, "llms", link.title ? { title: link.title } : {});
+          n++;
+        }
+      }
+      if (n) log({ kind: "llms", detail: "llms.txt curated links", count: n });
+    })
+  );
+  const queries = buildSearchQueries(host, opts.topics ?? []);
+  branches.push(
+    Promise.all(
+      queries.map(
+        (query) => d.webSearch({ query, includeSites: [host], count: 15 }).then((r) => {
+          r.results.forEach((res, i) => add(res.url, "search", { searchRank: i + 1 }));
+          log({ kind: "search", detail: query, count: r.results.length });
+          return r.results;
+        }).catch(() => [])
+      )
+    )
+  );
+  const siteTypeP = detectSiteType(origin, host, d.fetchJson).then((st) => {
+    if (st.type !== "unknown") log({ kind: "site-type", detail: st.type });
+    return st;
+  }).catch(() => ({ type: "unknown", excludePatterns: [], includeHints: [] }));
+  await Promise.allSettled(branches);
+  const siteType = await siteTypeP;
+  let round = 1;
+  while (byUrl.size < cfg.enoughUrls && round < cfg.maxRounds && !aborted()) {
+    round++;
+    const subdomainTargets = pickRediscoverTargets(byUrl, host);
+    if (subdomainTargets.length === 0) break;
+    log({ kind: "note", detail: `re-discover round ${round}: ${subdomainTargets.length} target(s)` });
+    await Promise.allSettled(
+      subdomainTargets.map(
+        (t) => d.discoverUrls({ url: t, count: 1e3, onlyHttps: true }).then((r) => {
+          r.urls.forEach((u) => add(u, "discoverUrls"));
+          log({ kind: "discoverUrls", detail: `re-discover ${t}`, count: r.urls.length });
+        }).catch(() => {
+        })
+      )
+    );
+  }
+  const allowedDomains = computeAllowedDomains(byUrl, host);
+  log({ kind: "note", detail: `same-site domains: ${[...allowedDomains].join(", ")}` });
+  let entries = [...byUrl.entries()].filter(([u]) => {
+    const h = hostOf(u);
+    return h != null && allowedDomains.has(registrableDomain3(h)) && isAllowedContent(u);
+  });
+  if (siteType.excludePatterns.length) {
+    const before = entries.length;
+    const keep = new Set(
+      applyExcludes(
+        entries.map(([u]) => u),
+        siteType.excludePatterns
+      )
+    );
+    entries = entries.filter(([u]) => keep.has(u));
+    log({ kind: "site-type", detail: `${siteType.type} excludes`, count: before - entries.length });
+  }
+  {
+    const before = entries.length;
+    entries = dedupeLocales(entries);
+    if (before !== entries.length) log({ kind: "note", detail: "locale-dedup", count: before - entries.length });
+  }
+  const scored = entries.map(([url, meta]) => scoreUrl(url, meta, void 0, opts.now));
+  const tree = buildTree(scored);
+  return {
+    tree,
+    total: scored.length,
+    steps,
+    aborted: aborted() && byUrl.size < cfg.enoughUrls,
+    siteType: siteType.type
+  };
+}
+__name(discover, "discover");
+function emptyMap(siteType) {
+  return { tree: { segment: "", children: [], score: 0, count: 0 }, total: 0, steps: [], aborted: false, siteType };
+}
+__name(emptyMap, "emptyMap");
+function stripUndefined(o) {
+  return Object.fromEntries(Object.entries(o).filter(([, v]) => v !== void 0));
+}
+__name(stripUndefined, "stripUndefined");
+function rankSitemaps(sitemaps) {
+  return sitemaps.filter((u) => {
+    const seg = firstSegment(u);
+    return !(seg && isLocaleSegment(seg));
+  }).map((u) => ({ u, s: scoreSitemapUrl(u) })).sort((a, b) => b.s - a.s).map((x) => x.u);
+}
+__name(rankSitemaps, "rankSitemaps");
+function firstSegment(url) {
+  try {
+    return new URL(url).pathname.split("/").filter(Boolean)[0];
+  } catch {
+    return void 0;
+  }
+}
+__name(firstSegment, "firstSegment");
+function registrableDomain3(host) {
+  return host.split(".").slice(-2).join(".");
+}
+__name(registrableDomain3, "registrableDomain");
+function computeAllowedDomains(byUrl, inputHost) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const url of byUrl.keys()) {
+    const h = hostOf(url);
+    if (!h) continue;
+    const d = registrableDomain3(h);
+    counts.set(d, (counts.get(d) ?? 0) + 1);
+  }
+  const allowed = /* @__PURE__ */ new Set([registrableDomain3(inputHost)]);
+  let top = null;
+  let topN = 0;
+  for (const [d, n] of counts) {
+    if (n > topN) {
+      topN = n;
+      top = d;
+    }
+  }
+  if (top) allowed.add(top);
+  return allowed;
+}
+__name(computeAllowedDomains, "computeAllowedDomains");
+function isNonEnglishLocalePath(url) {
+  const seg = firstSegment(url);
+  if (!seg || !isLocaleSegment(seg)) return false;
+  return !isEnglishOrNeutral(url);
+}
+__name(isNonEnglishLocalePath, "isNonEnglishLocalePath");
+async function discoverUrlsWithRetry(d, host) {
+  const call = /* @__PURE__ */ __name(() => d.discoverUrls({ url: host, count: 1e3, onlyHttps: true }), "call");
+  try {
+    const r = await call();
+    if (r.urls.length > 0) return r;
+  } catch {
+  }
+  return call().catch(() => ({ urls: [], stopReason: "error" }));
+}
+__name(discoverUrlsWithRetry, "discoverUrlsWithRetry");
+function localePrefix(url) {
+  const seg = firstSegment(url);
+  return seg && isLocaleSegment(seg) ? seg.toLowerCase() : "";
+}
+__name(localePrefix, "localePrefix");
+function dedupeLocales(entries) {
+  const best = /* @__PURE__ */ new Map();
+  const rank = /* @__PURE__ */ __name((url) => {
+    const p = localePrefix(url);
+    if (p === "") return 0;
+    if (p.slice(0, 2) === "en") return 1;
+    return 2;
+  }, "rank");
+  for (const entry of entries) {
+    const key = localeAgnosticKey(entry[0]);
+    const cur = best.get(key);
+    if (!cur) {
+      best.set(key, entry);
+      continue;
+    }
+    const better = rank(entry[0]) < rank(cur[0]) || rank(entry[0]) === rank(cur[0]) && entry[1].sources.length > cur[1].sources.length;
+    if (better) best.set(key, entry);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const entry of entries) {
+    const key = localeAgnosticKey(entry[0]);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(best.get(key));
+  }
+  return out;
+}
+__name(dedupeLocales, "dedupeLocales");
+function buildSearchQueries(host, topics) {
+  const queries = [`${host} help support documentation`];
+  const topic = topics.find((t) => t.trim().length > 0);
+  if (topic) queries.push(`${host} ${topic.slice(0, 40)}`);
+  return queries;
+}
+__name(buildSearchQueries, "buildSearchQueries");
+function pickRediscoverTargets(byUrl, host) {
+  const baseDomain = host.split(".").slice(-2).join(".");
+  const subroots = /* @__PURE__ */ new Set();
+  for (const url of byUrl.keys()) {
+    const h = hostOf(url);
+    if (!h || h === host) continue;
+    if (!h.endsWith(baseDomain)) continue;
+    if (/^(help|support|docs|kb|developer|developers)\./.test(h)) {
+      subroots.add(`https://${h}`);
+    }
+  }
+  return [...subroots].slice(0, 3);
+}
+__name(pickRediscoverTargets, "pickRediscoverTargets");
+async function enrichFromSitemaps(seeds, host, deps, add, log, cfg) {
+  let childBudget = cfg.maxChildren;
+  const seen = /* @__PURE__ */ new Set();
+  const fetchAndParse = /* @__PURE__ */ __name(async (url, depth = 0) => {
+    if (seen.has(url) || cfg.aborted() || depth > 3) return;
+    seen.add(url);
+    const xml = await deps.fetchText(url, host, cfg.remaining()).catch(() => null);
+    if (!xml) return;
+    const redirect = htmlSitemapRedirect(xml, url);
+    if (redirect && !seen.has(redirect)) {
+      log({ kind: "redirect", detail: `${url} \u2192 ${redirect}` });
+      await fetchAndParse(redirect, depth + 1);
+      return;
+    }
+    const parsed = parseSitemap(xml);
+    if (parsed.kind === "index") {
+      const children = prioritizeSitemapChildren(parsed.sitemaps).slice(0, Math.max(0, childBudget));
+      childBudget -= children.length;
+      log({ kind: "sitemap-index", detail: url, count: parsed.sitemaps.length });
+      await Promise.allSettled(children.map((c) => fetchAndParse(c, depth + 1)));
+    } else if (parsed.kind === "urlset") {
+      for (const e of parsed.urls) {
+        add(e.loc, "sitemap", {
+          ...e.lastmod ? { lastmod: e.lastmod } : {},
+          ...e.priority != null ? { priority: e.priority } : {},
+          ...e.changefreq ? { changefreq: e.changefreq } : {}
+        });
+      }
+      log({ kind: "sitemap", detail: url, count: parsed.urls.length });
+    }
+  }, "fetchAndParse");
+  await Promise.allSettled(seeds.map((s) => fetchAndParse(s, 0)));
+}
+__name(enrichFromSitemaps, "enrichFromSitemaps");
+function htmlSitemapRedirect(body, base) {
+  const head = body.slice(0, 1e3);
+  if (!/^\s*<!doctype html|^\s*<html|<meta/i.test(head)) return null;
+  const refresh = /http-equiv=["']?refresh["']?[^>]*content=["'][^"']*url=([^"'>\s]+)/i.exec(head);
+  const canonical = /<link[^>]+rel=["']?canonical["']?[^>]*href=["']([^"']+)["']/i.exec(head);
+  const target = refresh?.[1] ?? canonical?.[1];
+  if (!target) return null;
+  try {
+    const abs = new URL(target, base).toString();
+    return /sitemap/i.test(abs) ? abs : null;
+  } catch {
+    return null;
+  }
+}
+__name(htmlSitemapRedirect, "htmlSitemapRedirect");
+function prioritizeSitemapChildren(children) {
+  const neutral = [];
+  const english = [];
+  for (const url of children) {
+    const seg = firstSegment(url);
+    if (seg && isLocaleSegment(seg)) {
+      if (seg.slice(0, 2).toLowerCase() === "en") english.push(url);
+    } else {
+      neutral.push(url);
+    }
+  }
+  return [...neutral, ...english];
+}
+__name(prioritizeSitemapChildren, "prioritizeSitemapChildren");
+
+// src/selection/llm-select.ts
+var import_zui = require("@bpinternal/zui");
+
+// src/selection/candidate-tree.ts
+var DEFAULTS2 = { maxLines: 1e3 };
+var HIGH = [
+  "help",
+  "support",
+  "faq",
+  "faqs",
+  "hc",
+  "kb",
+  "knowledge-base",
+  "knowledgebase",
+  "docs",
+  "documentation",
+  "pricing",
+  "plans",
+  "price",
+  "prices",
+  "contact",
+  "contact-us",
+  "contacto",
+  "contatti",
+  "shipping",
+  "returns",
+  "return",
+  "refund",
+  "refunds",
+  "track",
+  "trackorder",
+  "order",
+  "orders",
+  "delivery",
+  "warranty",
+  "account",
+  "myaccount",
+  "sizing",
+  "size-guide",
+  "size-chart",
+  "how-it-works",
+  "get-started",
+  "getting-started",
+  "guide",
+  "guides",
+  "policy",
+  "policies",
+  "terms",
+  "privacy",
+  "about",
+  "about-us",
+  "integrations",
+  "api",
+  "gift-cards",
+  "gift-card",
+  // multilingual high-value
+  "tarifas",
+  "cuotas",
+  "precios",
+  "orari",
+  "ayuda",
+  "aiuto",
+  "contacto",
+  "envios",
+  "devoluciones"
+];
+var LOW2 = [
+  "blog",
+  "news",
+  "press",
+  "press-releases",
+  "investor",
+  "investors",
+  "investor-home",
+  "financials-filings",
+  "events-and-presentations",
+  "stock-information",
+  "esg",
+  "tag",
+  "tags",
+  "category",
+  "author",
+  "sitemap",
+  "archive"
+];
+var HIGH_SET = new Set(HIGH);
+var LOW_SET = new Set(LOW2);
+var DATED = /(^|[-/])(19|20)\d{2}([-/]|$)/;
+function importanceOf(url) {
+  let path2;
+  try {
+    path2 = new URL(url).pathname.toLowerCase();
+  } catch {
+    return 0;
+  }
+  const segs = path2.split("/").filter(Boolean);
+  if (segs.length === 0) return 100;
+  let score = 10 - Math.min(segs.length, 6);
+  for (const s of segs) {
+    if (HIGH_SET.has(s)) score += 40;
+    if (LOW_SET.has(s)) score -= 25;
+  }
+  if (/help|support|faq|\/hc\/|pricing|contact|shipping|return|track|refund|docs?|guide|policy/.test(path2)) score += 15;
+  if (/\/c\//.test(path2)) score += 8;
+  if (/\/p\//.test(path2)) score -= 15;
+  if (DATED.test(path2)) score -= 15;
+  return score;
+}
+__name(importanceOf, "importanceOf");
+function renderNumberedTree(candidates, host, opts = {}) {
+  const index = [];
+  const text2 = renderCandidateTree(candidates, host, opts, (url) => {
+    const i = index.length;
+    index.push(url);
+    return `[${i}] `;
+  });
+  return { text: text2, index };
+}
+__name(renderNumberedTree, "renderNumberedTree");
+function renderCandidateTree(candidates, host, opts = {}, numberPrefix) {
+  const o = { ...DEFAULTS2, ...opts };
+  const seen = /* @__PURE__ */ new Set();
+  const pages = [];
+  for (const c of candidates) {
+    if (seen.has(c.url)) continue;
+    seen.add(c.url);
+    let segs;
+    try {
+      segs = new URL(c.url).pathname.split("/").filter(Boolean);
+    } catch {
+      continue;
+    }
+    pages.push({ url: c.url, meta: c.meta, segs, imp: importanceOf(c.url) });
+  }
+  const totalPages = pages.length;
+  const byImp = [...pages].sort((a, b) => b.imp - a.imp || a.segs.length - b.segs.length || a.url.localeCompare(b.url));
+  const headerLines = 1;
+  const LOCALE = /^[a-z]{2}([-_][a-z]{2})?$/i;
+  const MARKET = /* @__PURE__ */ new Set(["us", "ca", "mx", "uk", "eu", "intl", "en", "fr", "es", "de", "it"]);
+  const groupKey = /* @__PURE__ */ __name((segs) => {
+    if (segs.length <= 1) return "";
+    let i = 0;
+    while (i < segs.length - 1 && (LOCALE.test(segs[i]) || MARKET.has(segs[i].toLowerCase()))) i++;
+    return segs.slice(0, i + 1).join("/");
+  }, "groupKey");
+  const totalByGroup = /* @__PURE__ */ new Map();
+  for (const p of pages) totalByGroup.set(groupKey(p.segs), (totalByGroup.get(groupKey(p.segs)) ?? 0) + 1);
+  const selectWithCap = /* @__PURE__ */ __name((cap) => {
+    const shownByGroup2 = /* @__PURE__ */ new Map();
+    let shownCount = 0;
+    for (const p of byImp) {
+      const g = groupKey(p.segs);
+      const arr = shownByGroup2.get(g) ?? shownByGroup2.set(g, []).get(g);
+      if (arr.length >= cap) continue;
+      arr.push(p);
+      shownCount++;
+    }
+    let foldLines = 0;
+    for (const [g, tot] of totalByGroup) if (tot > (shownByGroup2.get(g)?.length ?? 0)) foldLines++;
+    return { shownByGroup: shownByGroup2, lines: headerLines + shownCount + foldLines };
+  }, "selectWithCap");
+  let chosen = selectWithCap(1);
+  for (let cap = 2; cap <= 1e3; cap++) {
+    const next = selectWithCap(cap);
+    if (next.lines > o.maxLines) break;
+    chosen = next;
+  }
+  const shownByGroup = chosen.shownByGroup;
+  const num = /* @__PURE__ */ __name((url) => numberPrefix ? numberPrefix(url) : "", "num");
+  const metaOf = /* @__PURE__ */ __name((m) => {
+    if (!m) return "";
+    const bits = [];
+    if (m.sources?.includes("llms")) bits.push("llms");
+    if (m.lastmod && recentish(m.lastmod)) bits.push("recent");
+    return bits.length ? `  {${bits.join(",")}}` : "";
+  }, "metaOf");
+  const groups = [.../* @__PURE__ */ new Set([...totalByGroup.keys()])].sort((a, b) => {
+    const ia = Math.max(-999, ...shownByGroup.get(a)?.map((p) => p.imp) ?? []);
+    const ib = Math.max(-999, ...shownByGroup.get(b)?.map((p) => p.imp) ?? []);
+    return ib - ia || a.localeCompare(b);
+  });
+  const lines = [`${host} \u2014 ${totalPages} pages:`];
+  for (const g of groups) {
+    const shownHere = (shownByGroup.get(g) ?? []).sort((a, b) => b.imp - a.imp || a.url.localeCompare(b.url));
+    for (const p of shownHere) lines.push(`${num(p.url)}${pathStr(p.segs)}${metaOf(p.meta)}`);
+    const omitted = (totalByGroup.get(g) ?? 0) - shownHere.length;
+    if (omitted > 0) {
+      const under = g === "" ? "/" : `/${g}/`;
+      lines.push(`(+${omitted} more pages under ${under})`);
+    }
+  }
+  return lines.join("\n");
+}
+__name(renderCandidateTree, "renderCandidateTree");
+function pathStr(segs) {
+  return "/" + segs.join("/");
+}
+__name(pathStr, "pathStr");
+var TWO_YEARS_MS = 2 * 365.25 * 24 * 3600 * 1e3;
+function recentish(lastmod) {
+  const t = Date.parse(lastmod);
+  if (!Number.isFinite(t)) return false;
+  const ageMs = Date.now() - t;
+  return ageMs >= 0 && ageMs < TWO_YEARS_MS;
+}
+__name(recentish, "recentish");
+
+// src/selection/llm-select.ts
+var SELECT_PROMPT_VERSION = 3;
+async function llmSelect(candidates, host, limit, ctx, deps) {
+  const { text: tree, index } = renderNumberedTree(candidates, host);
+  const run = /* @__PURE__ */ __name(() => deps.extract(
+    `Site map for ${ctx.website ?? host} \u2014 a compact tree of the site's pages. Each pickable page is prefixed with a number [n]; folders show page counts; huge sections are folded to "+N more pages\u2026":
+
+${tree}`,
+    import_zui.z.object({
+      picks: import_zui.z.array(import_zui.z.number().int()).describe(`The \u2264${limit} most useful page NUMBERS, BEST FIRST. Use the [n] numbers from the tree.`),
+      rationale: import_zui.z.string().optional()
+    }),
+    {
+      instructions: `You are building a knowledge base for an AI SUPPORT agent${ctx.company ? ` for ${ctx.company}` : ""}. Pick the \u2264${limit} pages, BEST FIRST, that best let the agent answer real customer/visitor questions. Return their [n] NUMBERS.
+
+${ctx.overview ? `ABOUT THE BUSINESS: ${ctx.overview}
+` : ""}${ctx.prompt ? `USE-CASE FOCUS: ${ctx.prompt}
+` : ""}
+How to choose (judgment about THIS business, not rules):
+- Prioritize pages that answer questions: what the business offers, pricing/plans, how it works, how to buy/apply/book, FAQ/help/support articles, policies (returns/shipping/privacy), contact, locations/hours, key product/service pages.
+- A page's URL slug tells you what it's about \u2014 use it (any language: Spanish tarifas/cuotas, Italian orari/contatti \u2014 judge by MEANING).
+- Judge whether this site's value is in its SERVICE/PRODUCT pages or its ARTICLES: a news org / non-profit's articles ARE the product (include many); a shop or club with a peripheral blog should mostly SKIP the blog and take the service pages.
+- Do NOT spend slots on near-identical BULK: individual product SKUs, every staff bio, every location, every dated promo/event, blog archives. Take a few representative ones at most.
+- Skip non-content: cart/checkout/login/account, tag/category/author archives, search/sitemap pages, dated marketing campaigns, thin utility/landing pages.
+- Prefer ONE language (the site's default) \u2014 don't pick the same page in multiple locales.
+
+Return \u2264${limit} page NUMBERS, best first.`
+    }
+  ), "run");
+  const cache = deps.cache ?? (async (_kind, _keyParts, fn) => fn());
+  const raw = await cache("url-selection.select", { v: SELECT_PROMPT_VERSION, tree, limit, ctx }, async () => {
+    try {
+      return await run();
+    } catch {
+      return await run();
+    }
+  }).catch(() => null);
+  if (!raw || !Array.isArray(raw.picks)) return { urls: [], rationale: "llm-select failed" };
+  const seen = /* @__PURE__ */ new Set();
+  const urls = [];
+  for (const p of raw.picks) {
+    const i = typeof p === "number" ? p : NaN;
+    if (!Number.isInteger(i) || i < 0 || i >= index.length) continue;
+    const url = index[i];
+    if (seen.has(url)) continue;
+    seen.add(url);
+    urls.push(url);
+    if (urls.length >= limit) break;
+  }
+  return { urls, rationale: raw.rationale };
+}
+__name(llmSelect, "llmSelect");
+
+// src/selection/select.ts
+var DEFAULT_LIMIT = 100;
+async function select(map, opts = {}, deps) {
+  const limit = opts.limit ?? DEFAULT_LIMIT;
+  if (map.unsupported) return { urls: [], useful: 0 };
+  const candidates = [...leafUrls(map.tree)].filter((n) => !!n.url).map((n) => ({ url: n.url, meta: n.meta ?? { sources: [] } }));
+  if (candidates.length === 0) return { urls: [], useful: 0 };
+  const host = hostOf(candidates[0].url) ?? "";
+  const picked = await llmSelect(
+    candidates,
+    host,
+    limit,
+    {
+      ...opts.context?.company ? { company: opts.context.company } : {},
+      ...opts.context?.website ? { website: opts.context.website } : {},
+      ...opts.context?.overview ? { overview: opts.context.overview } : {},
+      ...opts.prompt ? { prompt: opts.prompt } : {}
+    },
+    deps
+  );
+  return { urls: picked.urls, useful: candidates.length };
+}
+__name(select, "select");
+
+// src/http-cache.ts
+var import_node_fs = __toESM(require("fs"), 1);
+var import_node_path = __toESM(require("path"), 1);
+function fastHash(str) {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = (hash << 5) - hash + str.charCodeAt(i);
+    hash |= 0;
+  }
+  return (hash >>> 0).toString(16);
+}
+__name(fastHash, "fastHash");
+function sortKeys(input) {
+  if (Array.isArray(input)) return input.map(sortKeys);
+  if (input && typeof input === "object" && input.constructor === Object) {
+    return Object.keys(input).sort().reduce(
+      (acc, key) => {
+        acc[key] = sortKeys(input[key]);
+        return acc;
+      },
+      {}
+    );
+  }
+  return input;
+}
+__name(sortKeys, "sortKeys");
+function stableStringify(obj) {
+  return JSON.stringify(sortKeys(obj));
+}
+__name(stableStringify, "stableStringify");
+function httpCacheMode() {
+  if (process.env.URL_FIXTURE_RECORD) return "record";
+  if (process.env.NODE_ENV === "test" || process.env.BUN_TEST || process.env.VITEST || process.env.URL_FIXTURE_REPLAY) {
+    return "replay";
+  }
+  return "passthrough";
+}
+__name(httpCacheMode, "httpCacheMode");
+var FIXTURE_DIR = import_node_path.default.resolve(__dirname, "./__fixtures__");
+var loaded = /* @__PURE__ */ new Map();
+function groupPath(group) {
+  return import_node_path.default.join(FIXTURE_DIR, `${group}.jsonl`);
+}
+__name(groupPath, "groupPath");
+function loadGroup(group) {
+  const existing = loaded.get(group);
+  if (existing) return existing;
+  const map = /* @__PURE__ */ new Map();
+  try {
+    const content = import_node_fs.default.readFileSync(groupPath(group), "utf-8");
+    for (const line of content.split(/\r?\n/).filter(Boolean)) {
+      try {
+        const entry = JSON.parse(line);
+        map.set(entry.key, entry);
+      } catch {
+      }
+    }
+  } catch {
+  }
+  loaded.set(group, map);
+  return map;
+}
+__name(loadGroup, "loadGroup");
+function appendEntry(entry) {
+  const map = loadGroup(entry.group);
+  map.set(entry.key, entry);
+  try {
+    import_node_fs.default.mkdirSync(FIXTURE_DIR, { recursive: true });
+  } catch {
+  }
+  import_node_fs.default.appendFileSync(groupPath(entry.group), JSON.stringify(entry) + "\n");
+}
+__name(appendEntry, "appendEntry");
+var FixtureMissError = class extends Error {
+  static {
+    __name(this, "FixtureMissError");
+  }
+  constructor(kind, group, input) {
+    super(
+      `[http-cache] no recorded fixture for kind="${kind}" group="${group}".
+Re-record with: URL_FIXTURE_RECORD=1 bun run scripts/record-fixtures.ts
+input: ${input.slice(0, 300)}`
+    );
+    this.name = "FixtureMissError";
+  }
+};
+var stats = { hits: 0, misses: 0, recorded: 0 };
+async function cachedHttp(group, kind, args, fn) {
+  const mode = httpCacheMode();
+  if (mode === "passthrough") return fn();
+  const key = fastHash(`${kind}:${stableStringify(args)}`);
+  const map = loadGroup(group);
+  const hit = map.get(key);
+  if (hit) {
+    stats.hits++;
+    return hit.value;
+  }
+  if (mode === "replay") {
+    stats.misses++;
+    throw new FixtureMissError(kind, group, stableStringify(args));
+  }
+  const value = await fn();
+  appendEntry({ key, kind, group, input: stableStringify(args), value });
+  stats.recorded++;
+  return value;
+}
+__name(cachedHttp, "cachedHttp");
+
+// src/discovery/cached-deps.ts
+function cachedDeps(group, real) {
+  return {
+    // extract/cache pass through untouched — the LLM side has its own
+    // record/replay via the injected cache dep.
+    ...real,
+    fetchText: /* @__PURE__ */ __name((url, baseHost, timeoutMs) => cachedHttp(group, "fetchText", { url }, () => real.fetchText(url, baseHost, timeoutMs)), "fetchText"),
+    fetchJson: /* @__PURE__ */ __name((url, baseHost, timeoutMs) => cachedHttp(group, "fetchJson", { url }, () => real.fetchJson(url, baseHost, timeoutMs)), "fetchJson"),
+    discoverUrls: /* @__PURE__ */ __name((args) => cachedHttp(group, "discoverUrls", args, () => real.discoverUrls(args)), "discoverUrls"),
+    webSearch: /* @__PURE__ */ __name((args) => cachedHttp(group, "webSearch", args, () => real.webSearch(args)), "webSearch")
+  };
+}
+__name(cachedDeps, "cachedDeps");
+
+// src/index.ts
+async function findKnowledgeUrls(input, deps, extra = {}) {
+  if (isUnsupportedSite(input.website)) {
+    return {
+      urls: [],
+      useful: 0,
+      discovered: 0,
+      aborted: false,
+      siteType: "unknown",
+      stopReason: "unsupported_site"
+    };
+  }
+  if (!hostOf(input.website)) {
+    return {
+      urls: [],
+      useful: 0,
+      discovered: 0,
+      aborted: false,
+      siteType: "unknown",
+      stopReason: "no_sources"
+    };
+  }
+  const map = await discover(
+    input.website,
+    {
+      ...input.topics ? { topics: input.topics } : {},
+      ...input.signal ? { signal: input.signal } : {},
+      ...input.budgetMs != null ? { budgetMs: input.budgetMs } : {},
+      ...input.onActivity ? { onActivity: input.onActivity } : {},
+      ...extra.now != null ? { now: extra.now } : {}
+    },
+    deps
+  );
+  const selection = await select(
+    map,
+    {
+      ...input.limit != null ? { limit: input.limit } : {},
+      ...input.prompt ? { prompt: input.prompt } : {},
+      ...input.context ? { context: input.context } : {}
+    },
+    deps
+  );
+  const stopReason = map.aborted ? "time_limit_reached" : map.total === 0 ? "no_sources" : "ok";
+  return { ...selection, discovered: map.total, aborted: map.aborted, siteType: map.siteType, stopReason };
+}
+__name(findKnowledgeUrls, "findKnowledgeUrls");
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  cachedDeps,
+  discover,
+  fetchJson,
+  fetchText,
+  findKnowledgeUrls,
+  hostOf,
+  select,
+  siteOrigin
+});
+//# sourceMappingURL=index.cjs.map