@snap-agent/rag-web 0.1.5 → 0.1.7

package/dist/index.js

@@ -30,16 +30,566 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
-  WebRAGPlugin: () => WebRAGPlugin
+  WebRAGPlugin: () => WebRAGPlugin,
+  bodyTextLengthHint: () => bodyTextLengthHint,
+  extractPageFromHtml: () => extractPageFromHtml,
+  extractProductMetadata: () => extractProductMetadata,
+  hardExcludePage: () => hardExcludePage,
+  inferTypeFromUrl: () => inferTypeFromUrl,
+  normalizeAvailability: () => normalizeAvailability,
+  normalizeCurrency: () => normalizeCurrency,
+  normalizeDisplayTitle: () => normalizeDisplayTitle,
+  parsePrice: () => parsePrice,
+  resolvePageCardMetadata: () => resolvePageCardMetadata,
+  urlToDocumentId: () => urlToDocumentId
 });
 module.exports = __toCommonJS(index_exports);
 
 // src/WebRAGPlugin.ts
 var import_mongodb = require("mongodb");
 var import_openai = __toESM(require("openai"));
-var cheerio = __toESM(require("cheerio"));
+var cheerio4 = __toESM(require("cheerio"));
 var fs = __toESM(require("fs"));
 var path = __toESM(require("path"));
+
+// src/htmlPageExtract.ts
+var cheerio3 = __toESM(require("cheerio"));
+
+// src/productMetadata.ts
+var cheerio = __toESM(require("cheerio"));
+function extractProductMetadata(html) {
+  const $ = cheerio.load(html);
+  const fromJsonLd = extractFromJsonLd($);
+  const fromOg = extractFromOpenGraph($);
+  const fromMicrodata = extractFromMicrodata($);
+  const result = {};
+  const price = fromJsonLd.price ?? fromOg.price ?? fromMicrodata.price;
+  if (price != null) result.price = price;
+  const currency = fromJsonLd.currency ?? fromOg.currency ?? fromMicrodata.currency;
+  if (currency) result.currency = currency;
+  const availability = fromJsonLd.availability ?? fromOg.availability ?? fromMicrodata.availability;
+  if (availability) result.availability = availability;
+  return result;
+}
+function extractFromJsonLd($) {
+  const result = {};
+  $('script[type="application/ld+json"]').each((_, el) => {
+    if (result.price != null && result.currency && result.availability) return false;
+    const raw = $(el).html()?.trim();
+    if (!raw) return;
+    let parsed;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      return;
+    }
+    for (const node of collectJsonLdNodes(parsed)) {
+      if (!isProductType(node)) continue;
+      const offer = pickOffer(node);
+      if (!offer) continue;
+      if (result.price == null) {
+        const price = parsePrice(offer.price ?? offer.lowPrice ?? offer.highPrice);
+        if (price != null) result.price = price;
+      }
+      if (!result.currency) {
+        const currency = normalizeCurrency(offer.priceCurrency);
+        if (currency) result.currency = currency;
+      }
+      if (!result.availability) {
+        const availability = normalizeAvailability(offer.availability);
+        if (availability) result.availability = availability;
+      }
+    }
+  });
+  return result;
+}
+function extractFromOpenGraph($) {
+  const result = {};
+  const priceRaw = $('meta[property="product:price:amount"]').attr("content") || $('meta[property="og:price:amount"]').attr("content");
+  const price = parsePrice(priceRaw);
+  if (price != null) result.price = price;
+  const currency = normalizeCurrency(
+    $('meta[property="product:price:currency"]').attr("content") || $('meta[property="og:price:currency"]').attr("content")
+  );
+  if (currency) result.currency = currency;
+  const availability = normalizeAvailability(
+    $('meta[property="product:availability"]').attr("content") || $('meta[property="og:availability"]').attr("content")
+  );
+  if (availability) result.availability = availability;
+  return result;
+}
+function microdataField($, itemprop) {
+  const scope = $('[itemtype*="schema.org/Product"], [itemtype*="schema.org/product"]').first();
+  return scope.length > 0 ? scope.find(`[itemprop="${itemprop}"]`).first() : $(`[itemprop="${itemprop}"]`).first();
+}
+function extractFromMicrodata($) {
+  const result = {};
+  const priceEl = microdataField($, "price");
+  const price = parsePrice(priceEl.attr("content") || priceEl.text());
+  if (price != null) result.price = price;
+  const currencyEl = microdataField($, "priceCurrency");
+  const currency = normalizeCurrency(currencyEl.attr("content") || currencyEl.text());
+  if (currency) result.currency = currency;
+  const availabilityEl = microdataField($, "availability");
+  const availability = normalizeAvailability(
+    availabilityEl.attr("content") || availabilityEl.attr("href") || availabilityEl.text()
+  );
+  if (availability) result.availability = availability;
+  return result;
+}
+function collectJsonLdNodes(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function isProductType(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  return types.some((t) => {
+    const s = String(t).toLowerCase();
+    return s === "product" || s.endsWith("/product");
+  });
+}
+function pickOffer(product) {
+  const offers = product.offers;
+  if (offers == null) return null;
+  if (Array.isArray(offers)) {
+    const first = offers.find((o) => o && typeof o === "object");
+    return first ?? null;
+  }
+  if (typeof offers === "object") return offers;
+  return null;
+}
+function parsePrice(value) {
+  if (value == null || value === "") return void 0;
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  let s = String(value).trim();
+  if (!s) return void 0;
+  s = s.replace(/[^\d.,\-]/g, "");
+  if (!s || s === "-" || s === ".") return void 0;
+  const lastComma = s.lastIndexOf(",");
+  const lastDot = s.lastIndexOf(".");
+  if (lastComma > -1 && lastDot > -1) {
+    if (lastComma > lastDot) {
+      s = s.replace(/\./g, "").replace(",", ".");
+    } else {
+      s = s.replace(/,/g, "");
+    }
+  } else if (lastComma > -1) {
+    const parts = s.split(",");
+    if (parts.length === 2 && parts[1].length <= 2) {
+      s = parts[0].replace(/\./g, "") + "." + parts[1];
+    } else {
+      s = s.replace(/,/g, "");
+    }
+  }
+  const num = parseFloat(s);
+  return Number.isFinite(num) ? num : void 0;
+}
+function normalizeCurrency(value) {
+  if (value == null) return void 0;
+  const s = String(value).trim().toUpperCase();
+  if (!s) return void 0;
+  const iso = s.match(/[A-Z]{3}/);
+  return iso ? iso[0] : s.length <= 4 ? s : void 0;
+}
+function normalizeAvailability(value) {
+  if (value == null) return void 0;
+  let s = String(value).trim();
+  if (!s) return void 0;
+  if (s.includes("schema.org/")) {
+    const parts = s.split("/");
+    s = parts[parts.length - 1] || s;
+  }
+  s = s.replace(/^https?:\/\/[^/]+\//, "");
+  if (s.includes("/")) {
+    const parts = s.split("/");
+    s = parts[parts.length - 1] || s;
+  }
+  return s.replace(/\s+/g, "") || void 0;
+}
+
+// src/pageCardMetadata.ts
+var cheerio2 = __toESM(require("cheerio"));
+var HARD_EXCLUDE_URL_RE = /(?:^|\/)(?:login|signin|sign-in|signup|sign-up|register|account|cart|checkout|admin|wp-admin|privacy|terms|legal|cookies|gdpr|thank|gracias|confirm|success|receipt|404|tag|tags|category|categories|author|archive|newsletter|careers|jobs)(?:\/|$|-|\.)/i;
+var HARD_EXCLUDE_TITLE_RE = /\b(?:login|sign\s*in|sign\s*up|privacy\s*policy|terms\s*(?:of\s*)?service|thank\s*you|gracias\s*por|admin|404|not\s*found)\b/i;
+var BLOG_URL_RE = /(?:^|\/)(?:blog|news|press|article|posts?)(?:\/|$)/i;
+var PROMOTION_URL_RE = /(?:^|\/)(?:offer|offers|sale|sales|promo|promotion|deal|deals|coupon|special-offer|buster)(?:\/|$|-|\.)/i;
+var PROMOTION_SLUG_RE = /(?:^|\/)[^/]*(?:-sale|-offer|-promo|-deal|-buster)(?:\/|$)/i;
+var ENTITY_DETAIL_PATH_RES = [
+  /\/projects\/[^/]+/i,
+  /\/project\/[^/]+/i,
+  /\/perspectives\/[^/]+/i,
+  /\/perspective\/[^/]+/i,
+  /\/portfolio\/[^/]+/i,
+  /\/case-stud(?:y|ies)\/[^/]+/i,
+  /\/insights?\/[^/]+/i,
+  /\/people\/[^/]+/i,
+  /\/person\/[^/]+/i,
+  /\/team-members?\/[^/]+/i,
+  /\/members?\/[^/]+/i,
+  /\/staff\/[^/]+/i,
+  /\/experts?\/[^/]+/i,
+  /\/authors?\/[^/]+/i,
+  /\/leadership\/[^/]+/i,
+  /\/biograph(?:y|ies)\/[^/]+/i
+];
+var DETAIL_URL_RE = /(?:^|\/)(?:product|products|item|items|p|room|rooms|suite|suites|habitacion|plan|plans|space|spaces|tour|tours|menu|project|perspective|person|team-member|team-members|staff|expert|case-study|author|biography)(?:\/|$)/i;
+var LISTING_URL_RE = /(?:^|\/)(?:catalog|catalogue|collection|collections|category|categories|shop|store|habitaciones|rooms|products|projects|perspectives|portfolio|people|team|members|insights|case-studies|thought-leadership)(?:\/|$)/i;
+var AMENITY_URL_RE = /(?:^|\/)(?:amenity|amenities|activity|activities|experience|experiences|service-page)(?:\/|$)/i;
+var CONTACT_URL_RE = /(?:^|\/)(?:contact|contacto|about|nosotros|faq|help|support)(?:\/|$)/i;
+var EN_DASH_SUFFIX_RE = /\s+[–—]\s+.+$/;
+var PIPE_SUFFIX_RE = /\s+\|\s+.+$/;
+var CARD_PRIORITY = {
+  detail: 10,
+  listing: 6,
+  amenity: 5,
+  promotion: 2,
+  contact: 1,
+  content: 1,
+  blog: 0,
+  system: 0,
+  page: 3
+};
+var CARD_ELIGIBLE_DEFAULT = {
+  detail: true,
+  listing: true,
+  amenity: true,
+  promotion: false,
+  contact: false,
+  content: false,
+  blog: false,
+  system: false,
+  page: false
+};
+var SCHEMA_TYPE_MAP = {
+  product: "detail",
+  service: "amenity",
+  hotelroom: "detail",
+  room: "detail",
+  apartment: "detail",
+  lodgingroom: "detail",
+  course: "detail",
+  event: "detail",
+  offer: "promotion",
+  person: "detail",
+  employee: "detail",
+  profilepage: "detail",
+  article: "detail",
+  newsarticle: "detail",
+  blogposting: "detail",
+  creativework: "detail"
+};
+function normalizeDisplayTitle(title) {
+  if (!title?.trim()) return title;
+  let t = title.trim();
+  for (let i = 0; i < 2; i++) {
+    const dash = t.match(EN_DASH_SUFFIX_RE);
+    if (dash && dash.index !== void 0 && dash.index >= 4) {
+      t = t.slice(0, dash.index).trim();
+      continue;
+    }
+    const pipe = t.match(PIPE_SUFFIX_RE);
+    if (pipe && pipe.index !== void 0 && pipe.index >= 8) {
+      t = t.slice(0, pipe.index).trim();
+      continue;
+    }
+    break;
+  }
+  return t || title.trim();
+}
+function hardExcludePage(url, title) {
+  const path2 = url.toLowerCase();
+  if (HARD_EXCLUDE_URL_RE.test(path2)) return true;
+  if (BLOG_URL_RE.test(path2)) return true;
+  if (title && HARD_EXCLUDE_TITLE_RE.test(title.toLowerCase())) return true;
+  try {
+    const u = new URL(url);
+    if (u.pathname === "/" || u.pathname === "") return true;
+  } catch {
+  }
+  return false;
+}
+function inferTypeFromUrl(url) {
+  const path2 = url.toLowerCase();
+  if (PROMOTION_URL_RE.test(path2) || PROMOTION_SLUG_RE.test(path2)) return "promotion";
+  if (CONTACT_URL_RE.test(path2)) return "contact";
+  if (ENTITY_DETAIL_PATH_RES.some((re) => re.test(path2))) return "detail";
+  if (AMENITY_URL_RE.test(path2)) return "amenity";
+  if (DETAIL_URL_RE.test(path2)) return "detail";
+  if (LISTING_URL_RE.test(path2)) return "listing";
+  if (BLOG_URL_RE.test(path2)) return "blog";
+  return void 0;
+}
+function collectJsonLdNodes2(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function schemaTypeName(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  const raw = types[0];
+  if (raw == null) return "";
+  const s = String(raw).toLowerCase();
+  const slash = s.lastIndexOf("/");
+  return slash >= 0 ? s.slice(slash + 1) : s;
+}
+function inferTypeFromSchema(html) {
+  const $ = cheerio2.load(html);
+  for (const el of $('script[type="application/ld+json"]').toArray()) {
+    const raw = $(el).html()?.trim();
+    if (!raw) continue;
+    try {
+      const parsed = JSON.parse(raw);
+      for (const node of collectJsonLdNodes2(parsed)) {
+        const name = schemaTypeName(node);
+        if (SCHEMA_TYPE_MAP[name]) return SCHEMA_TYPE_MAP[name];
+        if (name === "product" || node.offers != null) return "detail";
+      }
+    } catch {
+    }
+  }
+  const ogType = $('meta[property="og:type"]').attr("content")?.toLowerCase();
+  if (ogType === "product") return "detail";
+  return void 0;
+}
+function normalizePageType(raw) {
+  if (!raw) return "page";
+  const lower = raw.toLowerCase();
+  const known = [
+    "detail",
+    "listing",
+    "amenity",
+    "promotion",
+    "contact",
+    "content",
+    "blog",
+    "system",
+    "page"
+  ];
+  if (known.includes(lower)) return lower;
+  if (lower === "room" || lower === "product") return "detail";
+  if (lower === "offer" || lower === "sale") return "promotion";
+  return raw;
+}
+function resolveDisplayTitle(input) {
+  const heading = input.headingTitle?.trim();
+  if (heading) return normalizeDisplayTitle(heading);
+  return normalizeDisplayTitle(input.title);
+}
+function resolvePageCardMetadata(input) {
+  const title = input.title?.trim();
+  const url = input.url;
+  const displayTitle = resolveDisplayTitle(input);
+  if (hardExcludePage(url, title)) {
+    return {
+      type: "system",
+      cardEligible: false,
+      cardPriority: 0,
+      displayTitle,
+      displayDescription: input.description,
+      displayImageUrl: input.imageUrl
+    };
+  }
+  let type = normalizePageType(input.type);
+  if (type === "page" && input.html) {
+    const fromSchema = inferTypeFromSchema(input.html);
+    if (fromSchema) type = fromSchema;
+  }
+  if (type === "page") {
+    const fromUrl = inferTypeFromUrl(url);
+    if (fromUrl) type = fromUrl;
+  }
+  if (input.hasProductPrice && type === "page") {
+    type = "detail";
+  }
+  const typeKey = String(type).toLowerCase();
+  let cardEligible = CARD_ELIGIBLE_DEFAULT[typeKey] ?? false;
+  let cardPriority = CARD_PRIORITY[typeKey] ?? 3;
+  if (cardEligible && PROMOTION_URL_RE.test(url.toLowerCase())) {
+    cardEligible = false;
+  }
+  return {
+    type,
+    cardEligible,
+    cardPriority,
+    displayTitle,
+    displayDescription: input.description?.trim() || void 0,
+    displayImageUrl: input.imageUrl
+  };
+}
+
+// src/htmlPageExtract.ts
+var DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
+var DEFAULT_REMOVE_SELECTORS = [
+  "script",
+  "style",
+  "nav",
+  "header",
+  "footer",
+  ".sidebar",
+  ".navigation",
+  ".menu",
+  ".comments",
+  '[role="navigation"]',
+  '[role="banner"]'
+];
+function urlToDocumentId(url) {
+  return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+}
+function cleanContent(text) {
+  return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
+}
+function bodyTextLengthHint(html, options = {}) {
+  const $ = cheerio3.load(html);
+  stripNoiseFromDom($, options);
+  return cleanContent($("body").text().trim()).length;
+}
+function extractPageFromHtml(url, html, options = {}) {
+  const $ = cheerio3.load(html);
+  stripNoiseFromDom($, options);
+  const h1Title = $("h1").first().text().trim();
+  const docTitle = $("title").text().trim();
+  let title = "";
+  if (options.titleSelector) {
+    title = $(options.titleSelector).first().text().trim();
+  } else {
+    title = docTitle || h1Title;
+  }
+  if (!title) {
+    title = h1Title || docTitle;
+  }
+  const content = extractBestContentText($, options);
+  const minChars = options.minExtractedContentLength ?? 50;
+  const indexable = Boolean(content && content.length >= minChars);
+  const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || extractHeroImage($, url) || void 0;
+  let imageUrl;
+  if (image) {
+    try {
+      imageUrl = new URL(image, url).href;
+    } catch {
+      imageUrl = image;
+    }
+  }
+  const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
+  let type = options.defaultType || "page";
+  if (options.typeFromUrl) {
+    for (const [pattern, typeName] of Object.entries(options.typeFromUrl)) {
+      if (url.includes(pattern)) {
+        type = typeName;
+        break;
+      }
+    }
+  }
+  const productMeta = extractProductMetadata(html);
+  const cardMeta = resolvePageCardMetadata({
+    url,
+    title,
+    headingTitle: h1Title || void 0,
+    description,
+    imageUrl,
+    html,
+    type,
+    hasProductPrice: productMeta.price != null
+  });
+  const metadata = {
+    type: cardMeta.type,
+    cardEligible: cardMeta.cardEligible,
+    cardPriority: cardMeta.cardPriority,
+    ...title ? { title } : {},
+    ...cardMeta.displayTitle ? { displayTitle: cardMeta.displayTitle } : {},
+    url,
+    ...imageUrl ? { imageUrl } : {},
+    ...cardMeta.displayImageUrl ? { displayImageUrl: cardMeta.displayImageUrl } : {},
+    ...description ? { description } : {},
+    ...cardMeta.displayDescription ? { displayDescription: cardMeta.displayDescription } : {},
+    ...productMeta.price != null ? { price: productMeta.price } : {},
+    ...productMeta.currency ? { currency: productMeta.currency } : {},
+    ...productMeta.availability ? { availability: productMeta.availability } : {},
+    ...options.metadata
+  };
+  const previewLen = 400;
+  const contentPreview = content.length > previewLen ? `${content.slice(0, previewLen)}\u2026` : content;
+  return {
+    id: urlToDocumentId(url),
+    metadata,
+    content,
+    indexable,
+    contentPreview
+  };
+}
+function stripNoiseFromDom($, options) {
+  const removeSelectors = options.removeSelectors ?? DEFAULT_REMOVE_SELECTORS;
+  removeSelectors.forEach((selector) => $(selector).remove());
+}
+function extractBestContentText($, options) {
+  const contentSelector = options.contentSelector || DEFAULT_CONTENT_SELECTOR;
+  const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
+  let best = "";
+  for (const sel of selectors) {
+    $(sel).each((_, el) => {
+      const t = cleanContent($(el).text().trim());
+      if (t.length > best.length) best = t;
+    });
+  }
+  const bodyText = cleanContent($("body").text().trim());
+  if (bodyText.length > best.length) best = bodyText;
+  return best;
+}
+function extractHeroImage($, pageUrl) {
+  const containers = $('main, article, [role="main"], #content, .content');
+  const scope = containers.length > 0 ? containers : $("body");
+  let best;
+  scope.find("img[src]").each((_, el) => {
+    if (best) return false;
+    const src = $(el).attr("src") || "";
+    const alt = ($(el).attr("alt") || "").toLowerCase();
+    const width = parseInt($(el).attr("width") || "0", 10);
+    const height = parseInt($(el).attr("height") || "0", 10);
+    if (width > 0 && width < 80 || height > 0 && height < 80) return;
+    if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
+    if (src.startsWith("data:") || src.endsWith(".svg")) return;
+    if (src.includes("/_next/image")) {
+      try {
+        const nextUrl = new URL(src, pageUrl);
+        const realUrl = nextUrl.searchParams.get("url");
+        if (realUrl) {
+          best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
+          return false;
+        }
+      } catch {
+      }
+    }
+    best = src;
+    return false;
+  });
+  return best;
+}
+
+// src/WebRAGPlugin.ts
 function bulkOpCurrentUrl(op) {
   const meta = op.document?.metadata;
   if (typeof meta?.url === "string" && meta.url.trim()) return meta.url.trim();
@@ -58,7 +608,7 @@ function isUrlListingInsert(document) {
     return false;
   }
 }
-var WebRAGPlugin = class _WebRAGPlugin {
+var WebRAGPlugin = class {
   name = "web-rag";
   type = "rag";
   priority;
@@ -281,13 +831,21 @@ var WebRAGPlugin = class _WebRAGPlugin {
         plugin: this.name,
         contentCount: scoredResults.length,
         types: [...new Set(scoredResults.map((d) => d.metadata.type))],
-        topResults: scoredResults.slice(0, 5).map((doc) => ({
+        topResults: scoredResults.slice(0, 16).map((doc) => ({
           id: doc.id,
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
           imageUrl: doc.metadata.imageUrl,
           description: doc.metadata.description,
+          cardEligible: doc.metadata.cardEligible,
+          cardPriority: doc.metadata.cardPriority,
+          displayTitle: doc.metadata.displayTitle,
+          displayDescription: doc.metadata.displayDescription,
+          displayImageUrl: doc.metadata.displayImageUrl,
+          ...doc.metadata.price != null ? { price: doc.metadata.price } : {},
+          ...doc.metadata.currency ? { currency: doc.metadata.currency } : {},
+          ...doc.metadata.availability ? { availability: doc.metadata.availability } : {},
           score: doc.score
         }))
       }
@@ -1458,7 +2016,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return await response.text();
   }
   extractInternalLinks(html, base, stripQueryParams) {
-    const $ = cheerio.load(html);
+    const $ = cheerio4.load(html);
     const links = /* @__PURE__ */ new Set();
     $("a[href]").each((_, el) => {
       const href = ($(el).attr("href") || "").trim();
@@ -1624,7 +2182,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
             }
           }
           try {
-            const { doc, diag, bodyTextLengthHint } = await this.crawlPageSmart(url, config, timeout, {
+            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2 } = await this.crawlPageSmart(url, config, timeout, {
               renderMode,
               renderOptions,
               minContentLength,
@@ -1655,7 +2213,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
               status: crawlSt,
               modeUsed: diag?.modeUsed,
               contentLength: doc?.content?.length,
-              bodyTextLengthHint,
+              bodyTextLengthHint: bodyTextLengthHint2,
               title: doc?.metadata?.title,
               docId: doc?.id,
               error: diag?.errorMessage
@@ -1767,125 +2325,18 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const html = await response.text();
     return this.extractDocumentFromHtml(url, html, config);
   }
-  /**
-   * Default chain works for many WordPress / Elementor / block themes where `.first()`
-   * would otherwise hit an empty wrapper.
-   */
-  static DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
-  stripNoiseFromDom($, config) {
-    const removeSelectors = config.removeSelectors || [
-      "script",
-      "style",
-      "nav",
-      "header",
-      "footer",
-      ".sidebar",
-      ".navigation",
-      ".menu",
-      ".comments",
-      '[role="navigation"]',
-      '[role="banner"]'
-    ];
-    removeSelectors.forEach((selector) => $(selector).remove());
-  }
-  /** Longest cleaned text among selector matches and full body (after noise strip). */
-  extractBestContentText($, config) {
-    const contentSelector = config.contentSelector || _WebRAGPlugin.DEFAULT_CONTENT_SELECTOR;
-    const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
-    let best = "";
-    for (const sel of selectors) {
-      $(sel).each((_, el) => {
-        const t = this.cleanContent($(el).text().trim());
-        if (t.length > best.length) best = t;
-      });
-    }
-    const bodyText = this.cleanContent($("body").text().trim());
-    if (bodyText.length > best.length) best = bodyText;
-    return best;
-  }
   bodyTextLengthHint(html, config) {
-    const $ = cheerio.load(html);
-    this.stripNoiseFromDom($, config);
-    return this.cleanContent($("body").text().trim()).length;
+    return bodyTextLengthHint(html, config);
   }
   extractDocumentFromHtml(url, html, config) {
-    const $ = cheerio.load(html);
-    this.stripNoiseFromDom($, config);
-    const titleSelector = config.titleSelector || "h1, title";
-    let title = $(titleSelector).first().text().trim();
-    if (!title) {
-      title = $("title").text().trim();
-    }
-    const content = this.extractBestContentText($, config);
-    const minChars = config.minExtractedContentLength ?? 50;
-    if (!content || content.length < minChars) return null;
-    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || // Fallback: largest/first meaningful image in main content area
-    this.extractHeroImage($, url) || void 0;
-    let imageUrl;
-    if (image) {
-      try {
-        imageUrl = new URL(image, url).href;
-      } catch {
-        imageUrl = image;
-      }
-    }
-    const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
-    let type = config.defaultType || "page";
-    if (config.typeFromUrl) {
-      for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
-        if (url.includes(pattern)) {
-          type = typeName;
-          break;
-        }
-      }
-    }
-    const id = this.urlToId(url);
+    const extracted = extractPageFromHtml(url, html, config);
+    if (!extracted.indexable) return null;
     return {
-      id,
-      content,
-      metadata: {
-        type,
-        title,
-        url,
-        ...imageUrl ? { imageUrl } : {},
-        ...description ? { description } : {},
-        ...config.metadata
-      }
+      id: extracted.id,
+      content: extracted.content,
+      metadata: extracted.metadata
     };
   }
-  /**
-   * Fallback image extraction: finds the first meaningful image in the content area.
-   * Skips icons, avatars, and tiny assets by filtering on common patterns.
-   */
-  extractHeroImage($, pageUrl) {
-    const containers = $('main, article, [role="main"], #content, .content');
-    const scope = containers.length > 0 ? containers : $("body");
-    let best;
-    scope.find("img[src]").each((_, el) => {
-      if (best) return false;
-      const src = $(el).attr("src") || "";
-      const alt = ($(el).attr("alt") || "").toLowerCase();
-      const width = parseInt($(el).attr("width") || "0", 10);
-      const height = parseInt($(el).attr("height") || "0", 10);
-      if (width > 0 && width < 80 || height > 0 && height < 80) return;
-      if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
-      if (src.startsWith("data:") || src.endsWith(".svg")) return;
-      if (src.includes("/_next/image")) {
-        try {
-          const nextUrl = new URL(src, pageUrl);
-          const realUrl = nextUrl.searchParams.get("url");
-          if (realUrl) {
-            best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
-            return false;
-          }
-        } catch {
-        }
-      }
-      best = src;
-      return false;
-    });
-    return best;
-  }
   looksLikeDynamicShell(html) {
     const lower = html.toLowerCase();
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
@@ -1903,7 +2354,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
     return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
   }
-  diagFromRenderedAttempt(doc, bodyTextLengthHint, renderFailure, blockedSuspected, modeOk, modeFailed) {
+  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed) {
     if (blockedSuspected) {
       return {
         doc: null,
@@ -1919,12 +2370,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return {
       doc,
       diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
-      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2
     };
   }
   async crawlPageSmart(url, config, timeout, ctx) {
     if (ctx.renderMode === true) {
-      const { doc, bodyTextLengthHint, renderFailure, blockedSuspected } = await this.crawlPageRendered(
+      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected } = await this.crawlPageRendered(
         url,
         config,
         timeout,
@@ -1933,7 +2384,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       );
       return this.diagFromRenderedAttempt(
         doc,
-        bodyTextLengthHint,
+        bodyTextLengthHint2,
         renderFailure,
         blockedSuspected,
         "render_ok",
@@ -2050,7 +2501,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
         }
       }
       const html = await page.content();
-      const bodyTextLengthHint = this.bodyTextLengthHint(html, config);
+      const bodyTextLengthHint2 = this.bodyTextLengthHint(html, config);
       const doc = this.extractDocumentFromHtml(url, html, config);
       if (config.debug?.saveDir && config.debug?.enabled) {
         try {
@@ -2065,7 +2516,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
           dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
         }
       }
-      return { doc, bodyTextLengthHint };
+      return { doc, bodyTextLengthHint: bodyTextLengthHint2 };
     } catch (e) {
       const msg = String(e?.message || e || "render_failed");
       const lower = msg.toLowerCase();
@@ -2157,14 +2608,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
   /**
    * Clean extracted text content
    */
-  cleanContent(text) {
-    return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
-  }
-  /**
-   * Convert URL to a stable document ID
-   */
   urlToId(url) {
-    return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+    return urlToDocumentId(url);
   }
   /**
    * Delay helper
@@ -2434,5 +2879,16 @@ var WebRAGPlugin = class _WebRAGPlugin {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  WebRAGPlugin
+  WebRAGPlugin,
+  bodyTextLengthHint,
+  extractPageFromHtml,
+  extractProductMetadata,
+  hardExcludePage,
+  inferTypeFromUrl,
+  normalizeAvailability,
+  normalizeCurrency,
+  normalizeDisplayTitle,
+  parsePrice,
+  resolvePageCardMetadata,
+  urlToDocumentId
 });