@snap-agent/rag-web 0.1.5 → 0.1.6

package/dist/index.d.mts

@@ -690,21 +690,8 @@ declare class WebRAGPlugin implements RAGPlugin {
      * Crawl a single page and extract content
      */
     private crawlPage;
-    /**
-     * Default chain works for many WordPress / Elementor / block themes where `.first()`
-     * would otherwise hit an empty wrapper.
-     */
-    private static readonly DEFAULT_CONTENT_SELECTOR;
-    private stripNoiseFromDom;
-    /** Longest cleaned text among selector matches and full body (after noise strip). */
-    private extractBestContentText;
     private bodyTextLengthHint;
     private extractDocumentFromHtml;
-    /**
-     * Fallback image extraction: finds the first meaningful image in the content area.
-     * Skips icons, avatars, and tiny assets by filtering on common patterns.
-     */
-    private extractHeroImage;
     private looksLikeDynamicShell;
     private diagFromRenderedAttempt;
     private crawlPageSmart;
@@ -717,10 +704,6 @@ declare class WebRAGPlugin implements RAGPlugin {
     /**
      * Clean extracted text content
      */
-    private cleanContent;
-    /**
-     * Convert URL to a stable document ID
-     */
     private urlToId;
     /**
      * Delay helper
@@ -783,4 +766,43 @@ declare class WebRAGPlugin implements RAGPlugin {
     getConfig(): Record<string, any>;
 }
 
-export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig };
+interface HtmlPageExtractOptions {
+    titleSelector?: string;
+    contentSelector?: string;
+    removeSelectors?: string[];
+    defaultType?: string;
+    typeFromUrl?: Record<string, string>;
+    minExtractedContentLength?: number;
+    metadata?: Record<string, unknown>;
+}
+interface HtmlPageExtractResult {
+    id: string;
+    metadata: Record<string, unknown>;
+    content: string;
+    /** True when content meets minExtractedContentLength (default 50). */
+    indexable: boolean;
+    contentPreview: string;
+}
+declare function urlToDocumentId(url: string): string;
+declare function bodyTextLengthHint(html: string, options?: HtmlPageExtractOptions): number;
+/**
+ * Extract full page metadata + main content the same way web-rag does on HTML ingest.
+ * Unlike ingest, always returns metadata even when content is too short to index.
+ */
+declare function extractPageFromHtml(url: string, html: string, options?: HtmlPageExtractOptions): HtmlPageExtractResult;
+
+interface ProductMetadata {
+    price?: number;
+    currency?: string;
+    availability?: string;
+}
+/**
+ * Extract structured product fields from HTML (JSON-LD, Open Graph, microdata).
+ * Per-field priority: JSON-LD → Open Graph → microdata.
+ */
+declare function extractProductMetadata(html: string): ProductMetadata;
+declare function parsePrice(value: unknown): number | undefined;
+declare function normalizeCurrency(value: unknown): string | undefined;
+declare function normalizeAvailability(value: unknown): string | undefined;
+
+export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, normalizeAvailability, normalizeCurrency, parsePrice, urlToDocumentId };

package/dist/index.d.ts

@@ -690,21 +690,8 @@ declare class WebRAGPlugin implements RAGPlugin {
      * Crawl a single page and extract content
      */
     private crawlPage;
-    /**
-     * Default chain works for many WordPress / Elementor / block themes where `.first()`
-     * would otherwise hit an empty wrapper.
-     */
-    private static readonly DEFAULT_CONTENT_SELECTOR;
-    private stripNoiseFromDom;
-    /** Longest cleaned text among selector matches and full body (after noise strip). */
-    private extractBestContentText;
     private bodyTextLengthHint;
     private extractDocumentFromHtml;
-    /**
-     * Fallback image extraction: finds the first meaningful image in the content area.
-     * Skips icons, avatars, and tiny assets by filtering on common patterns.
-     */
-    private extractHeroImage;
     private looksLikeDynamicShell;
     private diagFromRenderedAttempt;
     private crawlPageSmart;
@@ -717,10 +704,6 @@ declare class WebRAGPlugin implements RAGPlugin {
     /**
      * Clean extracted text content
      */
-    private cleanContent;
-    /**
-     * Convert URL to a stable document ID
-     */
     private urlToId;
     /**
      * Delay helper
@@ -783,4 +766,43 @@ declare class WebRAGPlugin implements RAGPlugin {
     getConfig(): Record<string, any>;
 }
 
-export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig };
+interface HtmlPageExtractOptions {
+    titleSelector?: string;
+    contentSelector?: string;
+    removeSelectors?: string[];
+    defaultType?: string;
+    typeFromUrl?: Record<string, string>;
+    minExtractedContentLength?: number;
+    metadata?: Record<string, unknown>;
+}
+interface HtmlPageExtractResult {
+    id: string;
+    metadata: Record<string, unknown>;
+    content: string;
+    /** True when content meets minExtractedContentLength (default 50). */
+    indexable: boolean;
+    contentPreview: string;
+}
+declare function urlToDocumentId(url: string): string;
+declare function bodyTextLengthHint(html: string, options?: HtmlPageExtractOptions): number;
+/**
+ * Extract full page metadata + main content the same way web-rag does on HTML ingest.
+ * Unlike ingest, always returns metadata even when content is too short to index.
+ */
+declare function extractPageFromHtml(url: string, html: string, options?: HtmlPageExtractOptions): HtmlPageExtractResult;
+
+interface ProductMetadata {
+    price?: number;
+    currency?: string;
+    availability?: string;
+}
+/**
+ * Extract structured product fields from HTML (JSON-LD, Open Graph, microdata).
+ * Per-field priority: JSON-LD → Open Graph → microdata.
+ */
+declare function extractProductMetadata(html: string): ProductMetadata;
+declare function parsePrice(value: unknown): number | undefined;
+declare function normalizeCurrency(value: unknown): string | undefined;
+declare function normalizeAvailability(value: unknown): string | undefined;
+
+export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, normalizeAvailability, normalizeCurrency, parsePrice, urlToDocumentId };

package/dist/index.js

@@ -30,16 +30,319 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
-  WebRAGPlugin: () => WebRAGPlugin
+  WebRAGPlugin: () => WebRAGPlugin,
+  bodyTextLengthHint: () => bodyTextLengthHint,
+  extractPageFromHtml: () => extractPageFromHtml,
+  extractProductMetadata: () => extractProductMetadata,
+  normalizeAvailability: () => normalizeAvailability,
+  normalizeCurrency: () => normalizeCurrency,
+  parsePrice: () => parsePrice,
+  urlToDocumentId: () => urlToDocumentId
 });
 module.exports = __toCommonJS(index_exports);
 
 // src/WebRAGPlugin.ts
 var import_mongodb = require("mongodb");
 var import_openai = __toESM(require("openai"));
-var cheerio = __toESM(require("cheerio"));
+var cheerio3 = __toESM(require("cheerio"));
 var fs = __toESM(require("fs"));
 var path = __toESM(require("path"));
+
+// src/htmlPageExtract.ts
+var cheerio2 = __toESM(require("cheerio"));
+
+// src/productMetadata.ts
+var cheerio = __toESM(require("cheerio"));
+function extractProductMetadata(html) {
+  const $ = cheerio.load(html);
+  const fromJsonLd = extractFromJsonLd($);
+  const fromOg = extractFromOpenGraph($);
+  const fromMicrodata = extractFromMicrodata($);
+  const result = {};
+  const price = fromJsonLd.price ?? fromOg.price ?? fromMicrodata.price;
+  if (price != null) result.price = price;
+  const currency = fromJsonLd.currency ?? fromOg.currency ?? fromMicrodata.currency;
+  if (currency) result.currency = currency;
+  const availability = fromJsonLd.availability ?? fromOg.availability ?? fromMicrodata.availability;
+  if (availability) result.availability = availability;
+  return result;
+}
+function extractFromJsonLd($) {
+  const result = {};
+  $('script[type="application/ld+json"]').each((_, el) => {
+    if (result.price != null && result.currency && result.availability) return false;
+    const raw = $(el).html()?.trim();
+    if (!raw) return;
+    let parsed;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      return;
+    }
+    for (const node of collectJsonLdNodes(parsed)) {
+      if (!isProductType(node)) continue;
+      const offer = pickOffer(node);
+      if (!offer) continue;
+      if (result.price == null) {
+        const price = parsePrice(offer.price ?? offer.lowPrice ?? offer.highPrice);
+        if (price != null) result.price = price;
+      }
+      if (!result.currency) {
+        const currency = normalizeCurrency(offer.priceCurrency);
+        if (currency) result.currency = currency;
+      }
+      if (!result.availability) {
+        const availability = normalizeAvailability(offer.availability);
+        if (availability) result.availability = availability;
+      }
+    }
+  });
+  return result;
+}
+function extractFromOpenGraph($) {
+  const result = {};
+  const priceRaw = $('meta[property="product:price:amount"]').attr("content") || $('meta[property="og:price:amount"]').attr("content");
+  const price = parsePrice(priceRaw);
+  if (price != null) result.price = price;
+  const currency = normalizeCurrency(
+    $('meta[property="product:price:currency"]').attr("content") || $('meta[property="og:price:currency"]').attr("content")
+  );
+  if (currency) result.currency = currency;
+  const availability = normalizeAvailability(
+    $('meta[property="product:availability"]').attr("content") || $('meta[property="og:availability"]').attr("content")
+  );
+  if (availability) result.availability = availability;
+  return result;
+}
+function microdataField($, itemprop) {
+  const scope = $('[itemtype*="schema.org/Product"], [itemtype*="schema.org/product"]').first();
+  return scope.length > 0 ? scope.find(`[itemprop="${itemprop}"]`).first() : $(`[itemprop="${itemprop}"]`).first();
+}
+function extractFromMicrodata($) {
+  const result = {};
+  const priceEl = microdataField($, "price");
+  const price = parsePrice(priceEl.attr("content") || priceEl.text());
+  if (price != null) result.price = price;
+  const currencyEl = microdataField($, "priceCurrency");
+  const currency = normalizeCurrency(currencyEl.attr("content") || currencyEl.text());
+  if (currency) result.currency = currency;
+  const availabilityEl = microdataField($, "availability");
+  const availability = normalizeAvailability(
+    availabilityEl.attr("content") || availabilityEl.attr("href") || availabilityEl.text()
+  );
+  if (availability) result.availability = availability;
+  return result;
+}
+function collectJsonLdNodes(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function isProductType(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  return types.some((t) => {
+    const s = String(t).toLowerCase();
+    return s === "product" || s.endsWith("/product");
+  });
+}
+function pickOffer(product) {
+  const offers = product.offers;
+  if (offers == null) return null;
+  if (Array.isArray(offers)) {
+    const first = offers.find((o) => o && typeof o === "object");
+    return first ?? null;
+  }
+  if (typeof offers === "object") return offers;
+  return null;
+}
+function parsePrice(value) {
+  if (value == null || value === "") return void 0;
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  let s = String(value).trim();
+  if (!s) return void 0;
+  s = s.replace(/[^\d.,\-]/g, "");
+  if (!s || s === "-" || s === ".") return void 0;
+  const lastComma = s.lastIndexOf(",");
+  const lastDot = s.lastIndexOf(".");
+  if (lastComma > -1 && lastDot > -1) {
+    if (lastComma > lastDot) {
+      s = s.replace(/\./g, "").replace(",", ".");
+    } else {
+      s = s.replace(/,/g, "");
+    }
+  } else if (lastComma > -1) {
+    const parts = s.split(",");
+    if (parts.length === 2 && parts[1].length <= 2) {
+      s = parts[0].replace(/\./g, "") + "." + parts[1];
+    } else {
+      s = s.replace(/,/g, "");
+    }
+  }
+  const num = parseFloat(s);
+  return Number.isFinite(num) ? num : void 0;
+}
+function normalizeCurrency(value) {
+  if (value == null) return void 0;
+  const s = String(value).trim().toUpperCase();
+  if (!s) return void 0;
+  const iso = s.match(/[A-Z]{3}/);
+  return iso ? iso[0] : s.length <= 4 ? s : void 0;
+}
+function normalizeAvailability(value) {
+  if (value == null) return void 0;
+  let s = String(value).trim();
+  if (!s) return void 0;
+  if (s.includes("schema.org/")) {
+    const parts = s.split("/");
+    s = parts[parts.length - 1] || s;
+  }
+  s = s.replace(/^https?:\/\/[^/]+\//, "");
+  if (s.includes("/")) {
+    const parts = s.split("/");
+    s = parts[parts.length - 1] || s;
+  }
+  return s.replace(/\s+/g, "") || void 0;
+}
+
+// src/htmlPageExtract.ts
+var DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
+var DEFAULT_REMOVE_SELECTORS = [
+  "script",
+  "style",
+  "nav",
+  "header",
+  "footer",
+  ".sidebar",
+  ".navigation",
+  ".menu",
+  ".comments",
+  '[role="navigation"]',
+  '[role="banner"]'
+];
+function urlToDocumentId(url) {
+  return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+}
+function cleanContent(text) {
+  return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
+}
+function bodyTextLengthHint(html, options = {}) {
+  const $ = cheerio2.load(html);
+  stripNoiseFromDom($, options);
+  return cleanContent($("body").text().trim()).length;
+}
+function extractPageFromHtml(url, html, options = {}) {
+  const $ = cheerio2.load(html);
+  stripNoiseFromDom($, options);
+  const titleSelector = options.titleSelector || "h1, title";
+  let title = $(titleSelector).first().text().trim();
+  if (!title) {
+    title = $("title").text().trim();
+  }
+  const content = extractBestContentText($, options);
+  const minChars = options.minExtractedContentLength ?? 50;
+  const indexable = Boolean(content && content.length >= minChars);
+  const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || extractHeroImage($, url) || void 0;
+  let imageUrl;
+  if (image) {
+    try {
+      imageUrl = new URL(image, url).href;
+    } catch {
+      imageUrl = image;
+    }
+  }
+  const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
+  let type = options.defaultType || "page";
+  if (options.typeFromUrl) {
+    for (const [pattern, typeName] of Object.entries(options.typeFromUrl)) {
+      if (url.includes(pattern)) {
+        type = typeName;
+        break;
+      }
+    }
+  }
+  const productMeta = extractProductMetadata(html);
+  const metadata = {
+    type,
+    ...title ? { title } : {},
+    url,
+    ...imageUrl ? { imageUrl } : {},
+    ...description ? { description } : {},
+    ...productMeta.price != null ? { price: productMeta.price } : {},
+    ...productMeta.currency ? { currency: productMeta.currency } : {},
+    ...productMeta.availability ? { availability: productMeta.availability } : {},
+    ...options.metadata
+  };
+  const previewLen = 400;
+  const contentPreview = content.length > previewLen ? `${content.slice(0, previewLen)}\u2026` : content;
+  return {
+    id: urlToDocumentId(url),
+    metadata,
+    content,
+    indexable,
+    contentPreview
+  };
+}
+function stripNoiseFromDom($, options) {
+  const removeSelectors = options.removeSelectors ?? DEFAULT_REMOVE_SELECTORS;
+  removeSelectors.forEach((selector) => $(selector).remove());
+}
+function extractBestContentText($, options) {
+  const contentSelector = options.contentSelector || DEFAULT_CONTENT_SELECTOR;
+  const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
+  let best = "";
+  for (const sel of selectors) {
+    $(sel).each((_, el) => {
+      const t = cleanContent($(el).text().trim());
+      if (t.length > best.length) best = t;
+    });
+  }
+  const bodyText = cleanContent($("body").text().trim());
+  if (bodyText.length > best.length) best = bodyText;
+  return best;
+}
+function extractHeroImage($, pageUrl) {
+  const containers = $('main, article, [role="main"], #content, .content');
+  const scope = containers.length > 0 ? containers : $("body");
+  let best;
+  scope.find("img[src]").each((_, el) => {
+    if (best) return false;
+    const src = $(el).attr("src") || "";
+    const alt = ($(el).attr("alt") || "").toLowerCase();
+    const width = parseInt($(el).attr("width") || "0", 10);
+    const height = parseInt($(el).attr("height") || "0", 10);
+    if (width > 0 && width < 80 || height > 0 && height < 80) return;
+    if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
+    if (src.startsWith("data:") || src.endsWith(".svg")) return;
+    if (src.includes("/_next/image")) {
+      try {
+        const nextUrl = new URL(src, pageUrl);
+        const realUrl = nextUrl.searchParams.get("url");
+        if (realUrl) {
+          best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
+          return false;
+        }
+      } catch {
+      }
+    }
+    best = src;
+    return false;
+  });
+  return best;
+}
+
+// src/WebRAGPlugin.ts
 function bulkOpCurrentUrl(op) {
   const meta = op.document?.metadata;
   if (typeof meta?.url === "string" && meta.url.trim()) return meta.url.trim();
@@ -58,7 +361,7 @@ function isUrlListingInsert(document) {
     return false;
   }
 }
-var WebRAGPlugin = class _WebRAGPlugin {
+var WebRAGPlugin = class {
   name = "web-rag";
   type = "rag";
   priority;
@@ -288,6 +591,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
           url: doc.metadata.url,
           imageUrl: doc.metadata.imageUrl,
           description: doc.metadata.description,
+          ...doc.metadata.price != null ? { price: doc.metadata.price } : {},
+          ...doc.metadata.currency ? { currency: doc.metadata.currency } : {},
+          ...doc.metadata.availability ? { availability: doc.metadata.availability } : {},
           score: doc.score
         }))
       }
@@ -1458,7 +1764,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return await response.text();
   }
   extractInternalLinks(html, base, stripQueryParams) {
-    const $ = cheerio.load(html);
+    const $ = cheerio3.load(html);
     const links = /* @__PURE__ */ new Set();
     $("a[href]").each((_, el) => {
       const href = ($(el).attr("href") || "").trim();
@@ -1624,7 +1930,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
             }
           }
           try {
-            const { doc, diag, bodyTextLengthHint } = await this.crawlPageSmart(url, config, timeout, {
+            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2 } = await this.crawlPageSmart(url, config, timeout, {
               renderMode,
               renderOptions,
               minContentLength,
@@ -1655,7 +1961,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
               status: crawlSt,
               modeUsed: diag?.modeUsed,
               contentLength: doc?.content?.length,
-              bodyTextLengthHint,
+              bodyTextLengthHint: bodyTextLengthHint2,
               title: doc?.metadata?.title,
               docId: doc?.id,
               error: diag?.errorMessage
@@ -1767,125 +2073,18 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const html = await response.text();
     return this.extractDocumentFromHtml(url, html, config);
   }
-  /**
-   * Default chain works for many WordPress / Elementor / block themes where `.first()`
-   * would otherwise hit an empty wrapper.
-   */
-  static DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
-  stripNoiseFromDom($, config) {
-    const removeSelectors = config.removeSelectors || [
-      "script",
-      "style",
-      "nav",
-      "header",
-      "footer",
-      ".sidebar",
-      ".navigation",
-      ".menu",
-      ".comments",
-      '[role="navigation"]',
-      '[role="banner"]'
-    ];
-    removeSelectors.forEach((selector) => $(selector).remove());
-  }
-  /** Longest cleaned text among selector matches and full body (after noise strip). */
-  extractBestContentText($, config) {
-    const contentSelector = config.contentSelector || _WebRAGPlugin.DEFAULT_CONTENT_SELECTOR;
-    const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
-    let best = "";
-    for (const sel of selectors) {
-      $(sel).each((_, el) => {
-        const t = this.cleanContent($(el).text().trim());
-        if (t.length > best.length) best = t;
-      });
-    }
-    const bodyText = this.cleanContent($("body").text().trim());
-    if (bodyText.length > best.length) best = bodyText;
-    return best;
-  }
   bodyTextLengthHint(html, config) {
-    const $ = cheerio.load(html);
-    this.stripNoiseFromDom($, config);
-    return this.cleanContent($("body").text().trim()).length;
+    return bodyTextLengthHint(html, config);
   }
   extractDocumentFromHtml(url, html, config) {
-    const $ = cheerio.load(html);
-    this.stripNoiseFromDom($, config);
-    const titleSelector = config.titleSelector || "h1, title";
-    let title = $(titleSelector).first().text().trim();
-    if (!title) {
-      title = $("title").text().trim();
-    }
-    const content = this.extractBestContentText($, config);
-    const minChars = config.minExtractedContentLength ?? 50;
-    if (!content || content.length < minChars) return null;
-    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || // Fallback: largest/first meaningful image in main content area
-    this.extractHeroImage($, url) || void 0;
-    let imageUrl;
-    if (image) {
-      try {
-        imageUrl = new URL(image, url).href;
-      } catch {
-        imageUrl = image;
-      }
-    }
-    const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
-    let type = config.defaultType || "page";
-    if (config.typeFromUrl) {
-      for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
-        if (url.includes(pattern)) {
-          type = typeName;
-          break;
-        }
-      }
-    }
-    const id = this.urlToId(url);
+    const extracted = extractPageFromHtml(url, html, config);
+    if (!extracted.indexable) return null;
     return {
-      id,
-      content,
-      metadata: {
-        type,
-        title,
-        url,
-        ...imageUrl ? { imageUrl } : {},
-        ...description ? { description } : {},
-        ...config.metadata
-      }
+      id: extracted.id,
+      content: extracted.content,
+      metadata: extracted.metadata
     };
   }
-  /**
-   * Fallback image extraction: finds the first meaningful image in the content area.
-   * Skips icons, avatars, and tiny assets by filtering on common patterns.
-   */
-  extractHeroImage($, pageUrl) {
-    const containers = $('main, article, [role="main"], #content, .content');
-    const scope = containers.length > 0 ? containers : $("body");
-    let best;
-    scope.find("img[src]").each((_, el) => {
-      if (best) return false;
-      const src = $(el).attr("src") || "";
-      const alt = ($(el).attr("alt") || "").toLowerCase();
-      const width = parseInt($(el).attr("width") || "0", 10);
-      const height = parseInt($(el).attr("height") || "0", 10);
-      if (width > 0 && width < 80 || height > 0 && height < 80) return;
-      if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
-      if (src.startsWith("data:") || src.endsWith(".svg")) return;
-      if (src.includes("/_next/image")) {
-        try {
-          const nextUrl = new URL(src, pageUrl);
-          const realUrl = nextUrl.searchParams.get("url");
-          if (realUrl) {
-            best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
-            return false;
-          }
-        } catch {
-        }
-      }
-      best = src;
-      return false;
-    });
-    return best;
-  }
   looksLikeDynamicShell(html) {
     const lower = html.toLowerCase();
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
@@ -1903,7 +2102,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
     return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
   }
-  diagFromRenderedAttempt(doc, bodyTextLengthHint, renderFailure, blockedSuspected, modeOk, modeFailed) {
+  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed) {
     if (blockedSuspected) {
       return {
         doc: null,
@@ -1919,12 +2118,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return {
       doc,
       diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
-      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2
     };
   }
   async crawlPageSmart(url, config, timeout, ctx) {
     if (ctx.renderMode === true) {
-      const { doc, bodyTextLengthHint, renderFailure, blockedSuspected } = await this.crawlPageRendered(
+      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected } = await this.crawlPageRendered(
         url,
         config,
         timeout,
@@ -1933,7 +2132,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       );
       return this.diagFromRenderedAttempt(
         doc,
-        bodyTextLengthHint,
+        bodyTextLengthHint2,
         renderFailure,
         blockedSuspected,
         "render_ok",
@@ -2050,7 +2249,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
         }
       }
       const html = await page.content();
-      const bodyTextLengthHint = this.bodyTextLengthHint(html, config);
+      const bodyTextLengthHint2 = this.bodyTextLengthHint(html, config);
       const doc = this.extractDocumentFromHtml(url, html, config);
       if (config.debug?.saveDir && config.debug?.enabled) {
         try {
@@ -2065,7 +2264,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
           dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
         }
       }
-      return { doc, bodyTextLengthHint };
+      return { doc, bodyTextLengthHint: bodyTextLengthHint2 };
     } catch (e) {
       const msg = String(e?.message || e || "render_failed");
       const lower = msg.toLowerCase();
@@ -2157,14 +2356,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
   /**
    * Clean extracted text content
    */
-  cleanContent(text) {
-    return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
-  }
-  /**
-   * Convert URL to a stable document ID
-   */
   urlToId(url) {
-    return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+    return urlToDocumentId(url);
   }
   /**
    * Delay helper
@@ -2434,5 +2627,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  WebRAGPlugin
+  WebRAGPlugin,
+  bodyTextLengthHint,
+  extractPageFromHtml,
+  extractProductMetadata,
+  normalizeAvailability,
+  normalizeCurrency,
+  parsePrice,
+  urlToDocumentId
 });

package/dist/index.mjs

@@ -1,9 +1,305 @@
 // src/WebRAGPlugin.ts
 import { MongoClient } from "mongodb";
 import OpenAI from "openai";
-import * as cheerio from "cheerio";
+import * as cheerio3 from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
+
+// src/htmlPageExtract.ts
+import * as cheerio2 from "cheerio";
+
+// src/productMetadata.ts
+import * as cheerio from "cheerio";
+function extractProductMetadata(html) {
+  const $ = cheerio.load(html);
+  const fromJsonLd = extractFromJsonLd($);
+  const fromOg = extractFromOpenGraph($);
+  const fromMicrodata = extractFromMicrodata($);
+  const result = {};
+  const price = fromJsonLd.price ?? fromOg.price ?? fromMicrodata.price;
+  if (price != null) result.price = price;
+  const currency = fromJsonLd.currency ?? fromOg.currency ?? fromMicrodata.currency;
+  if (currency) result.currency = currency;
+  const availability = fromJsonLd.availability ?? fromOg.availability ?? fromMicrodata.availability;
+  if (availability) result.availability = availability;
+  return result;
+}
+function extractFromJsonLd($) {
+  const result = {};
+  $('script[type="application/ld+json"]').each((_, el) => {
+    if (result.price != null && result.currency && result.availability) return false;
+    const raw = $(el).html()?.trim();
+    if (!raw) return;
+    let parsed;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      return;
+    }
+    for (const node of collectJsonLdNodes(parsed)) {
+      if (!isProductType(node)) continue;
+      const offer = pickOffer(node);
+      if (!offer) continue;
+      if (result.price == null) {
+        const price = parsePrice(offer.price ?? offer.lowPrice ?? offer.highPrice);
+        if (price != null) result.price = price;
+      }
+      if (!result.currency) {
+        const currency = normalizeCurrency(offer.priceCurrency);
+        if (currency) result.currency = currency;
+      }
+      if (!result.availability) {
+        const availability = normalizeAvailability(offer.availability);
+        if (availability) result.availability = availability;
+      }
+    }
+  });
+  return result;
+}
+function extractFromOpenGraph($) {
+  const result = {};
+  const priceRaw = $('meta[property="product:price:amount"]').attr("content") || $('meta[property="og:price:amount"]').attr("content");
+  const price = parsePrice(priceRaw);
+  if (price != null) result.price = price;
+  const currency = normalizeCurrency(
+    $('meta[property="product:price:currency"]').attr("content") || $('meta[property="og:price:currency"]').attr("content")
+  );
+  if (currency) result.currency = currency;
+  const availability = normalizeAvailability(
+    $('meta[property="product:availability"]').attr("content") || $('meta[property="og:availability"]').attr("content")
+  );
+  if (availability) result.availability = availability;
+  return result;
+}
+function microdataField($, itemprop) {
+  const scope = $('[itemtype*="schema.org/Product"], [itemtype*="schema.org/product"]').first();
+  return scope.length > 0 ? scope.find(`[itemprop="${itemprop}"]`).first() : $(`[itemprop="${itemprop}"]`).first();
+}
+function extractFromMicrodata($) {
+  const result = {};
+  const priceEl = microdataField($, "price");
+  const price = parsePrice(priceEl.attr("content") || priceEl.text());
+  if (price != null) result.price = price;
+  const currencyEl = microdataField($, "priceCurrency");
+  const currency = normalizeCurrency(currencyEl.attr("content") || currencyEl.text());
+  if (currency) result.currency = currency;
+  const availabilityEl = microdataField($, "availability");
+  const availability = normalizeAvailability(
+    availabilityEl.attr("content") || availabilityEl.attr("href") || availabilityEl.text()
+  );
+  if (availability) result.availability = availability;
+  return result;
+}
+function collectJsonLdNodes(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function isProductType(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  return types.some((t) => {
+    const s = String(t).toLowerCase();
+    return s === "product" || s.endsWith("/product");
+  });
+}
+function pickOffer(product) {
+  const offers = product.offers;
+  if (offers == null) return null;
+  if (Array.isArray(offers)) {
+    const first = offers.find((o) => o && typeof o === "object");
+    return first ?? null;
+  }
+  if (typeof offers === "object") return offers;
+  return null;
+}
+function parsePrice(value) {
+  if (value == null || value === "") return void 0;
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  let s = String(value).trim();
+  if (!s) return void 0;
+  s = s.replace(/[^\d.,\-]/g, "");
+  if (!s || s === "-" || s === ".") return void 0;
+  const lastComma = s.lastIndexOf(",");
+  const lastDot = s.lastIndexOf(".");
+  if (lastComma > -1 && lastDot > -1) {
+    if (lastComma > lastDot) {
+      s = s.replace(/\./g, "").replace(",", ".");
+    } else {
+      s = s.replace(/,/g, "");
+    }
+  } else if (lastComma > -1) {
+    const parts = s.split(",");
+    if (parts.length === 2 && parts[1].length <= 2) {
+      s = parts[0].replace(/\./g, "") + "." + parts[1];
+    } else {
+      s = s.replace(/,/g, "");
+    }
+  }
+  const num = parseFloat(s);
+  return Number.isFinite(num) ? num : void 0;
+}
+function normalizeCurrency(value) {
+  if (value == null) return void 0;
+  const s = String(value).trim().toUpperCase();
+  if (!s) return void 0;
+  const iso = s.match(/[A-Z]{3}/);
+  return iso ? iso[0] : s.length <= 4 ? s : void 0;
+}
+function normalizeAvailability(value) {
+  if (value == null) return void 0;
+  let s = String(value).trim();
+  if (!s) return void 0;
+  if (s.includes("schema.org/")) {
+    const parts = s.split("/");
+    s = parts[parts.length - 1] || s;
+  }
+  s = s.replace(/^https?:\/\/[^/]+\//, "");
+  if (s.includes("/")) {
+    const parts = s.split("/");
+    s = parts[parts.length - 1] || s;
+  }
+  return s.replace(/\s+/g, "") || void 0;
+}
+
+// src/htmlPageExtract.ts
+var DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
+var DEFAULT_REMOVE_SELECTORS = [
+  "script",
+  "style",
+  "nav",
+  "header",
+  "footer",
+  ".sidebar",
+  ".navigation",
+  ".menu",
+  ".comments",
+  '[role="navigation"]',
+  '[role="banner"]'
+];
+function urlToDocumentId(url) {
+  return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+}
+function cleanContent(text) {
+  return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
+}
+function bodyTextLengthHint(html, options = {}) {
+  const $ = cheerio2.load(html);
+  stripNoiseFromDom($, options);
+  return cleanContent($("body").text().trim()).length;
+}
+function extractPageFromHtml(url, html, options = {}) {
+  const $ = cheerio2.load(html);
+  stripNoiseFromDom($, options);
+  const titleSelector = options.titleSelector || "h1, title";
+  let title = $(titleSelector).first().text().trim();
+  if (!title) {
+    title = $("title").text().trim();
+  }
+  const content = extractBestContentText($, options);
+  const minChars = options.minExtractedContentLength ?? 50;
+  const indexable = Boolean(content && content.length >= minChars);
+  const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || extractHeroImage($, url) || void 0;
+  let imageUrl;
+  if (image) {
+    try {
+      imageUrl = new URL(image, url).href;
+    } catch {
+      imageUrl = image;
+    }
+  }
+  const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
+  let type = options.defaultType || "page";
+  if (options.typeFromUrl) {
+    for (const [pattern, typeName] of Object.entries(options.typeFromUrl)) {
+      if (url.includes(pattern)) {
+        type = typeName;
+        break;
+      }
+    }
+  }
+  const productMeta = extractProductMetadata(html);
+  const metadata = {
+    type,
+    ...title ? { title } : {},
+    url,
+    ...imageUrl ? { imageUrl } : {},
+    ...description ? { description } : {},
+    ...productMeta.price != null ? { price: productMeta.price } : {},
+    ...productMeta.currency ? { currency: productMeta.currency } : {},
+    ...productMeta.availability ? { availability: productMeta.availability } : {},
+    ...options.metadata
+  };
+  const previewLen = 400;
+  const contentPreview = content.length > previewLen ? `${content.slice(0, previewLen)}\u2026` : content;
+  return {
+    id: urlToDocumentId(url),
+    metadata,
+    content,
+    indexable,
+    contentPreview
+  };
+}
+function stripNoiseFromDom($, options) {
+  const removeSelectors = options.removeSelectors ?? DEFAULT_REMOVE_SELECTORS;
+  removeSelectors.forEach((selector) => $(selector).remove());
+}
+function extractBestContentText($, options) {
+  const contentSelector = options.contentSelector || DEFAULT_CONTENT_SELECTOR;
+  const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
+  let best = "";
+  for (const sel of selectors) {
+    $(sel).each((_, el) => {
+      const t = cleanContent($(el).text().trim());
+      if (t.length > best.length) best = t;
+    });
+  }
+  const bodyText = cleanContent($("body").text().trim());
+  if (bodyText.length > best.length) best = bodyText;
+  return best;
+}
+function extractHeroImage($, pageUrl) {
+  const containers = $('main, article, [role="main"], #content, .content');
+  const scope = containers.length > 0 ? containers : $("body");
+  let best;
+  scope.find("img[src]").each((_, el) => {
+    if (best) return false;
+    const src = $(el).attr("src") || "";
+    const alt = ($(el).attr("alt") || "").toLowerCase();
+    const width = parseInt($(el).attr("width") || "0", 10);
+    const height = parseInt($(el).attr("height") || "0", 10);
+    if (width > 0 && width < 80 || height > 0 && height < 80) return;
+    if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
+    if (src.startsWith("data:") || src.endsWith(".svg")) return;
+    if (src.includes("/_next/image")) {
+      try {
+        const nextUrl = new URL(src, pageUrl);
+        const realUrl = nextUrl.searchParams.get("url");
+        if (realUrl) {
+          best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
+          return false;
+        }
+      } catch {
+      }
+    }
+    best = src;
+    return false;
+  });
+  return best;
+}
+
+// src/WebRAGPlugin.ts
 function bulkOpCurrentUrl(op) {
   const meta = op.document?.metadata;
   if (typeof meta?.url === "string" && meta.url.trim()) return meta.url.trim();
@@ -22,7 +318,7 @@ function isUrlListingInsert(document) {
     return false;
   }
 }
-var WebRAGPlugin = class _WebRAGPlugin {
+var WebRAGPlugin = class {
   name = "web-rag";
   type = "rag";
   priority;
@@ -252,6 +548,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
           url: doc.metadata.url,
           imageUrl: doc.metadata.imageUrl,
           description: doc.metadata.description,
+          ...doc.metadata.price != null ? { price: doc.metadata.price } : {},
+          ...doc.metadata.currency ? { currency: doc.metadata.currency } : {},
+          ...doc.metadata.availability ? { availability: doc.metadata.availability } : {},
           score: doc.score
         }))
       }
@@ -1422,7 +1721,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return await response.text();
   }
   extractInternalLinks(html, base, stripQueryParams) {
-    const $ = cheerio.load(html);
+    const $ = cheerio3.load(html);
     const links = /* @__PURE__ */ new Set();
     $("a[href]").each((_, el) => {
       const href = ($(el).attr("href") || "").trim();
@@ -1588,7 +1887,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
             }
           }
           try {
-            const { doc, diag, bodyTextLengthHint } = await this.crawlPageSmart(url, config, timeout, {
+            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2 } = await this.crawlPageSmart(url, config, timeout, {
               renderMode,
               renderOptions,
               minContentLength,
@@ -1619,7 +1918,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
               status: crawlSt,
               modeUsed: diag?.modeUsed,
               contentLength: doc?.content?.length,
-              bodyTextLengthHint,
+              bodyTextLengthHint: bodyTextLengthHint2,
               title: doc?.metadata?.title,
               docId: doc?.id,
               error: diag?.errorMessage
@@ -1731,125 +2030,18 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const html = await response.text();
     return this.extractDocumentFromHtml(url, html, config);
   }
-  /**
-   * Default chain works for many WordPress / Elementor / block themes where `.first()`
-   * would otherwise hit an empty wrapper.
-   */
-  static DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
-  stripNoiseFromDom($, config) {
-    const removeSelectors = config.removeSelectors || [
-      "script",
-      "style",
-      "nav",
-      "header",
-      "footer",
-      ".sidebar",
-      ".navigation",
-      ".menu",
-      ".comments",
-      '[role="navigation"]',
-      '[role="banner"]'
-    ];
-    removeSelectors.forEach((selector) => $(selector).remove());
-  }
-  /** Longest cleaned text among selector matches and full body (after noise strip). */
-  extractBestContentText($, config) {
-    const contentSelector = config.contentSelector || _WebRAGPlugin.DEFAULT_CONTENT_SELECTOR;
-    const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
-    let best = "";
-    for (const sel of selectors) {
-      $(sel).each((_, el) => {
-        const t = this.cleanContent($(el).text().trim());
-        if (t.length > best.length) best = t;
-      });
-    }
-    const bodyText = this.cleanContent($("body").text().trim());
-    if (bodyText.length > best.length) best = bodyText;
-    return best;
-  }
   bodyTextLengthHint(html, config) {
-    const $ = cheerio.load(html);
-    this.stripNoiseFromDom($, config);
-    return this.cleanContent($("body").text().trim()).length;
+    return bodyTextLengthHint(html, config);
   }
   extractDocumentFromHtml(url, html, config) {
-    const $ = cheerio.load(html);
-    this.stripNoiseFromDom($, config);
-    const titleSelector = config.titleSelector || "h1, title";
-    let title = $(titleSelector).first().text().trim();
-    if (!title) {
-      title = $("title").text().trim();
-    }
-    const content = this.extractBestContentText($, config);
-    const minChars = config.minExtractedContentLength ?? 50;
-    if (!content || content.length < minChars) return null;
-    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || // Fallback: largest/first meaningful image in main content area
-    this.extractHeroImage($, url) || void 0;
-    let imageUrl;
-    if (image) {
-      try {
-        imageUrl = new URL(image, url).href;
-      } catch {
-        imageUrl = image;
-      }
-    }
-    const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
-    let type = config.defaultType || "page";
-    if (config.typeFromUrl) {
-      for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
-        if (url.includes(pattern)) {
-          type = typeName;
-          break;
-        }
-      }
-    }
-    const id = this.urlToId(url);
+    const extracted = extractPageFromHtml(url, html, config);
+    if (!extracted.indexable) return null;
     return {
-      id,
-      content,
-      metadata: {
-        type,
-        title,
-        url,
-        ...imageUrl ? { imageUrl } : {},
-        ...description ? { description } : {},
-        ...config.metadata
-      }
+      id: extracted.id,
+      content: extracted.content,
+      metadata: extracted.metadata
     };
   }
-  /**
-   * Fallback image extraction: finds the first meaningful image in the content area.
-   * Skips icons, avatars, and tiny assets by filtering on common patterns.
-   */
-  extractHeroImage($, pageUrl) {
-    const containers = $('main, article, [role="main"], #content, .content');
-    const scope = containers.length > 0 ? containers : $("body");
-    let best;
-    scope.find("img[src]").each((_, el) => {
-      if (best) return false;
-      const src = $(el).attr("src") || "";
-      const alt = ($(el).attr("alt") || "").toLowerCase();
-      const width = parseInt($(el).attr("width") || "0", 10);
-      const height = parseInt($(el).attr("height") || "0", 10);
-      if (width > 0 && width < 80 || height > 0 && height < 80) return;
-      if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
-      if (src.startsWith("data:") || src.endsWith(".svg")) return;
-      if (src.includes("/_next/image")) {
-        try {
-          const nextUrl = new URL(src, pageUrl);
-          const realUrl = nextUrl.searchParams.get("url");
-          if (realUrl) {
-            best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
-            return false;
-          }
-        } catch {
-        }
-      }
-      best = src;
-      return false;
-    });
-    return best;
-  }
   looksLikeDynamicShell(html) {
     const lower = html.toLowerCase();
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
@@ -1867,7 +2059,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
     return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
   }
-  diagFromRenderedAttempt(doc, bodyTextLengthHint, renderFailure, blockedSuspected, modeOk, modeFailed) {
+  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed) {
     if (blockedSuspected) {
       return {
         doc: null,
@@ -1883,12 +2075,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return {
       doc,
       diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
-      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2
     };
   }
   async crawlPageSmart(url, config, timeout, ctx) {
     if (ctx.renderMode === true) {
-      const { doc, bodyTextLengthHint, renderFailure, blockedSuspected } = await this.crawlPageRendered(
+      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected } = await this.crawlPageRendered(
         url,
         config,
         timeout,
@@ -1897,7 +2089,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       );
       return this.diagFromRenderedAttempt(
         doc,
-        bodyTextLengthHint,
+        bodyTextLengthHint2,
         renderFailure,
         blockedSuspected,
         "render_ok",
@@ -2014,7 +2206,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
         }
       }
       const html = await page.content();
-      const bodyTextLengthHint = this.bodyTextLengthHint(html, config);
+      const bodyTextLengthHint2 = this.bodyTextLengthHint(html, config);
       const doc = this.extractDocumentFromHtml(url, html, config);
       if (config.debug?.saveDir && config.debug?.enabled) {
         try {
@@ -2029,7 +2221,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
           dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
         }
       }
-      return { doc, bodyTextLengthHint };
+      return { doc, bodyTextLengthHint: bodyTextLengthHint2 };
     } catch (e) {
       const msg = String(e?.message || e || "render_failed");
       const lower = msg.toLowerCase();
@@ -2121,14 +2313,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
   /**
    * Clean extracted text content
    */
-  cleanContent(text) {
-    return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
-  }
-  /**
-   * Convert URL to a stable document ID
-   */
   urlToId(url) {
-    return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+    return urlToDocumentId(url);
   }
   /**
    * Delay helper
@@ -2397,5 +2583,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
   }
 };
 export {
-  WebRAGPlugin
+  WebRAGPlugin,
+  bodyTextLengthHint,
+  extractPageFromHtml,
+  extractProductMetadata,
+  normalizeAvailability,
+  normalizeCurrency,
+  parsePrice,
+  urlToDocumentId
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snap-agent/rag-web",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Web RAG plugin for SnapAgent SDK - Schema-agnostic content search via web crawling, CMS APIs, sitemaps, and RSS feeds",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",