@snap-agent/rag-web 0.1.4 → 0.1.6

package/dist/index.mjs

@@ -1,10 +1,324 @@
 // src/WebRAGPlugin.ts
 import { MongoClient } from "mongodb";
 import OpenAI from "openai";
-import * as cheerio from "cheerio";
+import * as cheerio3 from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
-var WebRAGPlugin = class _WebRAGPlugin {
+
+// src/htmlPageExtract.ts
+import * as cheerio2 from "cheerio";
+
+// src/productMetadata.ts
+import * as cheerio from "cheerio";
+function extractProductMetadata(html) {
+  const $ = cheerio.load(html);
+  const fromJsonLd = extractFromJsonLd($);
+  const fromOg = extractFromOpenGraph($);
+  const fromMicrodata = extractFromMicrodata($);
+  const result = {};
+  const price = fromJsonLd.price ?? fromOg.price ?? fromMicrodata.price;
+  if (price != null) result.price = price;
+  const currency = fromJsonLd.currency ?? fromOg.currency ?? fromMicrodata.currency;
+  if (currency) result.currency = currency;
+  const availability = fromJsonLd.availability ?? fromOg.availability ?? fromMicrodata.availability;
+  if (availability) result.availability = availability;
+  return result;
+}
+function extractFromJsonLd($) {
+  const result = {};
+  $('script[type="application/ld+json"]').each((_, el) => {
+    if (result.price != null && result.currency && result.availability) return false;
+    const raw = $(el).html()?.trim();
+    if (!raw) return;
+    let parsed;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      return;
+    }
+    for (const node of collectJsonLdNodes(parsed)) {
+      if (!isProductType(node)) continue;
+      const offer = pickOffer(node);
+      if (!offer) continue;
+      if (result.price == null) {
+        const price = parsePrice(offer.price ?? offer.lowPrice ?? offer.highPrice);
+        if (price != null) result.price = price;
+      }
+      if (!result.currency) {
+        const currency = normalizeCurrency(offer.priceCurrency);
+        if (currency) result.currency = currency;
+      }
+      if (!result.availability) {
+        const availability = normalizeAvailability(offer.availability);
+        if (availability) result.availability = availability;
+      }
+    }
+  });
+  return result;
+}
+function extractFromOpenGraph($) {
+  const result = {};
+  const priceRaw = $('meta[property="product:price:amount"]').attr("content") || $('meta[property="og:price:amount"]').attr("content");
+  const price = parsePrice(priceRaw);
+  if (price != null) result.price = price;
+  const currency = normalizeCurrency(
+    $('meta[property="product:price:currency"]').attr("content") || $('meta[property="og:price:currency"]').attr("content")
+  );
+  if (currency) result.currency = currency;
+  const availability = normalizeAvailability(
+    $('meta[property="product:availability"]').attr("content") || $('meta[property="og:availability"]').attr("content")
+  );
+  if (availability) result.availability = availability;
+  return result;
+}
+function microdataField($, itemprop) {
+  const scope = $('[itemtype*="schema.org/Product"], [itemtype*="schema.org/product"]').first();
+  return scope.length > 0 ? scope.find(`[itemprop="${itemprop}"]`).first() : $(`[itemprop="${itemprop}"]`).first();
+}
+function extractFromMicrodata($) {
+  const result = {};
+  const priceEl = microdataField($, "price");
+  const price = parsePrice(priceEl.attr("content") || priceEl.text());
+  if (price != null) result.price = price;
+  const currencyEl = microdataField($, "priceCurrency");
+  const currency = normalizeCurrency(currencyEl.attr("content") || currencyEl.text());
+  if (currency) result.currency = currency;
+  const availabilityEl = microdataField($, "availability");
+  const availability = normalizeAvailability(
+    availabilityEl.attr("content") || availabilityEl.attr("href") || availabilityEl.text()
+  );
+  if (availability) result.availability = availability;
+  return result;
+}
+function collectJsonLdNodes(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function isProductType(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  return types.some((t) => {
+    const s = String(t).toLowerCase();
+    return s === "product" || s.endsWith("/product");
+  });
+}
+function pickOffer(product) {
+  const offers = product.offers;
+  if (offers == null) return null;
+  if (Array.isArray(offers)) {
+    const first = offers.find((o) => o && typeof o === "object");
+    return first ?? null;
+  }
+  if (typeof offers === "object") return offers;
+  return null;
+}
+function parsePrice(value) {
+  if (value == null || value === "") return void 0;
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  let s = String(value).trim();
+  if (!s) return void 0;
+  s = s.replace(/[^\d.,\-]/g, "");
+  if (!s || s === "-" || s === ".") return void 0;
+  const lastComma = s.lastIndexOf(",");
+  const lastDot = s.lastIndexOf(".");
+  if (lastComma > -1 && lastDot > -1) {
+    if (lastComma > lastDot) {
+      s = s.replace(/\./g, "").replace(",", ".");
+    } else {
+      s = s.replace(/,/g, "");
+    }
+  } else if (lastComma > -1) {
+    const parts = s.split(",");
+    if (parts.length === 2 && parts[1].length <= 2) {
+      s = parts[0].replace(/\./g, "") + "." + parts[1];
+    } else {
+      s = s.replace(/,/g, "");
+    }
+  }
+  const num = parseFloat(s);
+  return Number.isFinite(num) ? num : void 0;
+}
+function normalizeCurrency(value) {
+  if (value == null) return void 0;
+  const s = String(value).trim().toUpperCase();
+  if (!s) return void 0;
+  const iso = s.match(/[A-Z]{3}/);
+  return iso ? iso[0] : s.length <= 4 ? s : void 0;
+}
+function normalizeAvailability(value) {
+  if (value == null) return void 0;
+  let s = String(value).trim();
+  if (!s) return void 0;
+  if (s.includes("schema.org/")) {
+    const parts = s.split("/");
+    s = parts[parts.length - 1] || s;
+  }
+  s = s.replace(/^https?:\/\/[^/]+\//, "");
+  if (s.includes("/")) {
+    const parts = s.split("/");
+    s = parts[parts.length - 1] || s;
+  }
+  return s.replace(/\s+/g, "") || void 0;
+}
+
+// src/htmlPageExtract.ts
+var DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
+var DEFAULT_REMOVE_SELECTORS = [
+  "script",
+  "style",
+  "nav",
+  "header",
+  "footer",
+  ".sidebar",
+  ".navigation",
+  ".menu",
+  ".comments",
+  '[role="navigation"]',
+  '[role="banner"]'
+];
+function urlToDocumentId(url) {
+  return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+}
+function cleanContent(text) {
+  return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
+}
+function bodyTextLengthHint(html, options = {}) {
+  const $ = cheerio2.load(html);
+  stripNoiseFromDom($, options);
+  return cleanContent($("body").text().trim()).length;
+}
+function extractPageFromHtml(url, html, options = {}) {
+  const $ = cheerio2.load(html);
+  stripNoiseFromDom($, options);
+  const titleSelector = options.titleSelector || "h1, title";
+  let title = $(titleSelector).first().text().trim();
+  if (!title) {
+    title = $("title").text().trim();
+  }
+  const content = extractBestContentText($, options);
+  const minChars = options.minExtractedContentLength ?? 50;
+  const indexable = Boolean(content && content.length >= minChars);
+  const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || extractHeroImage($, url) || void 0;
+  let imageUrl;
+  if (image) {
+    try {
+      imageUrl = new URL(image, url).href;
+    } catch {
+      imageUrl = image;
+    }
+  }
+  const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
+  let type = options.defaultType || "page";
+  if (options.typeFromUrl) {
+    for (const [pattern, typeName] of Object.entries(options.typeFromUrl)) {
+      if (url.includes(pattern)) {
+        type = typeName;
+        break;
+      }
+    }
+  }
+  const productMeta = extractProductMetadata(html);
+  const metadata = {
+    type,
+    ...title ? { title } : {},
+    url,
+    ...imageUrl ? { imageUrl } : {},
+    ...description ? { description } : {},
+    ...productMeta.price != null ? { price: productMeta.price } : {},
+    ...productMeta.currency ? { currency: productMeta.currency } : {},
+    ...productMeta.availability ? { availability: productMeta.availability } : {},
+    ...options.metadata
+  };
+  const previewLen = 400;
+  const contentPreview = content.length > previewLen ? `${content.slice(0, previewLen)}\u2026` : content;
+  return {
+    id: urlToDocumentId(url),
+    metadata,
+    content,
+    indexable,
+    contentPreview
+  };
+}
+function stripNoiseFromDom($, options) {
+  const removeSelectors = options.removeSelectors ?? DEFAULT_REMOVE_SELECTORS;
+  removeSelectors.forEach((selector) => $(selector).remove());
+}
+function extractBestContentText($, options) {
+  const contentSelector = options.contentSelector || DEFAULT_CONTENT_SELECTOR;
+  const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
+  let best = "";
+  for (const sel of selectors) {
+    $(sel).each((_, el) => {
+      const t = cleanContent($(el).text().trim());
+      if (t.length > best.length) best = t;
+    });
+  }
+  const bodyText = cleanContent($("body").text().trim());
+  if (bodyText.length > best.length) best = bodyText;
+  return best;
+}
+function extractHeroImage($, pageUrl) {
+  const containers = $('main, article, [role="main"], #content, .content');
+  const scope = containers.length > 0 ? containers : $("body");
+  let best;
+  scope.find("img[src]").each((_, el) => {
+    if (best) return false;
+    const src = $(el).attr("src") || "";
+    const alt = ($(el).attr("alt") || "").toLowerCase();
+    const width = parseInt($(el).attr("width") || "0", 10);
+    const height = parseInt($(el).attr("height") || "0", 10);
+    if (width > 0 && width < 80 || height > 0 && height < 80) return;
+    if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
+    if (src.startsWith("data:") || src.endsWith(".svg")) return;
+    if (src.includes("/_next/image")) {
+      try {
+        const nextUrl = new URL(src, pageUrl);
+        const realUrl = nextUrl.searchParams.get("url");
+        if (realUrl) {
+          best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
+          return false;
+        }
+      } catch {
+      }
+    }
+    best = src;
+    return false;
+  });
+  return best;
+}
+
+// src/WebRAGPlugin.ts
+function bulkOpCurrentUrl(op) {
+  const meta = op.document?.metadata;
+  if (typeof meta?.url === "string" && meta.url.trim()) return meta.url.trim();
+  if (typeof meta?.source === "string" && meta.source.trim()) return meta.source.trim();
+  return void 0;
+}
+function isUrlListingInsert(document) {
+  const meta = document.metadata;
+  if (meta?.type !== "url") return false;
+  const url = typeof meta.url === "string" ? meta.url.trim() : "";
+  if (!url) return false;
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+var WebRAGPlugin = class {
   name = "web-rag";
   type = "rag";
   priority;
@@ -42,6 +356,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     }
     return this.db.collection(this.config.collection);
   }
+  ledgerIndexesEnsured = false;
   async getLedgerCollection() {
     if (!this.client) {
       this.client = new MongoClient(this.config.mongoUri);
@@ -49,7 +364,16 @@ var WebRAGPlugin = class _WebRAGPlugin {
       this.db = this.client.db(this.config.dbName);
     }
     const name = this.config.crawlLedger?.collection ?? "web_crawl_ledger";
-    return this.db.collection(name);
+    const col = this.db.collection(name);
+    if (!this.ledgerIndexesEnsured) {
+      this.ledgerIndexesEnsured = true;
+      await col.createIndex(
+        { tenantId: 1, agentId: 1, urlNormalized: 1 },
+        { unique: true }
+      );
+      await col.createIndex({ tenantId: 1, agentId: 1, ingestionId: 1, lastCrawledAt: -1 });
+    }
+    return col;
   }
   /**
    * List recent crawl ledger rows (for dashboards / pagination in the front).
@@ -59,6 +383,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const filter = { tenantId: this.config.tenantId };
     filter.agentId = options.agentId ?? "shared";
     if (options.domain) filter.domain = options.domain;
+    if (options.ingestionId) filter.ingestionId = options.ingestionId;
     if (options.status) filter.lastStatus = options.status;
     const limit = Math.min(Math.max(options.limit ?? 50, 1), 500);
     const skip = Math.max(options.skip ?? 0, 0);
@@ -127,6 +452,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
       lastCrawledAt: now,
       updatedAt: now
     };
+    if (params.ingestionId) {
+      $set.ingestionId = params.ingestionId;
+    }
     if (errMsg !== void 0) {
       $set.errorMessage = errMsg;
     } else if (params.status === "indexed" && params.doc) {
@@ -139,9 +467,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
       $set.docId = params.doc.id;
     } else {
       $set.modeUsed = params.diag?.modeUsed;
-      $set.contentLength = null;
-      $set.title = null;
-      $set.docId = null;
+      $set.contentLength = params.contentLength ?? null;
+      $set.title = params.title ?? null;
+      $set.docId = params.docId ?? null;
     }
     await col.updateOne(
       {
@@ -220,6 +548,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
           url: doc.metadata.url,
           imageUrl: doc.metadata.imageUrl,
           description: doc.metadata.description,
+          ...doc.metadata.price != null ? { price: doc.metadata.price } : {},
+          ...doc.metadata.currency ? { currency: doc.metadata.currency } : {},
+          ...doc.metadata.availability ? { availability: doc.metadata.availability } : {},
           score: doc.score
         }))
       }
@@ -385,9 +716,27 @@ var WebRAGPlugin = class _WebRAGPlugin {
     let indexed = 0;
     const errors = [];
     const agentId = options?.agentId || "shared";
-    for (const doc of documents) {
+    const onCrawlProgress = options?.metadata?.onCrawlProgress;
+    const indexingTotal = documents.length;
+    const chunkPlan = documents.map((doc) => this.chunkContent(doc.content));
+    const chunksTotal = chunkPlan.reduce((sum, chunks) => sum + chunks.length, 0);
+    let chunksProcessed = 0;
+    if (onCrawlProgress && indexingTotal > 0) {
+      this.emitCrawlProgress(
+        { metadata: options?.metadata },
+        {
+          phase: "indexing",
+          urlsScheduled: indexingTotal,
+          pagesProcessed: 0,
+          chunksTotal,
+          chunksProcessed: 0
+        }
+      );
+    }
+    for (let docIndex = 0; docIndex < documents.length; docIndex++) {
+      const doc = documents[docIndex];
+      const chunks = chunkPlan[docIndex];
       try {
-        const chunks = this.chunkContent(doc.content);
         const isChunked = chunks.length > 1;
         if (isChunked) {
           await collection.deleteMany({
@@ -422,6 +771,19 @@ var WebRAGPlugin = class _WebRAGPlugin {
             },
             { upsert: true }
           );
+          chunksProcessed++;
+          if (onCrawlProgress) {
+            this.emitCrawlProgress(
+              { metadata: options?.metadata },
+              {
+                phase: "indexing",
+                urlsScheduled: indexingTotal,
+                pagesProcessed: docIndex + (i + 1 === chunks.length ? 1 : 0),
+                chunksTotal,
+                chunksProcessed
+              }
+            );
+          }
         }
         indexed++;
       } catch (error) {
@@ -501,23 +863,57 @@ var WebRAGPlugin = class _WebRAGPlugin {
     let deleted = 0;
     let failed = 0;
     const errors = [];
+    const opsTotal = operations.length;
+    let opsDone = 0;
+    const ingestOptions = options ?? {};
+    this.emitBulkProgress(ingestOptions, {
+      phase: "processing",
+      opsTotal,
+      opsDone: 0
+    });
     for (const op of operations) {
+      const currentUrl = bulkOpCurrentUrl(op);
       try {
         switch (op.type) {
           case "insert":
             if (op.document) {
-              await this.ingest([op.document], options);
-              inserted++;
+              if (isUrlListingInsert(op.document)) {
+                const url = bulkOpCurrentUrl(op);
+                const crawlResult = await this.ingestSinglePageFromUrl(
+                  {
+                    url,
+                    metadata: {
+                      ...op.document.metadata ?? {},
+                      url
+                    }
+                  },
+                  ingestOptions
+                );
+                if (crawlResult.indexed > 0) {
+                  inserted++;
+                } else {
+                  failed++;
+                  const err = crawlResult.errors?.[0]?.error ?? `Failed to crawl ${url}`;
+                  errors.push({
+                    id: op.id,
+                    operation: op.type,
+                    error: err
+                  });
+                }
+              } else {
+                await this.ingest([op.document], ingestOptions);
+                inserted++;
+              }
             }
             break;
           case "update":
             if (op.document) {
-              await this.update(op.id, op.document, options);
+              await this.update(op.id, op.document, ingestOptions);
               updated++;
             }
             break;
           case "delete":
-            const count = await this.delete(op.id, options);
+            const count = await this.delete(op.id, ingestOptions);
             deleted += count;
             break;
         }
@@ -528,6 +924,15 @@ var WebRAGPlugin = class _WebRAGPlugin {
           operation: op.type,
           error: error.message || "Unknown error"
         });
+      } finally {
+        opsDone++;
+        this.emitBulkProgress(ingestOptions, {
+          phase: "processing",
+          opsTotal,
+          opsDone,
+          currentOpType: op.type,
+          ...currentUrl ? { currentUrl } : {}
+        });
       }
     }
     return {
@@ -1094,6 +1499,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       };
     }
     const dbg = this.createDebugCollector(config.debug);
+    this.emitCrawlProgress(config, { phase: "discovering", urlsDiscovered: 0 });
     const base = this.normalizeWebsiteUrl(config.baseUrl, stripQueryParams);
     if (!base) {
       return {
@@ -1125,6 +1531,10 @@ var WebRAGPlugin = class _WebRAGPlugin {
         if (config.excludePatterns?.length) {
           filteredUrls = filteredUrls.filter((u) => !config.excludePatterns.some((p) => u.includes(p)));
         }
+        this.emitCrawlProgress(config, {
+          phase: "discovering",
+          urlsDiscovered: filteredUrls.length
+        });
         urlsToCrawl = filteredUrls.slice(0, maxPages);
         urlsSkipped = Math.max(0, filteredUrls.length - urlsToCrawl.length);
         break;
@@ -1146,7 +1556,16 @@ var WebRAGPlugin = class _WebRAGPlugin {
       urlsToCrawl = discovery.urls;
       urlsSkipped = discovery.skipped;
       dbg.log("discovery.linkLookup", { discovered: urlsToCrawl.length, skipped: urlsSkipped });
+      this.emitCrawlProgress(config, {
+        phase: "discovering",
+        urlsDiscovered: urlsToCrawl.length
+      });
     }
+    this.emitCrawlProgress(config, {
+      phase: "crawling",
+      urlsDiscovered: urlsToCrawl.length,
+      urlsScheduled: urlsToCrawl.length
+    });
     const result = await this.crawlUrls(urlsToCrawl, {
       contentSelector: config.contentSelector,
       titleSelector: config.titleSelector,
@@ -1168,9 +1587,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return {
       ...result,
       urlsSkipped,
+      /** URLs selected for this crawl (≤ maxPages); use for progress UI denominador. */
+      urlsScheduled: urlsToCrawl.length,
       crawledAt: /* @__PURE__ */ new Date(),
       metadata: {
         ...result.metadata || {},
+        urlsScheduled: urlsToCrawl.length,
         discoveryDebug: dbg.summary()
       }
     };
@@ -1299,7 +1721,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return await response.text();
   }
   extractInternalLinks(html, base, stripQueryParams) {
-    const $ = cheerio.load(html);
+    const $ = cheerio3.load(html);
     const links = /* @__PURE__ */ new Set();
     $("a[href]").each((_, el) => {
       const href = ($(el).attr("href") || "").trim();
@@ -1398,6 +1820,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const forceRecrawl = !!(options && options.forceRecrawl);
     const agentId = options?.agentId ?? "shared";
     const stripQ = config.stripQueryParams ?? false;
+    const ingestionId = typeof config.metadata?.ingestionId === "string" && config.metadata.ingestionId.trim() ? config.metadata.ingestionId.trim() : void 0;
     const urlByNorm = /* @__PURE__ */ new Map();
     for (const u of urls) {
       const norm = this.normalizeLedgerUrl(u, stripQ) || u;
@@ -1426,6 +1849,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       const results = await Promise.allSettled(
         batch.map(async (url) => {
           const urlNormalized = this.normalizeLedgerUrl(url, stripQ) || url;
+          this.emitCrawlPage(config, { url, event: "start" });
           if (ledgerOpts && !forceRecrawl) {
             const entry = await this.findLedgerEntry(urlNormalized, agentId);
             if (this.shouldSkipLedger(
@@ -1446,11 +1870,24 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 docId: entry?.docId
               });
               dbg.log("crawl.ledgerSkip", { url, urlNormalized, lastStatus: entry?.lastStatus });
+              if (ledgerOpts) {
+                await this.upsertLedgerRecord({
+                  url,
+                  urlNormalized,
+                  agentId,
+                  ingestionId,
+                  status: "skipped_ledger",
+                  title: entry?.title,
+                  docId: entry?.docId,
+                  contentLength: entry?.contentLength
+                });
+              }
+              this.emitCrawlPage(config, { url, event: "done", status: "skipped_ledger" });
               return { kind: "ledger_skip", url };
             }
           }
           try {
-            const { doc, diag, bodyTextLengthHint } = await this.crawlPageSmart(url, config, timeout, {
+            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2 } = await this.crawlPageSmart(url, config, timeout, {
               renderMode,
               renderOptions,
               minContentLength,
@@ -1469,6 +1906,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 url,
                 urlNormalized,
                 agentId,
+                ingestionId,
                 status: crawlSt,
                 doc,
                 diag
@@ -1480,11 +1918,17 @@ var WebRAGPlugin = class _WebRAGPlugin {
               status: crawlSt,
               modeUsed: diag?.modeUsed,
               contentLength: doc?.content?.length,
-              bodyTextLengthHint,
+              bodyTextLengthHint: bodyTextLengthHint2,
               title: doc?.metadata?.title,
               docId: doc?.id,
               error: diag?.errorMessage
             });
+            this.emitCrawlPage(config, {
+              url,
+              event: "done",
+              status: crawlSt,
+              error: diag?.errorMessage
+            });
             return { kind: "doc", doc, url };
           } catch (error) {
             const msg = error instanceof Error ? error.message : String(error);
@@ -1493,6 +1937,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 url,
                 urlNormalized,
                 agentId,
+                ingestionId,
                 status: "error",
                 errorMessage: msg
               });
@@ -1503,6 +1948,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
               status: "error",
               error: msg
             });
+            this.emitCrawlPage(config, { url, event: "done", status: "error", error: msg });
             throw { url, error };
           }
         })
@@ -1525,12 +1971,23 @@ var WebRAGPlugin = class _WebRAGPlugin {
           });
         }
       }
+      this.emitCrawlProgress(config, {
+        phase: "crawling",
+        urlsScheduled: uniqueUrls.length,
+        pagesProcessed: Math.min(i + batch.length, uniqueUrls.length)
+      });
       if (i + concurrency < uniqueUrls.length) {
         await this.delay(delayMs);
       }
     }
     if (documents.length > 0) {
-      const ingestResult = await this.ingest(documents, options);
+      const ingestResult = await this.ingest(documents, {
+        ...options,
+        metadata: {
+          ...options?.metadata ?? {},
+          onCrawlProgress: config.metadata?.onCrawlProgress
+        }
+      });
       indexed = ingestResult.indexed;
       if (ingestResult.errors) {
         errors.push(...ingestResult.errors);
@@ -1573,125 +2030,18 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const html = await response.text();
     return this.extractDocumentFromHtml(url, html, config);
   }
-  /**
-   * Default chain works for many WordPress / Elementor / block themes where `.first()`
-   * would otherwise hit an empty wrapper.
-   */
-  static DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
-  stripNoiseFromDom($, config) {
-    const removeSelectors = config.removeSelectors || [
-      "script",
-      "style",
-      "nav",
-      "header",
-      "footer",
-      ".sidebar",
-      ".navigation",
-      ".menu",
-      ".comments",
-      '[role="navigation"]',
-      '[role="banner"]'
-    ];
-    removeSelectors.forEach((selector) => $(selector).remove());
-  }
-  /** Longest cleaned text among selector matches and full body (after noise strip). */
-  extractBestContentText($, config) {
-    const contentSelector = config.contentSelector || _WebRAGPlugin.DEFAULT_CONTENT_SELECTOR;
-    const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
-    let best = "";
-    for (const sel of selectors) {
-      $(sel).each((_, el) => {
-        const t = this.cleanContent($(el).text().trim());
-        if (t.length > best.length) best = t;
-      });
-    }
-    const bodyText = this.cleanContent($("body").text().trim());
-    if (bodyText.length > best.length) best = bodyText;
-    return best;
-  }
   bodyTextLengthHint(html, config) {
-    const $ = cheerio.load(html);
-    this.stripNoiseFromDom($, config);
-    return this.cleanContent($("body").text().trim()).length;
+    return bodyTextLengthHint(html, config);
   }
   extractDocumentFromHtml(url, html, config) {
-    const $ = cheerio.load(html);
-    this.stripNoiseFromDom($, config);
-    const titleSelector = config.titleSelector || "h1, title";
-    let title = $(titleSelector).first().text().trim();
-    if (!title) {
-      title = $("title").text().trim();
-    }
-    const content = this.extractBestContentText($, config);
-    const minChars = config.minExtractedContentLength ?? 50;
-    if (!content || content.length < minChars) return null;
-    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || // Fallback: largest/first meaningful image in main content area
-    this.extractHeroImage($, url) || void 0;
-    let imageUrl;
-    if (image) {
-      try {
-        imageUrl = new URL(image, url).href;
-      } catch {
-        imageUrl = image;
-      }
-    }
-    const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
-    let type = config.defaultType || "page";
-    if (config.typeFromUrl) {
-      for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
-        if (url.includes(pattern)) {
-          type = typeName;
-          break;
-        }
-      }
-    }
-    const id = this.urlToId(url);
+    const extracted = extractPageFromHtml(url, html, config);
+    if (!extracted.indexable) return null;
     return {
-      id,
-      content,
-      metadata: {
-        type,
-        title,
-        url,
-        ...imageUrl ? { imageUrl } : {},
-        ...description ? { description } : {},
-        ...config.metadata
-      }
+      id: extracted.id,
+      content: extracted.content,
+      metadata: extracted.metadata
     };
   }
-  /**
-   * Fallback image extraction: finds the first meaningful image in the content area.
-   * Skips icons, avatars, and tiny assets by filtering on common patterns.
-   */
-  extractHeroImage($, pageUrl) {
-    const containers = $('main, article, [role="main"], #content, .content');
-    const scope = containers.length > 0 ? containers : $("body");
-    let best;
-    scope.find("img[src]").each((_, el) => {
-      if (best) return false;
-      const src = $(el).attr("src") || "";
-      const alt = ($(el).attr("alt") || "").toLowerCase();
-      const width = parseInt($(el).attr("width") || "0", 10);
-      const height = parseInt($(el).attr("height") || "0", 10);
-      if (width > 0 && width < 80 || height > 0 && height < 80) return;
-      if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
-      if (src.startsWith("data:") || src.endsWith(".svg")) return;
-      if (src.includes("/_next/image")) {
-        try {
-          const nextUrl = new URL(src, pageUrl);
-          const realUrl = nextUrl.searchParams.get("url");
-          if (realUrl) {
-            best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
-            return false;
-          }
-        } catch {
-        }
-      }
-      best = src;
-      return false;
-    });
-    return best;
-  }
   looksLikeDynamicShell(html) {
     const lower = html.toLowerCase();
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
@@ -1709,7 +2059,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
     return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
   }
-  diagFromRenderedAttempt(doc, bodyTextLengthHint, renderFailure, blockedSuspected, modeOk, modeFailed) {
+  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed) {
     if (blockedSuspected) {
       return {
         doc: null,
@@ -1725,12 +2075,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return {
       doc,
       diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
-      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2
     };
   }
   async crawlPageSmart(url, config, timeout, ctx) {
     if (ctx.renderMode === true) {
-      const { doc, bodyTextLengthHint, renderFailure, blockedSuspected } = await this.crawlPageRendered(
+      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected } = await this.crawlPageRendered(
         url,
         config,
         timeout,
@@ -1739,7 +2089,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       );
       return this.diagFromRenderedAttempt(
         doc,
-        bodyTextLengthHint,
+        bodyTextLengthHint2,
         renderFailure,
         blockedSuspected,
         "render_ok",
@@ -1856,7 +2206,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
         }
       }
       const html = await page.content();
-      const bodyTextLengthHint = this.bodyTextLengthHint(html, config);
+      const bodyTextLengthHint2 = this.bodyTextLengthHint(html, config);
       const doc = this.extractDocumentFromHtml(url, html, config);
       if (config.debug?.saveDir && config.debug?.enabled) {
         try {
@@ -1871,7 +2221,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
           dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
         }
       }
-      return { doc, bodyTextLengthHint };
+      return { doc, bodyTextLengthHint: bodyTextLengthHint2 };
     } catch (e) {
       const msg = String(e?.message || e || "render_failed");
       const lower = msg.toLowerCase();
@@ -1921,6 +2271,30 @@ var WebRAGPlugin = class _WebRAGPlugin {
     }
     return Array.from(found);
   }
+  emitBulkProgress(options, update) {
+    const fn = options?.metadata?.onBulkProgress;
+    if (!fn) return;
+    try {
+      fn(update);
+    } catch {
+    }
+  }
+  emitCrawlProgress(config, update) {
+    const fn = config.metadata?.onCrawlProgress;
+    if (!fn) return;
+    try {
+      fn(update);
+    } catch {
+    }
+  }
+  emitCrawlPage(config, event) {
+    const fn = config.metadata?.onCrawlPage;
+    if (!fn) return;
+    try {
+      fn(event);
+    } catch {
+    }
+  }
   createDebugCollector(debug) {
     const enabled = !!debug?.enabled;
     const level = debug?.level || "summary";
@@ -1939,14 +2313,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
   /**
    * Clean extracted text content
    */
-  cleanContent(text) {
-    return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
-  }
-  /**
-   * Convert URL to a stable document ID
-   */
   urlToId(url) {
-    return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+    return urlToDocumentId(url);
   }
   /**
    * Delay helper
@@ -2209,10 +2577,18 @@ var WebRAGPlugin = class _WebRAGPlugin {
       filterableFields: this.config.filterableFields,
       typeBoosts: this.config.typeBoosts,
       recencyBoost: this.config.recencyBoost,
+      crawlLedger: this.config.crawlLedger,
       priority: this.priority
     };
   }
 };
 export {
-  WebRAGPlugin
+  WebRAGPlugin,
+  bodyTextLengthHint,
+  extractPageFromHtml,
+  extractProductMetadata,
+  normalizeAvailability,
+  normalizeCurrency,
+  parsePrice,
+  urlToDocumentId
 };