@snap-agent/rag-web 0.1.2 → 0.1.4

package/dist/index.d.mts

@@ -660,6 +660,11 @@ declare class WebRAGPlugin implements RAGPlugin {
     private extractBestContentText;
     private bodyTextLengthHint;
     private extractDocumentFromHtml;
+    /**
+     * Fallback image extraction: finds the first meaningful image in the content area.
+     * Skips icons, avatars, and tiny assets by filtering on common patterns.
+     */
+    private extractHeroImage;
     private looksLikeDynamicShell;
     private diagFromRenderedAttempt;
     private crawlPageSmart;

package/dist/index.d.ts

@@ -660,6 +660,11 @@ declare class WebRAGPlugin implements RAGPlugin {
     private extractBestContentText;
     private bodyTextLengthHint;
     private extractDocumentFromHtml;
+    /**
+     * Fallback image extraction: finds the first meaningful image in the content area.
+     * Skips icons, avatars, and tiny assets by filtering on common patterns.
+     */
+    private extractHeroImage;
     private looksLikeDynamicShell;
     private diagFromRenderedAttempt;
     private crawlPageSmart;

package/dist/index.js

@@ -254,6 +254,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
+          imageUrl: doc.metadata.imageUrl,
+          description: doc.metadata.description,
           score: doc.score
         }))
       }
@@ -1659,7 +1661,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const content = this.extractBestContentText($, config);
     const minChars = config.minExtractedContentLength ?? 50;
     if (!content || content.length < minChars) return null;
-    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || void 0;
+    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || // Fallback: largest/first meaningful image in main content area
+    this.extractHeroImage($, url) || void 0;
     let imageUrl;
     if (image) {
       try {
@@ -1668,6 +1671,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
         imageUrl = image;
       }
     }
+    const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
     let type = config.defaultType || "page";
     if (config.typeFromUrl) {
       for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
@@ -1686,10 +1690,44 @@ var WebRAGPlugin = class _WebRAGPlugin {
         title,
         url,
         ...imageUrl ? { imageUrl } : {},
+        ...description ? { description } : {},
         ...config.metadata
       }
     };
   }
+  /**
+   * Fallback image extraction: finds the first meaningful image in the content area.
+   * Skips icons, avatars, and tiny assets by filtering on common patterns.
+   */
+  extractHeroImage($, pageUrl) {
+    const containers = $('main, article, [role="main"], #content, .content');
+    const scope = containers.length > 0 ? containers : $("body");
+    let best;
+    scope.find("img[src]").each((_, el) => {
+      if (best) return false;
+      const src = $(el).attr("src") || "";
+      const alt = ($(el).attr("alt") || "").toLowerCase();
+      const width = parseInt($(el).attr("width") || "0", 10);
+      const height = parseInt($(el).attr("height") || "0", 10);
+      if (width > 0 && width < 80 || height > 0 && height < 80) return;
+      if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
+      if (src.startsWith("data:") || src.endsWith(".svg")) return;
+      if (src.includes("/_next/image")) {
+        try {
+          const nextUrl = new URL(src, pageUrl);
+          const realUrl = nextUrl.searchParams.get("url");
+          if (realUrl) {
+            best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
+            return false;
+          }
+        } catch {
+        }
+      }
+      best = src;
+      return false;
+    });
+    return best;
+  }
   looksLikeDynamicShell(html) {
     const lower = html.toLowerCase();
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);

package/dist/index.mjs

@@ -218,6 +218,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
+          imageUrl: doc.metadata.imageUrl,
+          description: doc.metadata.description,
           score: doc.score
         }))
       }
@@ -1623,7 +1625,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const content = this.extractBestContentText($, config);
     const minChars = config.minExtractedContentLength ?? 50;
     if (!content || content.length < minChars) return null;
-    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || void 0;
+    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || // Fallback: largest/first meaningful image in main content area
+    this.extractHeroImage($, url) || void 0;
     let imageUrl;
     if (image) {
       try {
@@ -1632,6 +1635,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
         imageUrl = image;
       }
     }
+    const description = $('meta[property="og:description"]').attr("content") || $('meta[name="description"]').attr("content") || void 0;
     let type = config.defaultType || "page";
     if (config.typeFromUrl) {
       for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
@@ -1650,10 +1654,44 @@ var WebRAGPlugin = class _WebRAGPlugin {
         title,
         url,
         ...imageUrl ? { imageUrl } : {},
+        ...description ? { description } : {},
         ...config.metadata
       }
     };
   }
+  /**
+   * Fallback image extraction: finds the first meaningful image in the content area.
+   * Skips icons, avatars, and tiny assets by filtering on common patterns.
+   */
+  extractHeroImage($, pageUrl) {
+    const containers = $('main, article, [role="main"], #content, .content');
+    const scope = containers.length > 0 ? containers : $("body");
+    let best;
+    scope.find("img[src]").each((_, el) => {
+      if (best) return false;
+      const src = $(el).attr("src") || "";
+      const alt = ($(el).attr("alt") || "").toLowerCase();
+      const width = parseInt($(el).attr("width") || "0", 10);
+      const height = parseInt($(el).attr("height") || "0", 10);
+      if (width > 0 && width < 80 || height > 0 && height < 80) return;
+      if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
+      if (src.startsWith("data:") || src.endsWith(".svg")) return;
+      if (src.includes("/_next/image")) {
+        try {
+          const nextUrl = new URL(src, pageUrl);
+          const realUrl = nextUrl.searchParams.get("url");
+          if (realUrl) {
+            best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
+            return false;
+          }
+        } catch {
+        }
+      }
+      best = src;
+      return false;
+    });
+    return best;
+  }
   looksLikeDynamicShell(html) {
     const lower = html.toLowerCase();
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snap-agent/rag-web",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "Web RAG plugin for SnapAgent SDK - Schema-agnostic content search via web crawling, CMS APIs, sitemaps, and RSS feeds",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",