@snap-agent/rag-web 0.1.3 → 0.1.5

package/dist/index.d.mts

@@ -118,6 +118,8 @@ interface CrawlPageStatusEntry {
 interface CrawlLedgerDocument {
     tenantId: string;
     agentId: string;
+    /** Correlates ledger rows with a single ingest run (from crawl metadata.ingestionId). */
+    ingestionId?: string;
     urlNormalized: string;
     url: string;
     domain: string;
@@ -369,10 +371,46 @@ interface RSSConfig {
 /**
  * Crawl result for sitemap/URL crawling
  */
+type CrawlProgressPhase = 'discovering' | 'crawling' | 'indexing';
+/** Live crawl progress (via `metadata.onCrawlProgress` on WebsiteCrawlConfig). */
+interface CrawlProgressUpdate {
+    phase: CrawlProgressPhase;
+    /** URLs found in sitemap/BFS (may exceed urlsScheduled when capped by maxPages). */
+    urlsDiscovered?: number;
+    /** URLs that will be crawled in this run (≤ maxPages). */
+    urlsScheduled?: number;
+    /** During crawl: batches done. During indexing: documents fully embedded so far. */
+    pagesProcessed?: number;
+    /** During indexing: total text chunks to embed (drives web_content writes). */
+    chunksTotal?: number;
+    /** During indexing: chunks embedded so far. */
+    chunksProcessed?: number;
+}
+type CrawlProgressCallback = (update: CrawlProgressUpdate) => void;
+type BulkProgressPhase = 'processing' | 'indexing';
+/** Live bulk progress (via `metadata.onBulkProgress` on IngestOptions). */
+interface BulkProgressUpdate {
+    phase: BulkProgressPhase;
+    opsTotal: number;
+    opsDone: number;
+    currentOpType?: 'insert' | 'update' | 'delete';
+    currentUrl?: string;
+}
+type BulkProgressCallback = (update: BulkProgressUpdate) => void;
+/** Per-URL crawl lifecycle (via `metadata.onCrawlPage` on WebsiteCrawlConfig). */
+type CrawlPageEvent = {
+    url: string;
+    event: 'start' | 'done';
+    status?: string;
+    error?: string;
+};
+type CrawlPageCallback = (event: CrawlPageEvent) => void;
 interface CrawlResult extends WebIngestResult {
     urlsCrawled: number;
     urlsSkipped: number;
     urlsFailed: number;
+    /** URLs selected for this crawl batch (≤ maxPages); for progress UI. */
+    urlsScheduled?: number;
     crawledAt: Date;
 }
 /**
@@ -423,6 +461,7 @@ declare class WebRAGPlugin implements RAGPlugin {
     private cacheStats;
     constructor(config: WebRAGConfig);
     private getCollection;
+    private ledgerIndexesEnsured;
     private getLedgerCollection;
     /**
      * List recent crawl ledger rows (for dashboards / pagination in the front).
@@ -430,6 +469,7 @@ declare class WebRAGPlugin implements RAGPlugin {
     listCrawlLedger(options?: {
         agentId?: string;
         domain?: string;
+        ingestionId?: string;
         status?: CrawlLedgerStatus;
         limit?: number;
         skip?: number;
@@ -660,11 +700,19 @@ declare class WebRAGPlugin implements RAGPlugin {
     private extractBestContentText;
     private bodyTextLengthHint;
     private extractDocumentFromHtml;
+    /**
+     * Fallback image extraction: finds the first meaningful image in the content area.
+     * Skips icons, avatars, and tiny assets by filtering on common patterns.
+     */
+    private extractHeroImage;
     private looksLikeDynamicShell;
     private diagFromRenderedAttempt;
     private crawlPageSmart;
     private crawlPageRendered;
     private discoverSitemaps;
+    private emitBulkProgress;
+    private emitCrawlProgress;
+    private emitCrawlPage;
     private createDebugCollector;
     /**
      * Clean extracted text content
@@ -735,4 +783,4 @@ declare class WebRAGPlugin implements RAGPlugin {
     getConfig(): Record<string, any>;
 }
 
-export { type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageStatusEntry, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig };
+export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig };

package/dist/index.d.ts

@@ -118,6 +118,8 @@ interface CrawlPageStatusEntry {
 interface CrawlLedgerDocument {
     tenantId: string;
     agentId: string;
+    /** Correlates ledger rows with a single ingest run (from crawl metadata.ingestionId). */
+    ingestionId?: string;
     urlNormalized: string;
     url: string;
     domain: string;
@@ -369,10 +371,46 @@ interface RSSConfig {
 /**
  * Crawl result for sitemap/URL crawling
  */
+type CrawlProgressPhase = 'discovering' | 'crawling' | 'indexing';
+/** Live crawl progress (via `metadata.onCrawlProgress` on WebsiteCrawlConfig). */
+interface CrawlProgressUpdate {
+    phase: CrawlProgressPhase;
+    /** URLs found in sitemap/BFS (may exceed urlsScheduled when capped by maxPages). */
+    urlsDiscovered?: number;
+    /** URLs that will be crawled in this run (≤ maxPages). */
+    urlsScheduled?: number;
+    /** During crawl: batches done. During indexing: documents fully embedded so far. */
+    pagesProcessed?: number;
+    /** During indexing: total text chunks to embed (drives web_content writes). */
+    chunksTotal?: number;
+    /** During indexing: chunks embedded so far. */
+    chunksProcessed?: number;
+}
+type CrawlProgressCallback = (update: CrawlProgressUpdate) => void;
+type BulkProgressPhase = 'processing' | 'indexing';
+/** Live bulk progress (via `metadata.onBulkProgress` on IngestOptions). */
+interface BulkProgressUpdate {
+    phase: BulkProgressPhase;
+    opsTotal: number;
+    opsDone: number;
+    currentOpType?: 'insert' | 'update' | 'delete';
+    currentUrl?: string;
+}
+type BulkProgressCallback = (update: BulkProgressUpdate) => void;
+/** Per-URL crawl lifecycle (via `metadata.onCrawlPage` on WebsiteCrawlConfig). */
+type CrawlPageEvent = {
+    url: string;
+    event: 'start' | 'done';
+    status?: string;
+    error?: string;
+};
+type CrawlPageCallback = (event: CrawlPageEvent) => void;
 interface CrawlResult extends WebIngestResult {
     urlsCrawled: number;
     urlsSkipped: number;
     urlsFailed: number;
+    /** URLs selected for this crawl batch (≤ maxPages); for progress UI. */
+    urlsScheduled?: number;
     crawledAt: Date;
 }
 /**
@@ -423,6 +461,7 @@ declare class WebRAGPlugin implements RAGPlugin {
     private cacheStats;
     constructor(config: WebRAGConfig);
     private getCollection;
+    private ledgerIndexesEnsured;
     private getLedgerCollection;
     /**
      * List recent crawl ledger rows (for dashboards / pagination in the front).
@@ -430,6 +469,7 @@ declare class WebRAGPlugin implements RAGPlugin {
     listCrawlLedger(options?: {
         agentId?: string;
         domain?: string;
+        ingestionId?: string;
         status?: CrawlLedgerStatus;
         limit?: number;
         skip?: number;
@@ -660,11 +700,19 @@ declare class WebRAGPlugin implements RAGPlugin {
     private extractBestContentText;
     private bodyTextLengthHint;
     private extractDocumentFromHtml;
+    /**
+     * Fallback image extraction: finds the first meaningful image in the content area.
+     * Skips icons, avatars, and tiny assets by filtering on common patterns.
+     */
+    private extractHeroImage;
     private looksLikeDynamicShell;
     private diagFromRenderedAttempt;
     private crawlPageSmart;
     private crawlPageRendered;
     private discoverSitemaps;
+    private emitBulkProgress;
+    private emitCrawlProgress;
+    private emitCrawlPage;
     private createDebugCollector;
     /**
      * Clean extracted text content
@@ -735,4 +783,4 @@ declare class WebRAGPlugin implements RAGPlugin {
     getConfig(): Record<string, any>;
 }
 
-export { type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageStatusEntry, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig };
+export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig };

package/dist/index.js

@@ -40,6 +40,24 @@ var import_openai = __toESM(require("openai"));
 var cheerio = __toESM(require("cheerio"));
 var fs = __toESM(require("fs"));
 var path = __toESM(require("path"));
+function bulkOpCurrentUrl(op) {
+  const meta = op.document?.metadata;
+  if (typeof meta?.url === "string" && meta.url.trim()) return meta.url.trim();
+  if (typeof meta?.source === "string" && meta.source.trim()) return meta.source.trim();
+  return void 0;
+}
+function isUrlListingInsert(document) {
+  const meta = document.metadata;
+  if (meta?.type !== "url") return false;
+  const url = typeof meta.url === "string" ? meta.url.trim() : "";
+  if (!url) return false;
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
 var WebRAGPlugin = class _WebRAGPlugin {
   name = "web-rag";
   type = "rag";
@@ -78,6 +96,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     }
     return this.db.collection(this.config.collection);
   }
+  ledgerIndexesEnsured = false;
   async getLedgerCollection() {
     if (!this.client) {
       this.client = new import_mongodb.MongoClient(this.config.mongoUri);
@@ -85,7 +104,16 @@ var WebRAGPlugin = class _WebRAGPlugin {
       this.db = this.client.db(this.config.dbName);
     }
     const name = this.config.crawlLedger?.collection ?? "web_crawl_ledger";
-    return this.db.collection(name);
+    const col = this.db.collection(name);
+    if (!this.ledgerIndexesEnsured) {
+      this.ledgerIndexesEnsured = true;
+      await col.createIndex(
+        { tenantId: 1, agentId: 1, urlNormalized: 1 },
+        { unique: true }
+      );
+      await col.createIndex({ tenantId: 1, agentId: 1, ingestionId: 1, lastCrawledAt: -1 });
+    }
+    return col;
   }
   /**
    * List recent crawl ledger rows (for dashboards / pagination in the front).
@@ -95,6 +123,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const filter = { tenantId: this.config.tenantId };
     filter.agentId = options.agentId ?? "shared";
     if (options.domain) filter.domain = options.domain;
+    if (options.ingestionId) filter.ingestionId = options.ingestionId;
     if (options.status) filter.lastStatus = options.status;
     const limit = Math.min(Math.max(options.limit ?? 50, 1), 500);
     const skip = Math.max(options.skip ?? 0, 0);
@@ -163,6 +192,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
       lastCrawledAt: now,
       updatedAt: now
     };
+    if (params.ingestionId) {
+      $set.ingestionId = params.ingestionId;
+    }
     if (errMsg !== void 0) {
       $set.errorMessage = errMsg;
     } else if (params.status === "indexed" && params.doc) {
@@ -175,9 +207,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
       $set.docId = params.doc.id;
     } else {
       $set.modeUsed = params.diag?.modeUsed;
-      $set.contentLength = null;
-      $set.title = null;
-      $set.docId = null;
+      $set.contentLength = params.contentLength ?? null;
+      $set.title = params.title ?? null;
+      $set.docId = params.docId ?? null;
     }
     await col.updateOne(
       {
@@ -254,6 +286,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
+          imageUrl: doc.metadata.imageUrl,
+          description: doc.metadata.description,
           score: doc.score
         }))
       }
@@ -419,9 +453,27 @@ var WebRAGPlugin = class _WebRAGPlugin {
     let indexed = 0;
     const errors = [];
     const agentId = options?.agentId || "shared";
-    for (const doc of documents) {
+    const onCrawlProgress = options?.metadata?.onCrawlProgress;
+    const indexingTotal = documents.length;
+    const chunkPlan = documents.map((doc) => this.chunkContent(doc.content));
+    const chunksTotal = chunkPlan.reduce((sum, chunks) => sum + chunks.length, 0);
+    let chunksProcessed = 0;
+    if (onCrawlProgress && indexingTotal > 0) {
+      this.emitCrawlProgress(
+        { metadata: options?.metadata },
+        {
+          phase: "indexing",
+          urlsScheduled: indexingTotal,
+          pagesProcessed: 0,
+          chunksTotal,
+          chunksProcessed: 0
+        }
+      );
+    }
+    for (let docIndex = 0; docIndex < documents.length; docIndex++) {
+      const doc = documents[docIndex];
+      const chunks = chunkPlan[docIndex];
       try {
-        const chunks = this.chunkContent(doc.content);
         const isChunked = chunks.length > 1;
         if (isChunked) {
           await collection.deleteMany({
@@ -456,6 +508,19 @@ var WebRAGPlugin = class _WebRAGPlugin {
             },
             { upsert: true }
           );
+          chunksProcessed++;
+          if (onCrawlProgress) {
+            this.emitCrawlProgress(
+              { metadata: options?.metadata },
+              {
+                phase: "indexing",
+                urlsScheduled: indexingTotal,
+                pagesProcessed: docIndex + (i + 1 === chunks.length ? 1 : 0),
+                chunksTotal,
+                chunksProcessed
+              }
+            );
+          }
         }
         indexed++;
       } catch (error) {
@@ -535,23 +600,57 @@ var WebRAGPlugin = class _WebRAGPlugin {
     let deleted = 0;
     let failed = 0;
     const errors = [];
+    const opsTotal = operations.length;
+    let opsDone = 0;
+    const ingestOptions = options ?? {};
+    this.emitBulkProgress(ingestOptions, {
+      phase: "processing",
+      opsTotal,
+      opsDone: 0
+    });
     for (const op of operations) {
+      const currentUrl = bulkOpCurrentUrl(op);
       try {
         switch (op.type) {
           case "insert":
             if (op.document) {
-              await this.ingest([op.document], options);
-              inserted++;
+              if (isUrlListingInsert(op.document)) {
+                const url = bulkOpCurrentUrl(op);
+                const crawlResult = await this.ingestSinglePageFromUrl(
+                  {
+                    url,
+                    metadata: {
+                      ...op.document.metadata ?? {},
+                      url
+                    }
+                  },
+                  ingestOptions
+                );
+                if (crawlResult.indexed > 0) {
+                  inserted++;
+                } else {
+                  failed++;
+                  const err = crawlResult.errors?.[0]?.error ?? `Failed to crawl ${url}`;
+                  errors.push({
+                    id: op.id,
+                    operation: op.type,
+                    error: err
+                  });
+                }
+              } else {
+                await this.ingest([op.document], ingestOptions);
+                inserted++;
+              }
             }
             break;
           case "update":
             if (op.document) {
-              await this.update(op.id, op.document, options);
+              await this.update(op.id, op.document, ingestOptions);
               updated++;
             }
             break;
           case "delete":
-            const count = await this.delete(op.id, options);
+            const count = await this.delete(op.id, ingestOptions);
             deleted += count;
             break;
         }
@@ -562,6 +661,15 @@ var WebRAGPlugin = class _WebRAGPlugin {
           operation: op.type,
           error: error.message || "Unknown error"
         });
+      } finally {
+        opsDone++;
+        this.emitBulkProgress(ingestOptions, {
+          phase: "processing",
+          opsTotal,
+          opsDone,
+          currentOpType: op.type,
+          ...currentUrl ? { currentUrl } : {}
+        });
       }
     }
     return {
@@ -1128,6 +1236,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       };
     }
     const dbg = this.createDebugCollector(config.debug);
+    this.emitCrawlProgress(config, { phase: "discovering", urlsDiscovered: 0 });
     const base = this.normalizeWebsiteUrl(config.baseUrl, stripQueryParams);
     if (!base) {
       return {
@@ -1159,6 +1268,10 @@ var WebRAGPlugin = class _WebRAGPlugin {
         if (config.excludePatterns?.length) {
           filteredUrls = filteredUrls.filter((u) => !config.excludePatterns.some((p) => u.includes(p)));
         }
+        this.emitCrawlProgress(config, {
+          phase: "discovering",
+          urlsDiscovered: filteredUrls.length
+        });
         urlsToCrawl = filteredUrls.slice(0, maxPages);
         urlsSkipped = Math.max(0, filteredUrls.length - urlsToCrawl.length);
         break;
@@ -1180,7 +1293,16 @@ var WebRAGPlugin = class _WebRAGPlugin {
       urlsToCrawl = discovery.urls;
       urlsSkipped = discovery.skipped;
       dbg.log("discovery.linkLookup", { discovered: urlsToCrawl.length, skipped: urlsSkipped });
+      this.emitCrawlProgress(config, {
+        phase: "discovering",
+        urlsDiscovered: urlsToCrawl.length
+      });
     }
+    this.emitCrawlProgress(config, {
+      phase: "crawling",
+      urlsDiscovered: urlsToCrawl.length,
+      urlsScheduled: urlsToCrawl.length
+    });
     const result = await this.crawlUrls(urlsToCrawl, {
       contentSelector: config.contentSelector,
       titleSelector: config.titleSelector,
@@ -1202,9 +1324,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return {
       ...result,
       urlsSkipped,
+      /** URLs selected for this crawl (≤ maxPages); use for progress UI denominador. */
+      urlsScheduled: urlsToCrawl.length,
       crawledAt: /* @__PURE__ */ new Date(),
       metadata: {
         ...result.metadata || {},
+        urlsScheduled: urlsToCrawl.length,
         discoveryDebug: dbg.summary()
       }
     };
@@ -1432,6 +1557,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const forceRecrawl = !!(options && options.forceRecrawl);
     const agentId = options?.agentId ?? "shared";
     const stripQ = config.stripQueryParams ?? false;
+    const ingestionId = typeof config.metadata?.ingestionId === "string" && config.metadata.ingestionId.trim() ? config.metadata.ingestionId.trim() : void 0;
     const urlByNorm = /* @__PURE__ */ new Map();
     for (const u of urls) {
       const norm = this.normalizeLedgerUrl(u, stripQ) || u;
@@ -1460,6 +1586,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       const results = await Promise.allSettled(
         batch.map(async (url) => {
           const urlNormalized = this.normalizeLedgerUrl(url, stripQ) || url;
+          this.emitCrawlPage(config, { url, event: "start" });
           if (ledgerOpts && !forceRecrawl) {
             const entry = await this.findLedgerEntry(urlNormalized, agentId);
             if (this.shouldSkipLedger(
@@ -1480,6 +1607,19 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 docId: entry?.docId
               });
               dbg.log("crawl.ledgerSkip", { url, urlNormalized, lastStatus: entry?.lastStatus });
+              if (ledgerOpts) {
+                await this.upsertLedgerRecord({
+                  url,
+                  urlNormalized,
+                  agentId,
+                  ingestionId,
+                  status: "skipped_ledger",
+                  title: entry?.title,
+                  docId: entry?.docId,
+                  contentLength: entry?.contentLength
+                });
+              }
+              this.emitCrawlPage(config, { url, event: "done", status: "skipped_ledger" });
               return { kind: "ledger_skip", url };
             }
           }
@@ -1503,6 +1643,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 url,
                 urlNormalized,
                 agentId,
+                ingestionId,
                 status: crawlSt,
                 doc,
                 diag
@@ -1519,6 +1660,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
               docId: doc?.id,
               error: diag?.errorMessage
             });
+            this.emitCrawlPage(config, {
+              url,
+              event: "done",
+              status: crawlSt,
+              error: diag?.errorMessage
+            });
             return { kind: "doc", doc, url };
           } catch (error) {
             const msg = error instanceof Error ? error.message : String(error);
@@ -1527,6 +1674,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 url,
                 urlNormalized,
                 agentId,
+                ingestionId,
                 status: "error",
                 errorMessage: msg
               });
@@ -1537,6 +1685,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
               status: "error",
               error: msg
             });
+            this.emitCrawlPage(config, { url, event: "done", status: "error", error: msg });
             throw { url, error };
           }
         })
@@ -1559,12 +1708,23 @@ var WebRAGPlugin = class _WebRAGPlugin {
           });
         }
       }
+      this.emitCrawlProgress(config, {
+        phase: "crawling",
+        urlsScheduled: uniqueUrls.length,
+        pagesProcessed: Math.min(i + batch.length, uniqueUrls.length)
+      });
       if (i + concurrency < uniqueUrls.length) {
         await this.delay(delayMs);
       }
     }
     if (documents.length > 0) {
-      const ingestResult = await this.ingest(documents, options);
+      const ingestResult = await this.ingest(documents, {
+        ...options,
+        metadata: {
+          ...options?.metadata ?? {},
+          onCrawlProgress: config.metadata?.onCrawlProgress
+        }
+      });
       indexed = ingestResult.indexed;
       if (ingestResult.errors) {
         errors.push(...ingestResult.errors);
@@ -1659,7 +1819,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const content = this.extractBestContentText($, config);
     const minChars = config.minExtractedContentLength ?? 50;
     if (!content || content.length < minChars) return null;
-    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || void 0;
+    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || // Fallback: largest/first meaningful image in main content area
+    this.extractHeroImage($, url) || void 0;
     let imageUrl;
     if (image) {
       try {
@@ -1692,6 +1853,39 @@ var WebRAGPlugin = class _WebRAGPlugin {
       }
     };
   }
+  /**
+   * Fallback image extraction: finds the first meaningful image in the content area.
+   * Skips icons, avatars, and tiny assets by filtering on common patterns.
+   */
+  extractHeroImage($, pageUrl) {
+    const containers = $('main, article, [role="main"], #content, .content');
+    const scope = containers.length > 0 ? containers : $("body");
+    let best;
+    scope.find("img[src]").each((_, el) => {
+      if (best) return false;
+      const src = $(el).attr("src") || "";
+      const alt = ($(el).attr("alt") || "").toLowerCase();
+      const width = parseInt($(el).attr("width") || "0", 10);
+      const height = parseInt($(el).attr("height") || "0", 10);
+      if (width > 0 && width < 80 || height > 0 && height < 80) return;
+      if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
+      if (src.startsWith("data:") || src.endsWith(".svg")) return;
+      if (src.includes("/_next/image")) {
+        try {
+          const nextUrl = new URL(src, pageUrl);
+          const realUrl = nextUrl.searchParams.get("url");
+          if (realUrl) {
+            best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
+            return false;
+          }
+        } catch {
+        }
+      }
+      best = src;
+      return false;
+    });
+    return best;
+  }
   looksLikeDynamicShell(html) {
     const lower = html.toLowerCase();
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
@@ -1921,6 +2115,30 @@ var WebRAGPlugin = class _WebRAGPlugin {
     }
     return Array.from(found);
   }
+  emitBulkProgress(options, update) {
+    const fn = options?.metadata?.onBulkProgress;
+    if (!fn) return;
+    try {
+      fn(update);
+    } catch {
+    }
+  }
+  emitCrawlProgress(config, update) {
+    const fn = config.metadata?.onCrawlProgress;
+    if (!fn) return;
+    try {
+      fn(update);
+    } catch {
+    }
+  }
+  emitCrawlPage(config, event) {
+    const fn = config.metadata?.onCrawlPage;
+    if (!fn) return;
+    try {
+      fn(event);
+    } catch {
+    }
+  }
   createDebugCollector(debug) {
     const enabled = !!debug?.enabled;
     const level = debug?.level || "summary";
@@ -2209,6 +2427,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       filterableFields: this.config.filterableFields,
       typeBoosts: this.config.typeBoosts,
       recencyBoost: this.config.recencyBoost,
+      crawlLedger: this.config.crawlLedger,
       priority: this.priority
     };
   }

package/dist/index.mjs

@@ -4,6 +4,24 @@ import OpenAI from "openai";
 import * as cheerio from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
+function bulkOpCurrentUrl(op) {
+  const meta = op.document?.metadata;
+  if (typeof meta?.url === "string" && meta.url.trim()) return meta.url.trim();
+  if (typeof meta?.source === "string" && meta.source.trim()) return meta.source.trim();
+  return void 0;
+}
+function isUrlListingInsert(document) {
+  const meta = document.metadata;
+  if (meta?.type !== "url") return false;
+  const url = typeof meta.url === "string" ? meta.url.trim() : "";
+  if (!url) return false;
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
 var WebRAGPlugin = class _WebRAGPlugin {
   name = "web-rag";
   type = "rag";
@@ -42,6 +60,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     }
     return this.db.collection(this.config.collection);
   }
+  ledgerIndexesEnsured = false;
   async getLedgerCollection() {
     if (!this.client) {
       this.client = new MongoClient(this.config.mongoUri);
@@ -49,7 +68,16 @@ var WebRAGPlugin = class _WebRAGPlugin {
       this.db = this.client.db(this.config.dbName);
     }
     const name = this.config.crawlLedger?.collection ?? "web_crawl_ledger";
-    return this.db.collection(name);
+    const col = this.db.collection(name);
+    if (!this.ledgerIndexesEnsured) {
+      this.ledgerIndexesEnsured = true;
+      await col.createIndex(
+        { tenantId: 1, agentId: 1, urlNormalized: 1 },
+        { unique: true }
+      );
+      await col.createIndex({ tenantId: 1, agentId: 1, ingestionId: 1, lastCrawledAt: -1 });
+    }
+    return col;
   }
   /**
    * List recent crawl ledger rows (for dashboards / pagination in the front).
@@ -59,6 +87,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const filter = { tenantId: this.config.tenantId };
     filter.agentId = options.agentId ?? "shared";
     if (options.domain) filter.domain = options.domain;
+    if (options.ingestionId) filter.ingestionId = options.ingestionId;
     if (options.status) filter.lastStatus = options.status;
     const limit = Math.min(Math.max(options.limit ?? 50, 1), 500);
     const skip = Math.max(options.skip ?? 0, 0);
@@ -127,6 +156,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
       lastCrawledAt: now,
       updatedAt: now
     };
+    if (params.ingestionId) {
+      $set.ingestionId = params.ingestionId;
+    }
     if (errMsg !== void 0) {
       $set.errorMessage = errMsg;
     } else if (params.status === "indexed" && params.doc) {
@@ -139,9 +171,9 @@ var WebRAGPlugin = class _WebRAGPlugin {
       $set.docId = params.doc.id;
     } else {
       $set.modeUsed = params.diag?.modeUsed;
-      $set.contentLength = null;
-      $set.title = null;
-      $set.docId = null;
+      $set.contentLength = params.contentLength ?? null;
+      $set.title = params.title ?? null;
+      $set.docId = params.docId ?? null;
     }
     await col.updateOne(
       {
@@ -218,6 +250,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
+          imageUrl: doc.metadata.imageUrl,
+          description: doc.metadata.description,
           score: doc.score
         }))
       }
@@ -383,9 +417,27 @@ var WebRAGPlugin = class _WebRAGPlugin {
     let indexed = 0;
     const errors = [];
     const agentId = options?.agentId || "shared";
-    for (const doc of documents) {
+    const onCrawlProgress = options?.metadata?.onCrawlProgress;
+    const indexingTotal = documents.length;
+    const chunkPlan = documents.map((doc) => this.chunkContent(doc.content));
+    const chunksTotal = chunkPlan.reduce((sum, chunks) => sum + chunks.length, 0);
+    let chunksProcessed = 0;
+    if (onCrawlProgress && indexingTotal > 0) {
+      this.emitCrawlProgress(
+        { metadata: options?.metadata },
+        {
+          phase: "indexing",
+          urlsScheduled: indexingTotal,
+          pagesProcessed: 0,
+          chunksTotal,
+          chunksProcessed: 0
+        }
+      );
+    }
+    for (let docIndex = 0; docIndex < documents.length; docIndex++) {
+      const doc = documents[docIndex];
+      const chunks = chunkPlan[docIndex];
       try {
-        const chunks = this.chunkContent(doc.content);
         const isChunked = chunks.length > 1;
         if (isChunked) {
           await collection.deleteMany({
@@ -420,6 +472,19 @@ var WebRAGPlugin = class _WebRAGPlugin {
             },
             { upsert: true }
           );
+          chunksProcessed++;
+          if (onCrawlProgress) {
+            this.emitCrawlProgress(
+              { metadata: options?.metadata },
+              {
+                phase: "indexing",
+                urlsScheduled: indexingTotal,
+                pagesProcessed: docIndex + (i + 1 === chunks.length ? 1 : 0),
+                chunksTotal,
+                chunksProcessed
+              }
+            );
+          }
         }
         indexed++;
       } catch (error) {
@@ -499,23 +564,57 @@ var WebRAGPlugin = class _WebRAGPlugin {
     let deleted = 0;
     let failed = 0;
     const errors = [];
+    const opsTotal = operations.length;
+    let opsDone = 0;
+    const ingestOptions = options ?? {};
+    this.emitBulkProgress(ingestOptions, {
+      phase: "processing",
+      opsTotal,
+      opsDone: 0
+    });
     for (const op of operations) {
+      const currentUrl = bulkOpCurrentUrl(op);
       try {
         switch (op.type) {
           case "insert":
             if (op.document) {
-              await this.ingest([op.document], options);
-              inserted++;
+              if (isUrlListingInsert(op.document)) {
+                const url = bulkOpCurrentUrl(op);
+                const crawlResult = await this.ingestSinglePageFromUrl(
+                  {
+                    url,
+                    metadata: {
+                      ...op.document.metadata ?? {},
+                      url
+                    }
+                  },
+                  ingestOptions
+                );
+                if (crawlResult.indexed > 0) {
+                  inserted++;
+                } else {
+                  failed++;
+                  const err = crawlResult.errors?.[0]?.error ?? `Failed to crawl ${url}`;
+                  errors.push({
+                    id: op.id,
+                    operation: op.type,
+                    error: err
+                  });
+                }
+              } else {
+                await this.ingest([op.document], ingestOptions);
+                inserted++;
+              }
             }
             break;
           case "update":
             if (op.document) {
-              await this.update(op.id, op.document, options);
+              await this.update(op.id, op.document, ingestOptions);
               updated++;
             }
             break;
           case "delete":
-            const count = await this.delete(op.id, options);
+            const count = await this.delete(op.id, ingestOptions);
             deleted += count;
             break;
         }
@@ -526,6 +625,15 @@ var WebRAGPlugin = class _WebRAGPlugin {
           operation: op.type,
           error: error.message || "Unknown error"
         });
+      } finally {
+        opsDone++;
+        this.emitBulkProgress(ingestOptions, {
+          phase: "processing",
+          opsTotal,
+          opsDone,
+          currentOpType: op.type,
+          ...currentUrl ? { currentUrl } : {}
+        });
       }
     }
     return {
@@ -1092,6 +1200,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       };
     }
     const dbg = this.createDebugCollector(config.debug);
+    this.emitCrawlProgress(config, { phase: "discovering", urlsDiscovered: 0 });
     const base = this.normalizeWebsiteUrl(config.baseUrl, stripQueryParams);
     if (!base) {
       return {
@@ -1123,6 +1232,10 @@ var WebRAGPlugin = class _WebRAGPlugin {
         if (config.excludePatterns?.length) {
           filteredUrls = filteredUrls.filter((u) => !config.excludePatterns.some((p) => u.includes(p)));
         }
+        this.emitCrawlProgress(config, {
+          phase: "discovering",
+          urlsDiscovered: filteredUrls.length
+        });
         urlsToCrawl = filteredUrls.slice(0, maxPages);
         urlsSkipped = Math.max(0, filteredUrls.length - urlsToCrawl.length);
         break;
@@ -1144,7 +1257,16 @@ var WebRAGPlugin = class _WebRAGPlugin {
       urlsToCrawl = discovery.urls;
       urlsSkipped = discovery.skipped;
       dbg.log("discovery.linkLookup", { discovered: urlsToCrawl.length, skipped: urlsSkipped });
+      this.emitCrawlProgress(config, {
+        phase: "discovering",
+        urlsDiscovered: urlsToCrawl.length
+      });
     }
+    this.emitCrawlProgress(config, {
+      phase: "crawling",
+      urlsDiscovered: urlsToCrawl.length,
+      urlsScheduled: urlsToCrawl.length
+    });
     const result = await this.crawlUrls(urlsToCrawl, {
       contentSelector: config.contentSelector,
       titleSelector: config.titleSelector,
@@ -1166,9 +1288,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return {
       ...result,
       urlsSkipped,
+      /** URLs selected for this crawl (≤ maxPages); use for progress UI denominador. */
+      urlsScheduled: urlsToCrawl.length,
       crawledAt: /* @__PURE__ */ new Date(),
       metadata: {
         ...result.metadata || {},
+        urlsScheduled: urlsToCrawl.length,
         discoveryDebug: dbg.summary()
       }
     };
@@ -1396,6 +1521,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const forceRecrawl = !!(options && options.forceRecrawl);
     const agentId = options?.agentId ?? "shared";
     const stripQ = config.stripQueryParams ?? false;
+    const ingestionId = typeof config.metadata?.ingestionId === "string" && config.metadata.ingestionId.trim() ? config.metadata.ingestionId.trim() : void 0;
     const urlByNorm = /* @__PURE__ */ new Map();
     for (const u of urls) {
       const norm = this.normalizeLedgerUrl(u, stripQ) || u;
@@ -1424,6 +1550,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       const results = await Promise.allSettled(
         batch.map(async (url) => {
           const urlNormalized = this.normalizeLedgerUrl(url, stripQ) || url;
+          this.emitCrawlPage(config, { url, event: "start" });
           if (ledgerOpts && !forceRecrawl) {
             const entry = await this.findLedgerEntry(urlNormalized, agentId);
             if (this.shouldSkipLedger(
@@ -1444,6 +1571,19 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 docId: entry?.docId
               });
               dbg.log("crawl.ledgerSkip", { url, urlNormalized, lastStatus: entry?.lastStatus });
+              if (ledgerOpts) {
+                await this.upsertLedgerRecord({
+                  url,
+                  urlNormalized,
+                  agentId,
+                  ingestionId,
+                  status: "skipped_ledger",
+                  title: entry?.title,
+                  docId: entry?.docId,
+                  contentLength: entry?.contentLength
+                });
+              }
+              this.emitCrawlPage(config, { url, event: "done", status: "skipped_ledger" });
               return { kind: "ledger_skip", url };
             }
           }
@@ -1467,6 +1607,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 url,
                 urlNormalized,
                 agentId,
+                ingestionId,
                 status: crawlSt,
                 doc,
                 diag
@@ -1483,6 +1624,12 @@ var WebRAGPlugin = class _WebRAGPlugin {
               docId: doc?.id,
               error: diag?.errorMessage
             });
+            this.emitCrawlPage(config, {
+              url,
+              event: "done",
+              status: crawlSt,
+              error: diag?.errorMessage
+            });
             return { kind: "doc", doc, url };
           } catch (error) {
             const msg = error instanceof Error ? error.message : String(error);
@@ -1491,6 +1638,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
                 url,
                 urlNormalized,
                 agentId,
+                ingestionId,
                 status: "error",
                 errorMessage: msg
               });
@@ -1501,6 +1649,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
               status: "error",
               error: msg
             });
+            this.emitCrawlPage(config, { url, event: "done", status: "error", error: msg });
             throw { url, error };
           }
         })
@@ -1523,12 +1672,23 @@ var WebRAGPlugin = class _WebRAGPlugin {
           });
         }
       }
+      this.emitCrawlProgress(config, {
+        phase: "crawling",
+        urlsScheduled: uniqueUrls.length,
+        pagesProcessed: Math.min(i + batch.length, uniqueUrls.length)
+      });
       if (i + concurrency < uniqueUrls.length) {
         await this.delay(delayMs);
       }
     }
     if (documents.length > 0) {
-      const ingestResult = await this.ingest(documents, options);
+      const ingestResult = await this.ingest(documents, {
+        ...options,
+        metadata: {
+          ...options?.metadata ?? {},
+          onCrawlProgress: config.metadata?.onCrawlProgress
+        }
+      });
       indexed = ingestResult.indexed;
       if (ingestResult.errors) {
         errors.push(...ingestResult.errors);
@@ -1623,7 +1783,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const content = this.extractBestContentText($, config);
     const minChars = config.minExtractedContentLength ?? 50;
     if (!content || content.length < minChars) return null;
-    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || void 0;
+    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || // Fallback: largest/first meaningful image in main content area
+    this.extractHeroImage($, url) || void 0;
     let imageUrl;
     if (image) {
       try {
@@ -1656,6 +1817,39 @@ var WebRAGPlugin = class _WebRAGPlugin {
       }
     };
   }
+  /**
+   * Fallback image extraction: finds the first meaningful image in the content area.
+   * Skips icons, avatars, and tiny assets by filtering on common patterns.
+   */
+  extractHeroImage($, pageUrl) {
+    const containers = $('main, article, [role="main"], #content, .content');
+    const scope = containers.length > 0 ? containers : $("body");
+    let best;
+    scope.find("img[src]").each((_, el) => {
+      if (best) return false;
+      const src = $(el).attr("src") || "";
+      const alt = ($(el).attr("alt") || "").toLowerCase();
+      const width = parseInt($(el).attr("width") || "0", 10);
+      const height = parseInt($(el).attr("height") || "0", 10);
+      if (width > 0 && width < 80 || height > 0 && height < 80) return;
+      if (/logo|icon|avatar|favicon|badge|spinner|loading/i.test(src + " " + alt)) return;
+      if (src.startsWith("data:") || src.endsWith(".svg")) return;
+      if (src.includes("/_next/image")) {
+        try {
+          const nextUrl = new URL(src, pageUrl);
+          const realUrl = nextUrl.searchParams.get("url");
+          if (realUrl) {
+            best = realUrl.startsWith("http") ? realUrl : new URL(realUrl, pageUrl).href;
+            return false;
+          }
+        } catch {
+        }
+      }
+      best = src;
+      return false;
+    });
+    return best;
+  }
   looksLikeDynamicShell(html) {
     const lower = html.toLowerCase();
     const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
@@ -1885,6 +2079,30 @@ var WebRAGPlugin = class _WebRAGPlugin {
     }
     return Array.from(found);
   }
+  emitBulkProgress(options, update) {
+    const fn = options?.metadata?.onBulkProgress;
+    if (!fn) return;
+    try {
+      fn(update);
+    } catch {
+    }
+  }
+  emitCrawlProgress(config, update) {
+    const fn = config.metadata?.onCrawlProgress;
+    if (!fn) return;
+    try {
+      fn(update);
+    } catch {
+    }
+  }
+  emitCrawlPage(config, event) {
+    const fn = config.metadata?.onCrawlPage;
+    if (!fn) return;
+    try {
+      fn(event);
+    } catch {
+    }
+  }
   createDebugCollector(debug) {
     const enabled = !!debug?.enabled;
     const level = debug?.level || "summary";
@@ -2173,6 +2391,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
       filterableFields: this.config.filterableFields,
       typeBoosts: this.config.typeBoosts,
       recencyBoost: this.config.recencyBoost,
+      crawlLedger: this.config.crawlLedger,
       priority: this.priority
     };
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snap-agent/rag-web",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Web RAG plugin for SnapAgent SDK - Schema-agnostic content search via web crawling, CMS APIs, sitemaps, and RSS feeds",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",