@snap-agent/rag-web 0.1.7 → 0.1.8

package/dist/index.d.mts

@@ -114,6 +114,12 @@ interface CrawlPageStatusEntry {
     httpStatus?: number;
     error?: string;
     skippedReason?: string;
+    /**
+     * Same-origin internal links found on this page, populated only when `extractLinks` is set on
+     * the crawl config. Enables resumable recursive (BFS) crawling: the caller feeds these back into
+     * its own frontier instead of the SDK doing a separate link-discovery fetch.
+     */
+    links?: string[];
 }
 interface CrawlLedgerDocument {
     tenantId: string;
@@ -258,6 +264,10 @@ interface SitemapConfig {
      */
     debug?: DebugOptions;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 /**
  * Direct URL list crawling configuration
@@ -277,6 +287,10 @@ interface UrlListConfig {
     debug?: DebugOptions;
     stripQueryParams?: boolean;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 /**
  * Single page ingestion (no discovery)
@@ -321,6 +335,10 @@ interface WebsiteCrawlConfig {
     renderOptions?: RenderOptions;
     debug?: DebugOptions;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 interface RenderOptions {
     /**
@@ -661,6 +679,12 @@ declare class WebRAGPlugin implements RAGPlugin {
     private normalizeWebsiteUrl;
     private fetchHtml;
     private extractInternalLinks;
+    /**
+     * When `config.extractLinks` is set, parse same-origin internal links from a page's HTML so the
+     * caller can drive a resumable recursive (BFS) crawl without a separate discovery fetch. Returns
+     * undefined when disabled or on any parse error (link extraction must never fail a crawl).
+     */
+    private extractLinksIfEnabled;
     /**
      * Ingest content from a list of URLs
      *

package/dist/index.d.ts

@@ -114,6 +114,12 @@ interface CrawlPageStatusEntry {
     httpStatus?: number;
     error?: string;
     skippedReason?: string;
+    /**
+     * Same-origin internal links found on this page, populated only when `extractLinks` is set on
+     * the crawl config. Enables resumable recursive (BFS) crawling: the caller feeds these back into
+     * its own frontier instead of the SDK doing a separate link-discovery fetch.
+     */
+    links?: string[];
 }
 interface CrawlLedgerDocument {
     tenantId: string;
@@ -258,6 +264,10 @@ interface SitemapConfig {
      */
     debug?: DebugOptions;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 /**
  * Direct URL list crawling configuration
@@ -277,6 +287,10 @@ interface UrlListConfig {
     debug?: DebugOptions;
     stripQueryParams?: boolean;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 /**
  * Single page ingestion (no discovery)
@@ -321,6 +335,10 @@ interface WebsiteCrawlConfig {
     renderOptions?: RenderOptions;
     debug?: DebugOptions;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 interface RenderOptions {
     /**
@@ -661,6 +679,12 @@ declare class WebRAGPlugin implements RAGPlugin {
     private normalizeWebsiteUrl;
     private fetchHtml;
     private extractInternalLinks;
+    /**
+     * When `config.extractLinks` is set, parse same-origin internal links from a page's HTML so the
+     * caller can drive a resumable recursive (BFS) crawl without a separate discovery fetch. Returns
+     * undefined when disabled or on any parse error (link extraction must never fail a crawl).
+     */
+    private extractLinksIfEnabled;
     /**
      * Ingest content from a list of URLs
      *

package/dist/index.js

@@ -1877,7 +1877,9 @@ var WebRAGPlugin = class {
       render: config.render,
       renderOptions: config.renderOptions,
       debug: config.debug,
-      crawlLedger: config.crawlLedger
+      crawlLedger: config.crawlLedger,
+      extractLinks: config.extractLinks,
+      maxLinksPerPage: config.maxLinksPerPage
     }, options);
     return {
       ...result,
@@ -2033,6 +2035,22 @@ var WebRAGPlugin = class {
     });
     return Array.from(links);
   }
+  /**
+   * When `config.extractLinks` is set, parse same-origin internal links from a page's HTML so the
+   * caller can drive a resumable recursive (BFS) crawl without a separate discovery fetch. Returns
+   * undefined when disabled or on any parse error (link extraction must never fail a crawl).
+   */
+  extractLinksIfEnabled(url, html, config) {
+    if (!config.extractLinks) return void 0;
+    try {
+      const base = new URL(url);
+      const links = this.extractInternalLinks(html, base, config.stripQueryParams ?? false);
+      const cap = config.maxLinksPerPage ?? 200;
+      return links.length > cap ? links.slice(0, cap) : links;
+    } catch {
+      return void 0;
+    }
+  }
   /**
    * Ingest content from a list of URLs
    * 
@@ -2063,7 +2081,9 @@ var WebRAGPlugin = class {
       render: config.render,
       renderOptions: config.renderOptions,
       debug: config.debug,
-      crawlLedger: config.crawlLedger
+      crawlLedger: config.crawlLedger,
+      extractLinks: config.extractLinks,
+      maxLinksPerPage: config.maxLinksPerPage
     }, options);
   }
   /**
@@ -2182,7 +2202,7 @@ var WebRAGPlugin = class {
             }
           }
           try {
-            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2 } = await this.crawlPageSmart(url, config, timeout, {
+            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2, links } = await this.crawlPageSmart(url, config, timeout, {
               renderMode,
               renderOptions,
               minContentLength,
@@ -2216,7 +2236,8 @@ var WebRAGPlugin = class {
               bodyTextLengthHint: bodyTextLengthHint2,
               title: doc?.metadata?.title,
               docId: doc?.id,
-              error: diag?.errorMessage
+              error: diag?.errorMessage,
+              ...links ? { links } : {}
             });
             this.emitCrawlPage(config, {
               url,
@@ -2354,41 +2375,39 @@ var WebRAGPlugin = class {
     const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
     return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
   }
-  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed) {
+  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed, links) {
     if (blockedSuspected) {
       return {
         doc: null,
-        diag: { modeUsed: modeFailed, reason: "blocked_suspected" }
+        diag: { modeUsed: modeFailed, reason: "blocked_suspected" },
+        links
       };
     }
     if (renderFailure) {
       return {
         doc: null,
-        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure }
+        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure },
+        links
       };
     }
     return {
       doc,
       diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
-      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2,
+      links
     };
   }
   async crawlPageSmart(url, config, timeout, ctx) {
     if (ctx.renderMode === true) {
-      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected } = await this.crawlPageRendered(
-        url,
-        config,
-        timeout,
-        ctx.renderOptions,
-        ctx.dbg
-      );
+      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected, links } = await this.crawlPageRendered(url, config, timeout, ctx.renderOptions, ctx.dbg);
       return this.diagFromRenderedAttempt(
         doc,
         bodyTextLengthHint2,
         renderFailure,
         blockedSuspected,
         "render_ok",
-        "render_failed"
+        "render_failed",
+        links
       );
     }
     try {
@@ -2414,8 +2433,9 @@ var WebRAGPlugin = class {
       const html = await response.text();
       const doc = this.extractDocumentFromHtml(url, html, config);
       const staticHint = !doc ? this.bodyTextLengthHint(html, config) : void 0;
+      const staticLinks = this.extractLinksIfEnabled(url, html, config);
       if (doc && doc.content.length >= ctx.minContentLength) {
-        return { doc, diag: { modeUsed: "static_ok" } };
+        return { doc, diag: { modeUsed: "static_ok" }, links: staticLinks };
       }
       if (ctx.renderMode === "auto") {
         const shouldRender = this.looksLikeDynamicShell(html) || !doc || doc.content.length < ctx.minContentLength;
@@ -2429,7 +2449,8 @@ var WebRAGPlugin = class {
             doc: rendered,
             bodyTextLengthHint: rHint,
             renderFailure,
-            blockedSuspected
+            blockedSuspected,
+            links: renderedLinks
           } = await this.crawlPageRendered(
             url,
             config,
@@ -2444,7 +2465,9 @@ var WebRAGPlugin = class {
             renderFailure,
             blockedSuspected,
             "render_fallback_ok",
-            "render_fallback_failed"
+            "render_fallback_failed",
+            // Prefer links from the rendered DOM; fall back to the static HTML's links.
+            renderedLinks ?? staticLinks
           );
           if (!rendered && (renderFailure || blockedSuspected)) {
             fb.bodyTextLengthHint = staticHint ?? rHint;
@@ -2455,7 +2478,8 @@ var WebRAGPlugin = class {
       return {
         doc: null,
         diag: { modeUsed: "static_failed", reason: "too_small" },
-        bodyTextLengthHint: staticHint
+        bodyTextLengthHint: staticHint,
+        links: staticLinks
       };
     } catch (e) {
       throw e;
@@ -2503,6 +2527,7 @@ var WebRAGPlugin = class {
       const html = await page.content();
       const bodyTextLengthHint2 = this.bodyTextLengthHint(html, config);
       const doc = this.extractDocumentFromHtml(url, html, config);
+      const links = this.extractLinksIfEnabled(url, html, config);
       if (config.debug?.saveDir && config.debug?.enabled) {
         try {
           const saveDir = config.debug.saveDir;
@@ -2516,7 +2541,7 @@ var WebRAGPlugin = class {
           dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
         }
       }
-      return { doc, bodyTextLengthHint: bodyTextLengthHint2 };
+      return { doc, bodyTextLengthHint: bodyTextLengthHint2, links };
     } catch (e) {
       const msg = String(e?.message || e || "render_failed");
       const lower = msg.toLowerCase();

package/dist/index.mjs

@@ -1830,7 +1830,9 @@ var WebRAGPlugin = class {
       render: config.render,
       renderOptions: config.renderOptions,
       debug: config.debug,
-      crawlLedger: config.crawlLedger
+      crawlLedger: config.crawlLedger,
+      extractLinks: config.extractLinks,
+      maxLinksPerPage: config.maxLinksPerPage
     }, options);
     return {
       ...result,
@@ -1986,6 +1988,22 @@ var WebRAGPlugin = class {
     });
     return Array.from(links);
   }
+  /**
+   * When `config.extractLinks` is set, parse same-origin internal links from a page's HTML so the
+   * caller can drive a resumable recursive (BFS) crawl without a separate discovery fetch. Returns
+   * undefined when disabled or on any parse error (link extraction must never fail a crawl).
+   */
+  extractLinksIfEnabled(url, html, config) {
+    if (!config.extractLinks) return void 0;
+    try {
+      const base = new URL(url);
+      const links = this.extractInternalLinks(html, base, config.stripQueryParams ?? false);
+      const cap = config.maxLinksPerPage ?? 200;
+      return links.length > cap ? links.slice(0, cap) : links;
+    } catch {
+      return void 0;
+    }
+  }
   /**
    * Ingest content from a list of URLs
    * 
@@ -2016,7 +2034,9 @@ var WebRAGPlugin = class {
       render: config.render,
       renderOptions: config.renderOptions,
       debug: config.debug,
-      crawlLedger: config.crawlLedger
+      crawlLedger: config.crawlLedger,
+      extractLinks: config.extractLinks,
+      maxLinksPerPage: config.maxLinksPerPage
     }, options);
   }
   /**
@@ -2135,7 +2155,7 @@ var WebRAGPlugin = class {
             }
           }
           try {
-            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2 } = await this.crawlPageSmart(url, config, timeout, {
+            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2, links } = await this.crawlPageSmart(url, config, timeout, {
               renderMode,
               renderOptions,
               minContentLength,
@@ -2169,7 +2189,8 @@ var WebRAGPlugin = class {
               bodyTextLengthHint: bodyTextLengthHint2,
               title: doc?.metadata?.title,
               docId: doc?.id,
-              error: diag?.errorMessage
+              error: diag?.errorMessage,
+              ...links ? { links } : {}
             });
             this.emitCrawlPage(config, {
               url,
@@ -2307,41 +2328,39 @@ var WebRAGPlugin = class {
     const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
     return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
   }
-  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed) {
+  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed, links) {
     if (blockedSuspected) {
       return {
         doc: null,
-        diag: { modeUsed: modeFailed, reason: "blocked_suspected" }
+        diag: { modeUsed: modeFailed, reason: "blocked_suspected" },
+        links
       };
     }
     if (renderFailure) {
       return {
         doc: null,
-        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure }
+        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure },
+        links
       };
     }
     return {
       doc,
       diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
-      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2,
+      links
     };
   }
   async crawlPageSmart(url, config, timeout, ctx) {
     if (ctx.renderMode === true) {
-      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected } = await this.crawlPageRendered(
-        url,
-        config,
-        timeout,
-        ctx.renderOptions,
-        ctx.dbg
-      );
+      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected, links } = await this.crawlPageRendered(url, config, timeout, ctx.renderOptions, ctx.dbg);
       return this.diagFromRenderedAttempt(
         doc,
         bodyTextLengthHint2,
         renderFailure,
         blockedSuspected,
         "render_ok",
-        "render_failed"
+        "render_failed",
+        links
       );
     }
     try {
@@ -2367,8 +2386,9 @@ var WebRAGPlugin = class {
       const html = await response.text();
       const doc = this.extractDocumentFromHtml(url, html, config);
       const staticHint = !doc ? this.bodyTextLengthHint(html, config) : void 0;
+      const staticLinks = this.extractLinksIfEnabled(url, html, config);
       if (doc && doc.content.length >= ctx.minContentLength) {
-        return { doc, diag: { modeUsed: "static_ok" } };
+        return { doc, diag: { modeUsed: "static_ok" }, links: staticLinks };
       }
       if (ctx.renderMode === "auto") {
         const shouldRender = this.looksLikeDynamicShell(html) || !doc || doc.content.length < ctx.minContentLength;
@@ -2382,7 +2402,8 @@ var WebRAGPlugin = class {
             doc: rendered,
             bodyTextLengthHint: rHint,
             renderFailure,
-            blockedSuspected
+            blockedSuspected,
+            links: renderedLinks
           } = await this.crawlPageRendered(
             url,
             config,
@@ -2397,7 +2418,9 @@ var WebRAGPlugin = class {
             renderFailure,
             blockedSuspected,
             "render_fallback_ok",
-            "render_fallback_failed"
+            "render_fallback_failed",
+            // Prefer links from the rendered DOM; fall back to the static HTML's links.
+            renderedLinks ?? staticLinks
           );
           if (!rendered && (renderFailure || blockedSuspected)) {
             fb.bodyTextLengthHint = staticHint ?? rHint;
@@ -2408,7 +2431,8 @@ var WebRAGPlugin = class {
       return {
         doc: null,
         diag: { modeUsed: "static_failed", reason: "too_small" },
-        bodyTextLengthHint: staticHint
+        bodyTextLengthHint: staticHint,
+        links: staticLinks
       };
     } catch (e) {
       throw e;
@@ -2456,6 +2480,7 @@ var WebRAGPlugin = class {
       const html = await page.content();
       const bodyTextLengthHint2 = this.bodyTextLengthHint(html, config);
       const doc = this.extractDocumentFromHtml(url, html, config);
+      const links = this.extractLinksIfEnabled(url, html, config);
       if (config.debug?.saveDir && config.debug?.enabled) {
         try {
           const saveDir = config.debug.saveDir;
@@ -2469,7 +2494,7 @@ var WebRAGPlugin = class {
           dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
         }
       }
-      return { doc, bodyTextLengthHint: bodyTextLengthHint2 };
+      return { doc, bodyTextLengthHint: bodyTextLengthHint2, links };
     } catch (e) {
       const msg = String(e?.message || e || "render_failed");
       const lower = msg.toLowerCase();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snap-agent/rag-web",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "Web RAG plugin for SnapAgent SDK - Schema-agnostic content search via web crawling, CMS APIs, sitemaps, and RSS feeds",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
@@ -68,4 +68,3 @@
     "url": "https://github.com/vilo-hq/snap-agent/issues"
   }
 }
-