@snap-agent/rag-web 0.1.6 → 0.1.8

package/dist/index.d.mts

@@ -114,6 +114,12 @@ interface CrawlPageStatusEntry {
     httpStatus?: number;
     error?: string;
     skippedReason?: string;
+    /**
+     * Same-origin internal links found on this page, populated only when `extractLinks` is set on
+     * the crawl config. Enables resumable recursive (BFS) crawling: the caller feeds these back into
+     * its own frontier instead of the SDK doing a separate link-discovery fetch.
+     */
+    links?: string[];
 }
 interface CrawlLedgerDocument {
     tenantId: string;
@@ -258,6 +264,10 @@ interface SitemapConfig {
      */
     debug?: DebugOptions;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 /**
  * Direct URL list crawling configuration
@@ -277,6 +287,10 @@ interface UrlListConfig {
     debug?: DebugOptions;
     stripQueryParams?: boolean;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 /**
  * Single page ingestion (no discovery)
@@ -321,6 +335,10 @@ interface WebsiteCrawlConfig {
     renderOptions?: RenderOptions;
     debug?: DebugOptions;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 interface RenderOptions {
     /**
@@ -661,6 +679,12 @@ declare class WebRAGPlugin implements RAGPlugin {
     private normalizeWebsiteUrl;
     private fetchHtml;
     private extractInternalLinks;
+    /**
+     * When `config.extractLinks` is set, parse same-origin internal links from a page's HTML so the
+     * caller can drive a resumable recursive (BFS) crawl without a separate discovery fetch. Returns
+     * undefined when disabled or on any parse error (link extraction must never fail a crawl).
+     */
+    private extractLinksIfEnabled;
     /**
      * Ingest content from a list of URLs
      *
@@ -805,4 +829,31 @@ declare function parsePrice(value: unknown): number | undefined;
 declare function normalizeCurrency(value: unknown): string | undefined;
 declare function normalizeAvailability(value: unknown): string | undefined;
 
-export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, normalizeAvailability, normalizeCurrency, parsePrice, urlToDocumentId };
+/** Abstract page roles — vertical-agnostic. */
+type PageCardType = 'detail' | 'listing' | 'amenity' | 'promotion' | 'contact' | 'content' | 'blog' | 'system' | 'page';
+interface PageCardMetadataInput {
+    url: string;
+    title?: string;
+    /** Primary heading (h1) — preferred for displayTitle over the document title tag. */
+    headingTitle?: string;
+    description?: string;
+    imageUrl?: string;
+    html?: string;
+    /** Type already resolved from typeFromUrl / defaultType. */
+    type?: string;
+    hasProductPrice?: boolean;
+}
+interface PageCardMetadataResult {
+    type: PageCardType | string;
+    cardEligible: boolean;
+    cardPriority: number;
+    displayTitle?: string;
+    displayDescription?: string;
+    displayImageUrl?: string;
+}
+declare function normalizeDisplayTitle(title?: string): string | undefined;
+declare function hardExcludePage(url: string, title?: string): boolean;
+declare function inferTypeFromUrl(url: string): PageCardType | undefined;
+declare function resolvePageCardMetadata(input: PageCardMetadataInput): PageCardMetadataResult;
+
+export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type PageCardMetadataInput, type PageCardMetadataResult, type PageCardType, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, hardExcludePage, inferTypeFromUrl, normalizeAvailability, normalizeCurrency, normalizeDisplayTitle, parsePrice, resolvePageCardMetadata, urlToDocumentId };

package/dist/index.d.ts

@@ -114,6 +114,12 @@ interface CrawlPageStatusEntry {
     httpStatus?: number;
     error?: string;
     skippedReason?: string;
+    /**
+     * Same-origin internal links found on this page, populated only when `extractLinks` is set on
+     * the crawl config. Enables resumable recursive (BFS) crawling: the caller feeds these back into
+     * its own frontier instead of the SDK doing a separate link-discovery fetch.
+     */
+    links?: string[];
 }
 interface CrawlLedgerDocument {
     tenantId: string;
@@ -258,6 +264,10 @@ interface SitemapConfig {
      */
     debug?: DebugOptions;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 /**
  * Direct URL list crawling configuration
@@ -277,6 +287,10 @@ interface UrlListConfig {
     debug?: DebugOptions;
     stripQueryParams?: boolean;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 /**
  * Single page ingestion (no discovery)
@@ -321,6 +335,10 @@ interface WebsiteCrawlConfig {
     renderOptions?: RenderOptions;
     debug?: DebugOptions;
     crawlLedger?: CrawlLedgerOptions;
+    /** Extract same-origin internal links per page into `pageStatuses[].links` (for caller-driven BFS). */
+    extractLinks?: boolean;
+    /** Max links kept per page when `extractLinks` is set (default: 200). */
+    maxLinksPerPage?: number;
 }
 interface RenderOptions {
     /**
@@ -661,6 +679,12 @@ declare class WebRAGPlugin implements RAGPlugin {
     private normalizeWebsiteUrl;
     private fetchHtml;
     private extractInternalLinks;
+    /**
+     * When `config.extractLinks` is set, parse same-origin internal links from a page's HTML so the
+     * caller can drive a resumable recursive (BFS) crawl without a separate discovery fetch. Returns
+     * undefined when disabled or on any parse error (link extraction must never fail a crawl).
+     */
+    private extractLinksIfEnabled;
     /**
      * Ingest content from a list of URLs
      *
@@ -805,4 +829,31 @@ declare function parsePrice(value: unknown): number | undefined;
 declare function normalizeCurrency(value: unknown): string | undefined;
 declare function normalizeAvailability(value: unknown): string | undefined;
 
-export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, normalizeAvailability, normalizeCurrency, parsePrice, urlToDocumentId };
+/** Abstract page roles — vertical-agnostic. */
+type PageCardType = 'detail' | 'listing' | 'amenity' | 'promotion' | 'contact' | 'content' | 'blog' | 'system' | 'page';
+interface PageCardMetadataInput {
+    url: string;
+    title?: string;
+    /** Primary heading (h1) — preferred for displayTitle over the document title tag. */
+    headingTitle?: string;
+    description?: string;
+    imageUrl?: string;
+    html?: string;
+    /** Type already resolved from typeFromUrl / defaultType. */
+    type?: string;
+    hasProductPrice?: boolean;
+}
+interface PageCardMetadataResult {
+    type: PageCardType | string;
+    cardEligible: boolean;
+    cardPriority: number;
+    displayTitle?: string;
+    displayDescription?: string;
+    displayImageUrl?: string;
+}
+declare function normalizeDisplayTitle(title?: string): string | undefined;
+declare function hardExcludePage(url: string, title?: string): boolean;
+declare function inferTypeFromUrl(url: string): PageCardType | undefined;
+declare function resolvePageCardMetadata(input: PageCardMetadataInput): PageCardMetadataResult;
+
+export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type PageCardMetadataInput, type PageCardMetadataResult, type PageCardType, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, hardExcludePage, inferTypeFromUrl, normalizeAvailability, normalizeCurrency, normalizeDisplayTitle, parsePrice, resolvePageCardMetadata, urlToDocumentId };

package/dist/index.js

@@ -34,9 +34,13 @@ __export(index_exports, {
   bodyTextLengthHint: () => bodyTextLengthHint,
   extractPageFromHtml: () => extractPageFromHtml,
   extractProductMetadata: () => extractProductMetadata,
+  hardExcludePage: () => hardExcludePage,
+  inferTypeFromUrl: () => inferTypeFromUrl,
   normalizeAvailability: () => normalizeAvailability,
   normalizeCurrency: () => normalizeCurrency,
+  normalizeDisplayTitle: () => normalizeDisplayTitle,
   parsePrice: () => parsePrice,
+  resolvePageCardMetadata: () => resolvePageCardMetadata,
   urlToDocumentId: () => urlToDocumentId
 });
 module.exports = __toCommonJS(index_exports);
@@ -44,12 +48,12 @@ module.exports = __toCommonJS(index_exports);
 // src/WebRAGPlugin.ts
 var import_mongodb = require("mongodb");
 var import_openai = __toESM(require("openai"));
-var cheerio3 = __toESM(require("cheerio"));
+var cheerio4 = __toESM(require("cheerio"));
 var fs = __toESM(require("fs"));
 var path = __toESM(require("path"));
 
 // src/htmlPageExtract.ts
-var cheerio2 = __toESM(require("cheerio"));
+var cheerio3 = __toESM(require("cheerio"));
 
 // src/productMetadata.ts
 var cheerio = __toESM(require("cheerio"));
@@ -216,6 +220,228 @@ function normalizeAvailability(value) {
   return s.replace(/\s+/g, "") || void 0;
 }
 
+// src/pageCardMetadata.ts
+var cheerio2 = __toESM(require("cheerio"));
+var HARD_EXCLUDE_URL_RE = /(?:^|\/)(?:login|signin|sign-in|signup|sign-up|register|account|cart|checkout|admin|wp-admin|privacy|terms|legal|cookies|gdpr|thank|gracias|confirm|success|receipt|404|tag|tags|category|categories|author|archive|newsletter|careers|jobs)(?:\/|$|-|\.)/i;
+var HARD_EXCLUDE_TITLE_RE = /\b(?:login|sign\s*in|sign\s*up|privacy\s*policy|terms\s*(?:of\s*)?service|thank\s*you|gracias\s*por|admin|404|not\s*found)\b/i;
+var BLOG_URL_RE = /(?:^|\/)(?:blog|news|press|article|posts?)(?:\/|$)/i;
+var PROMOTION_URL_RE = /(?:^|\/)(?:offer|offers|sale|sales|promo|promotion|deal|deals|coupon|special-offer|buster)(?:\/|$|-|\.)/i;
+var PROMOTION_SLUG_RE = /(?:^|\/)[^/]*(?:-sale|-offer|-promo|-deal|-buster)(?:\/|$)/i;
+var ENTITY_DETAIL_PATH_RES = [
+  /\/projects\/[^/]+/i,
+  /\/project\/[^/]+/i,
+  /\/perspectives\/[^/]+/i,
+  /\/perspective\/[^/]+/i,
+  /\/portfolio\/[^/]+/i,
+  /\/case-stud(?:y|ies)\/[^/]+/i,
+  /\/insights?\/[^/]+/i,
+  /\/people\/[^/]+/i,
+  /\/person\/[^/]+/i,
+  /\/team-members?\/[^/]+/i,
+  /\/members?\/[^/]+/i,
+  /\/staff\/[^/]+/i,
+  /\/experts?\/[^/]+/i,
+  /\/authors?\/[^/]+/i,
+  /\/leadership\/[^/]+/i,
+  /\/biograph(?:y|ies)\/[^/]+/i
+];
+var DETAIL_URL_RE = /(?:^|\/)(?:product|products|item|items|p|room|rooms|suite|suites|habitacion|plan|plans|space|spaces|tour|tours|menu|project|perspective|person|team-member|team-members|staff|expert|case-study|author|biography)(?:\/|$)/i;
+var LISTING_URL_RE = /(?:^|\/)(?:catalog|catalogue|collection|collections|category|categories|shop|store|habitaciones|rooms|products|projects|perspectives|portfolio|people|team|members|insights|case-studies|thought-leadership)(?:\/|$)/i;
+var AMENITY_URL_RE = /(?:^|\/)(?:amenity|amenities|activity|activities|experience|experiences|service-page)(?:\/|$)/i;
+var CONTACT_URL_RE = /(?:^|\/)(?:contact|contacto|about|nosotros|faq|help|support)(?:\/|$)/i;
+var EN_DASH_SUFFIX_RE = /\s+[–—]\s+.+$/;
+var PIPE_SUFFIX_RE = /\s+\|\s+.+$/;
+var CARD_PRIORITY = {
+  detail: 10,
+  listing: 6,
+  amenity: 5,
+  promotion: 2,
+  contact: 1,
+  content: 1,
+  blog: 0,
+  system: 0,
+  page: 3
+};
+var CARD_ELIGIBLE_DEFAULT = {
+  detail: true,
+  listing: true,
+  amenity: true,
+  promotion: false,
+  contact: false,
+  content: false,
+  blog: false,
+  system: false,
+  page: false
+};
+var SCHEMA_TYPE_MAP = {
+  product: "detail",
+  service: "amenity",
+  hotelroom: "detail",
+  room: "detail",
+  apartment: "detail",
+  lodgingroom: "detail",
+  course: "detail",
+  event: "detail",
+  offer: "promotion",
+  person: "detail",
+  employee: "detail",
+  profilepage: "detail",
+  article: "detail",
+  newsarticle: "detail",
+  blogposting: "detail",
+  creativework: "detail"
+};
+function normalizeDisplayTitle(title) {
+  if (!title?.trim()) return title;
+  let t = title.trim();
+  for (let i = 0; i < 2; i++) {
+    const dash = t.match(EN_DASH_SUFFIX_RE);
+    if (dash && dash.index !== void 0 && dash.index >= 4) {
+      t = t.slice(0, dash.index).trim();
+      continue;
+    }
+    const pipe = t.match(PIPE_SUFFIX_RE);
+    if (pipe && pipe.index !== void 0 && pipe.index >= 8) {
+      t = t.slice(0, pipe.index).trim();
+      continue;
+    }
+    break;
+  }
+  return t || title.trim();
+}
+function hardExcludePage(url, title) {
+  const path2 = url.toLowerCase();
+  if (HARD_EXCLUDE_URL_RE.test(path2)) return true;
+  if (BLOG_URL_RE.test(path2)) return true;
+  if (title && HARD_EXCLUDE_TITLE_RE.test(title.toLowerCase())) return true;
+  try {
+    const u = new URL(url);
+    if (u.pathname === "/" || u.pathname === "") return true;
+  } catch {
+  }
+  return false;
+}
+function inferTypeFromUrl(url) {
+  const path2 = url.toLowerCase();
+  if (PROMOTION_URL_RE.test(path2) || PROMOTION_SLUG_RE.test(path2)) return "promotion";
+  if (CONTACT_URL_RE.test(path2)) return "contact";
+  if (ENTITY_DETAIL_PATH_RES.some((re) => re.test(path2))) return "detail";
+  if (AMENITY_URL_RE.test(path2)) return "amenity";
+  if (DETAIL_URL_RE.test(path2)) return "detail";
+  if (LISTING_URL_RE.test(path2)) return "listing";
+  if (BLOG_URL_RE.test(path2)) return "blog";
+  return void 0;
+}
+function collectJsonLdNodes2(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function schemaTypeName(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  const raw = types[0];
+  if (raw == null) return "";
+  const s = String(raw).toLowerCase();
+  const slash = s.lastIndexOf("/");
+  return slash >= 0 ? s.slice(slash + 1) : s;
+}
+function inferTypeFromSchema(html) {
+  const $ = cheerio2.load(html);
+  for (const el of $('script[type="application/ld+json"]').toArray()) {
+    const raw = $(el).html()?.trim();
+    if (!raw) continue;
+    try {
+      const parsed = JSON.parse(raw);
+      for (const node of collectJsonLdNodes2(parsed)) {
+        const name = schemaTypeName(node);
+        if (SCHEMA_TYPE_MAP[name]) return SCHEMA_TYPE_MAP[name];
+        if (name === "product" || node.offers != null) return "detail";
+      }
+    } catch {
+    }
+  }
+  const ogType = $('meta[property="og:type"]').attr("content")?.toLowerCase();
+  if (ogType === "product") return "detail";
+  return void 0;
+}
+function normalizePageType(raw) {
+  if (!raw) return "page";
+  const lower = raw.toLowerCase();
+  const known = [
+    "detail",
+    "listing",
+    "amenity",
+    "promotion",
+    "contact",
+    "content",
+    "blog",
+    "system",
+    "page"
+  ];
+  if (known.includes(lower)) return lower;
+  if (lower === "room" || lower === "product") return "detail";
+  if (lower === "offer" || lower === "sale") return "promotion";
+  return raw;
+}
+function resolveDisplayTitle(input) {
+  const heading = input.headingTitle?.trim();
+  if (heading) return normalizeDisplayTitle(heading);
+  return normalizeDisplayTitle(input.title);
+}
+function resolvePageCardMetadata(input) {
+  const title = input.title?.trim();
+  const url = input.url;
+  const displayTitle = resolveDisplayTitle(input);
+  if (hardExcludePage(url, title)) {
+    return {
+      type: "system",
+      cardEligible: false,
+      cardPriority: 0,
+      displayTitle,
+      displayDescription: input.description,
+      displayImageUrl: input.imageUrl
+    };
+  }
+  let type = normalizePageType(input.type);
+  if (type === "page" && input.html) {
+    const fromSchema = inferTypeFromSchema(input.html);
+    if (fromSchema) type = fromSchema;
+  }
+  if (type === "page") {
+    const fromUrl = inferTypeFromUrl(url);
+    if (fromUrl) type = fromUrl;
+  }
+  if (input.hasProductPrice && type === "page") {
+    type = "detail";
+  }
+  const typeKey = String(type).toLowerCase();
+  let cardEligible = CARD_ELIGIBLE_DEFAULT[typeKey] ?? false;
+  let cardPriority = CARD_PRIORITY[typeKey] ?? 3;
+  if (cardEligible && PROMOTION_URL_RE.test(url.toLowerCase())) {
+    cardEligible = false;
+  }
+  return {
+    type,
+    cardEligible,
+    cardPriority,
+    displayTitle,
+    displayDescription: input.description?.trim() || void 0,
+    displayImageUrl: input.imageUrl
+  };
+}
+
 // src/htmlPageExtract.ts
 var DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
 var DEFAULT_REMOVE_SELECTORS = [
@@ -238,17 +464,23 @@ function cleanContent(text) {
   return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
 }
 function bodyTextLengthHint(html, options = {}) {
-  const $ = cheerio2.load(html);
+  const $ = cheerio3.load(html);
   stripNoiseFromDom($, options);
   return cleanContent($("body").text().trim()).length;
 }
 function extractPageFromHtml(url, html, options = {}) {
-  const $ = cheerio2.load(html);
+  const $ = cheerio3.load(html);
   stripNoiseFromDom($, options);
-  const titleSelector = options.titleSelector || "h1, title";
-  let title = $(titleSelector).first().text().trim();
+  const h1Title = $("h1").first().text().trim();
+  const docTitle = $("title").text().trim();
+  let title = "";
+  if (options.titleSelector) {
+    title = $(options.titleSelector).first().text().trim();
+  } else {
+    title = docTitle || h1Title;
+  }
   if (!title) {
-    title = $("title").text().trim();
+    title = h1Title || docTitle;
   }
   const content = extractBestContentText($, options);
   const minChars = options.minExtractedContentLength ?? 50;
@@ -273,12 +505,27 @@ function extractPageFromHtml(url, html, options = {}) {
     }
   }
   const productMeta = extractProductMetadata(html);
-  const metadata = {
+  const cardMeta = resolvePageCardMetadata({
+    url,
+    title,
+    headingTitle: h1Title || void 0,
+    description,
+    imageUrl,
+    html,
     type,
+    hasProductPrice: productMeta.price != null
+  });
+  const metadata = {
+    type: cardMeta.type,
+    cardEligible: cardMeta.cardEligible,
+    cardPriority: cardMeta.cardPriority,
     ...title ? { title } : {},
+    ...cardMeta.displayTitle ? { displayTitle: cardMeta.displayTitle } : {},
     url,
     ...imageUrl ? { imageUrl } : {},
+    ...cardMeta.displayImageUrl ? { displayImageUrl: cardMeta.displayImageUrl } : {},
     ...description ? { description } : {},
+    ...cardMeta.displayDescription ? { displayDescription: cardMeta.displayDescription } : {},
     ...productMeta.price != null ? { price: productMeta.price } : {},
     ...productMeta.currency ? { currency: productMeta.currency } : {},
     ...productMeta.availability ? { availability: productMeta.availability } : {},
@@ -584,13 +831,18 @@ var WebRAGPlugin = class {
         plugin: this.name,
         contentCount: scoredResults.length,
         types: [...new Set(scoredResults.map((d) => d.metadata.type))],
-        topResults: scoredResults.slice(0, 5).map((doc) => ({
+        topResults: scoredResults.slice(0, 16).map((doc) => ({
           id: doc.id,
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
           imageUrl: doc.metadata.imageUrl,
           description: doc.metadata.description,
+          cardEligible: doc.metadata.cardEligible,
+          cardPriority: doc.metadata.cardPriority,
+          displayTitle: doc.metadata.displayTitle,
+          displayDescription: doc.metadata.displayDescription,
+          displayImageUrl: doc.metadata.displayImageUrl,
           ...doc.metadata.price != null ? { price: doc.metadata.price } : {},
           ...doc.metadata.currency ? { currency: doc.metadata.currency } : {},
           ...doc.metadata.availability ? { availability: doc.metadata.availability } : {},
@@ -1625,7 +1877,9 @@ var WebRAGPlugin = class {
       render: config.render,
       renderOptions: config.renderOptions,
       debug: config.debug,
-      crawlLedger: config.crawlLedger
+      crawlLedger: config.crawlLedger,
+      extractLinks: config.extractLinks,
+      maxLinksPerPage: config.maxLinksPerPage
     }, options);
     return {
       ...result,
@@ -1764,7 +2018,7 @@ var WebRAGPlugin = class {
     return await response.text();
   }
   extractInternalLinks(html, base, stripQueryParams) {
-    const $ = cheerio3.load(html);
+    const $ = cheerio4.load(html);
     const links = /* @__PURE__ */ new Set();
     $("a[href]").each((_, el) => {
       const href = ($(el).attr("href") || "").trim();
@@ -1781,6 +2035,22 @@ var WebRAGPlugin = class {
     });
     return Array.from(links);
   }
+  /**
+   * When `config.extractLinks` is set, parse same-origin internal links from a page's HTML so the
+   * caller can drive a resumable recursive (BFS) crawl without a separate discovery fetch. Returns
+   * undefined when disabled or on any parse error (link extraction must never fail a crawl).
+   */
+  extractLinksIfEnabled(url, html, config) {
+    if (!config.extractLinks) return void 0;
+    try {
+      const base = new URL(url);
+      const links = this.extractInternalLinks(html, base, config.stripQueryParams ?? false);
+      const cap = config.maxLinksPerPage ?? 200;
+      return links.length > cap ? links.slice(0, cap) : links;
+    } catch {
+      return void 0;
+    }
+  }
   /**
    * Ingest content from a list of URLs
    * 
@@ -1811,7 +2081,9 @@ var WebRAGPlugin = class {
       render: config.render,
       renderOptions: config.renderOptions,
       debug: config.debug,
-      crawlLedger: config.crawlLedger
+      crawlLedger: config.crawlLedger,
+      extractLinks: config.extractLinks,
+      maxLinksPerPage: config.maxLinksPerPage
     }, options);
   }
   /**
@@ -1930,7 +2202,7 @@ var WebRAGPlugin = class {
             }
           }
           try {
-            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2 } = await this.crawlPageSmart(url, config, timeout, {
+            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2, links } = await this.crawlPageSmart(url, config, timeout, {
               renderMode,
               renderOptions,
               minContentLength,
@@ -1964,7 +2236,8 @@ var WebRAGPlugin = class {
               bodyTextLengthHint: bodyTextLengthHint2,
               title: doc?.metadata?.title,
               docId: doc?.id,
-              error: diag?.errorMessage
+              error: diag?.errorMessage,
+              ...links ? { links } : {}
             });
             this.emitCrawlPage(config, {
               url,
@@ -2102,41 +2375,39 @@ var WebRAGPlugin = class {
     const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
     return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
   }
-  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed) {
+  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed, links) {
     if (blockedSuspected) {
       return {
         doc: null,
-        diag: { modeUsed: modeFailed, reason: "blocked_suspected" }
+        diag: { modeUsed: modeFailed, reason: "blocked_suspected" },
+        links
       };
     }
     if (renderFailure) {
       return {
         doc: null,
-        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure }
+        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure },
+        links
       };
     }
     return {
       doc,
       diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
-      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2,
+      links
     };
   }
   async crawlPageSmart(url, config, timeout, ctx) {
     if (ctx.renderMode === true) {
-      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected } = await this.crawlPageRendered(
-        url,
-        config,
-        timeout,
-        ctx.renderOptions,
-        ctx.dbg
-      );
+      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected, links } = await this.crawlPageRendered(url, config, timeout, ctx.renderOptions, ctx.dbg);
       return this.diagFromRenderedAttempt(
         doc,
         bodyTextLengthHint2,
         renderFailure,
         blockedSuspected,
         "render_ok",
-        "render_failed"
+        "render_failed",
+        links
       );
     }
     try {
@@ -2162,8 +2433,9 @@ var WebRAGPlugin = class {
       const html = await response.text();
       const doc = this.extractDocumentFromHtml(url, html, config);
       const staticHint = !doc ? this.bodyTextLengthHint(html, config) : void 0;
+      const staticLinks = this.extractLinksIfEnabled(url, html, config);
       if (doc && doc.content.length >= ctx.minContentLength) {
-        return { doc, diag: { modeUsed: "static_ok" } };
+        return { doc, diag: { modeUsed: "static_ok" }, links: staticLinks };
       }
       if (ctx.renderMode === "auto") {
         const shouldRender = this.looksLikeDynamicShell(html) || !doc || doc.content.length < ctx.minContentLength;
@@ -2177,7 +2449,8 @@ var WebRAGPlugin = class {
             doc: rendered,
             bodyTextLengthHint: rHint,
             renderFailure,
-            blockedSuspected
+            blockedSuspected,
+            links: renderedLinks
           } = await this.crawlPageRendered(
             url,
             config,
@@ -2192,7 +2465,9 @@ var WebRAGPlugin = class {
             renderFailure,
             blockedSuspected,
             "render_fallback_ok",
-            "render_fallback_failed"
+            "render_fallback_failed",
+            // Prefer links from the rendered DOM; fall back to the static HTML's links.
+            renderedLinks ?? staticLinks
           );
           if (!rendered && (renderFailure || blockedSuspected)) {
             fb.bodyTextLengthHint = staticHint ?? rHint;
@@ -2203,7 +2478,8 @@ var WebRAGPlugin = class {
       return {
         doc: null,
         diag: { modeUsed: "static_failed", reason: "too_small" },
-        bodyTextLengthHint: staticHint
+        bodyTextLengthHint: staticHint,
+        links: staticLinks
       };
     } catch (e) {
       throw e;
@@ -2251,6 +2527,7 @@ var WebRAGPlugin = class {
       const html = await page.content();
       const bodyTextLengthHint2 = this.bodyTextLengthHint(html, config);
       const doc = this.extractDocumentFromHtml(url, html, config);
+      const links = this.extractLinksIfEnabled(url, html, config);
       if (config.debug?.saveDir && config.debug?.enabled) {
         try {
           const saveDir = config.debug.saveDir;
@@ -2264,7 +2541,7 @@ var WebRAGPlugin = class {
           dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
         }
       }
-      return { doc, bodyTextLengthHint: bodyTextLengthHint2 };
+      return { doc, bodyTextLengthHint: bodyTextLengthHint2, links };
     } catch (e) {
       const msg = String(e?.message || e || "render_failed");
       const lower = msg.toLowerCase();
@@ -2631,8 +2908,12 @@ var WebRAGPlugin = class {
   bodyTextLengthHint,
   extractPageFromHtml,
   extractProductMetadata,
+  hardExcludePage,
+  inferTypeFromUrl,
   normalizeAvailability,
   normalizeCurrency,
+  normalizeDisplayTitle,
   parsePrice,
+  resolvePageCardMetadata,
   urlToDocumentId
 });

package/dist/index.mjs

@@ -1,12 +1,12 @@
 // src/WebRAGPlugin.ts
 import { MongoClient } from "mongodb";
 import OpenAI from "openai";
-import * as cheerio3 from "cheerio";
+import * as cheerio4 from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
 
 // src/htmlPageExtract.ts
-import * as cheerio2 from "cheerio";
+import * as cheerio3 from "cheerio";
 
 // src/productMetadata.ts
 import * as cheerio from "cheerio";
@@ -173,6 +173,228 @@ function normalizeAvailability(value) {
   return s.replace(/\s+/g, "") || void 0;
 }
 
+// src/pageCardMetadata.ts
+import * as cheerio2 from "cheerio";
+var HARD_EXCLUDE_URL_RE = /(?:^|\/)(?:login|signin|sign-in|signup|sign-up|register|account|cart|checkout|admin|wp-admin|privacy|terms|legal|cookies|gdpr|thank|gracias|confirm|success|receipt|404|tag|tags|category|categories|author|archive|newsletter|careers|jobs)(?:\/|$|-|\.)/i;
+var HARD_EXCLUDE_TITLE_RE = /\b(?:login|sign\s*in|sign\s*up|privacy\s*policy|terms\s*(?:of\s*)?service|thank\s*you|gracias\s*por|admin|404|not\s*found)\b/i;
+var BLOG_URL_RE = /(?:^|\/)(?:blog|news|press|article|posts?)(?:\/|$)/i;
+var PROMOTION_URL_RE = /(?:^|\/)(?:offer|offers|sale|sales|promo|promotion|deal|deals|coupon|special-offer|buster)(?:\/|$|-|\.)/i;
+var PROMOTION_SLUG_RE = /(?:^|\/)[^/]*(?:-sale|-offer|-promo|-deal|-buster)(?:\/|$)/i;
+var ENTITY_DETAIL_PATH_RES = [
+  /\/projects\/[^/]+/i,
+  /\/project\/[^/]+/i,
+  /\/perspectives\/[^/]+/i,
+  /\/perspective\/[^/]+/i,
+  /\/portfolio\/[^/]+/i,
+  /\/case-stud(?:y|ies)\/[^/]+/i,
+  /\/insights?\/[^/]+/i,
+  /\/people\/[^/]+/i,
+  /\/person\/[^/]+/i,
+  /\/team-members?\/[^/]+/i,
+  /\/members?\/[^/]+/i,
+  /\/staff\/[^/]+/i,
+  /\/experts?\/[^/]+/i,
+  /\/authors?\/[^/]+/i,
+  /\/leadership\/[^/]+/i,
+  /\/biograph(?:y|ies)\/[^/]+/i
+];
+var DETAIL_URL_RE = /(?:^|\/)(?:product|products|item|items|p|room|rooms|suite|suites|habitacion|plan|plans|space|spaces|tour|tours|menu|project|perspective|person|team-member|team-members|staff|expert|case-study|author|biography)(?:\/|$)/i;
+var LISTING_URL_RE = /(?:^|\/)(?:catalog|catalogue|collection|collections|category|categories|shop|store|habitaciones|rooms|products|projects|perspectives|portfolio|people|team|members|insights|case-studies|thought-leadership)(?:\/|$)/i;
+var AMENITY_URL_RE = /(?:^|\/)(?:amenity|amenities|activity|activities|experience|experiences|service-page)(?:\/|$)/i;
+var CONTACT_URL_RE = /(?:^|\/)(?:contact|contacto|about|nosotros|faq|help|support)(?:\/|$)/i;
+var EN_DASH_SUFFIX_RE = /\s+[–—]\s+.+$/;
+var PIPE_SUFFIX_RE = /\s+\|\s+.+$/;
+var CARD_PRIORITY = {
+  detail: 10,
+  listing: 6,
+  amenity: 5,
+  promotion: 2,
+  contact: 1,
+  content: 1,
+  blog: 0,
+  system: 0,
+  page: 3
+};
+var CARD_ELIGIBLE_DEFAULT = {
+  detail: true,
+  listing: true,
+  amenity: true,
+  promotion: false,
+  contact: false,
+  content: false,
+  blog: false,
+  system: false,
+  page: false
+};
+var SCHEMA_TYPE_MAP = {
+  product: "detail",
+  service: "amenity",
+  hotelroom: "detail",
+  room: "detail",
+  apartment: "detail",
+  lodgingroom: "detail",
+  course: "detail",
+  event: "detail",
+  offer: "promotion",
+  person: "detail",
+  employee: "detail",
+  profilepage: "detail",
+  article: "detail",
+  newsarticle: "detail",
+  blogposting: "detail",
+  creativework: "detail"
+};
+function normalizeDisplayTitle(title) {
+  if (!title?.trim()) return title;
+  let t = title.trim();
+  for (let i = 0; i < 2; i++) {
+    const dash = t.match(EN_DASH_SUFFIX_RE);
+    if (dash && dash.index !== void 0 && dash.index >= 4) {
+      t = t.slice(0, dash.index).trim();
+      continue;
+    }
+    const pipe = t.match(PIPE_SUFFIX_RE);
+    if (pipe && pipe.index !== void 0 && pipe.index >= 8) {
+      t = t.slice(0, pipe.index).trim();
+      continue;
+    }
+    break;
+  }
+  return t || title.trim();
+}
+function hardExcludePage(url, title) {
+  const path2 = url.toLowerCase();
+  if (HARD_EXCLUDE_URL_RE.test(path2)) return true;
+  if (BLOG_URL_RE.test(path2)) return true;
+  if (title && HARD_EXCLUDE_TITLE_RE.test(title.toLowerCase())) return true;
+  try {
+    const u = new URL(url);
+    if (u.pathname === "/" || u.pathname === "") return true;
+  } catch {
+  }
+  return false;
+}
+function inferTypeFromUrl(url) {
+  const path2 = url.toLowerCase();
+  if (PROMOTION_URL_RE.test(path2) || PROMOTION_SLUG_RE.test(path2)) return "promotion";
+  if (CONTACT_URL_RE.test(path2)) return "contact";
+  if (ENTITY_DETAIL_PATH_RES.some((re) => re.test(path2))) return "detail";
+  if (AMENITY_URL_RE.test(path2)) return "amenity";
+  if (DETAIL_URL_RE.test(path2)) return "detail";
+  if (LISTING_URL_RE.test(path2)) return "listing";
+  if (BLOG_URL_RE.test(path2)) return "blog";
+  return void 0;
+}
+function collectJsonLdNodes2(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function schemaTypeName(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  const raw = types[0];
+  if (raw == null) return "";
+  const s = String(raw).toLowerCase();
+  const slash = s.lastIndexOf("/");
+  return slash >= 0 ? s.slice(slash + 1) : s;
+}
+function inferTypeFromSchema(html) {
+  const $ = cheerio2.load(html);
+  for (const el of $('script[type="application/ld+json"]').toArray()) {
+    const raw = $(el).html()?.trim();
+    if (!raw) continue;
+    try {
+      const parsed = JSON.parse(raw);
+      for (const node of collectJsonLdNodes2(parsed)) {
+        const name = schemaTypeName(node);
+        if (SCHEMA_TYPE_MAP[name]) return SCHEMA_TYPE_MAP[name];
+        if (name === "product" || node.offers != null) return "detail";
+      }
+    } catch {
+    }
+  }
+  const ogType = $('meta[property="og:type"]').attr("content")?.toLowerCase();
+  if (ogType === "product") return "detail";
+  return void 0;
+}
+function normalizePageType(raw) {
+  if (!raw) return "page";
+  const lower = raw.toLowerCase();
+  const known = [
+    "detail",
+    "listing",
+    "amenity",
+    "promotion",
+    "contact",
+    "content",
+    "blog",
+    "system",
+    "page"
+  ];
+  if (known.includes(lower)) return lower;
+  if (lower === "room" || lower === "product") return "detail";
+  if (lower === "offer" || lower === "sale") return "promotion";
+  return raw;
+}
+function resolveDisplayTitle(input) {
+  const heading = input.headingTitle?.trim();
+  if (heading) return normalizeDisplayTitle(heading);
+  return normalizeDisplayTitle(input.title);
+}
+function resolvePageCardMetadata(input) {
+  const title = input.title?.trim();
+  const url = input.url;
+  const displayTitle = resolveDisplayTitle(input);
+  if (hardExcludePage(url, title)) {
+    return {
+      type: "system",
+      cardEligible: false,
+      cardPriority: 0,
+      displayTitle,
+      displayDescription: input.description,
+      displayImageUrl: input.imageUrl
+    };
+  }
+  let type = normalizePageType(input.type);
+  if (type === "page" && input.html) {
+    const fromSchema = inferTypeFromSchema(input.html);
+    if (fromSchema) type = fromSchema;
+  }
+  if (type === "page") {
+    const fromUrl = inferTypeFromUrl(url);
+    if (fromUrl) type = fromUrl;
+  }
+  if (input.hasProductPrice && type === "page") {
+    type = "detail";
+  }
+  const typeKey = String(type).toLowerCase();
+  let cardEligible = CARD_ELIGIBLE_DEFAULT[typeKey] ?? false;
+  let cardPriority = CARD_PRIORITY[typeKey] ?? 3;
+  if (cardEligible && PROMOTION_URL_RE.test(url.toLowerCase())) {
+    cardEligible = false;
+  }
+  return {
+    type,
+    cardEligible,
+    cardPriority,
+    displayTitle,
+    displayDescription: input.description?.trim() || void 0,
+    displayImageUrl: input.imageUrl
+  };
+}
+
 // src/htmlPageExtract.ts
 var DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
 var DEFAULT_REMOVE_SELECTORS = [
@@ -195,17 +417,23 @@ function cleanContent(text) {
   return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
 }
 function bodyTextLengthHint(html, options = {}) {
-  const $ = cheerio2.load(html);
+  const $ = cheerio3.load(html);
   stripNoiseFromDom($, options);
   return cleanContent($("body").text().trim()).length;
 }
 function extractPageFromHtml(url, html, options = {}) {
-  const $ = cheerio2.load(html);
+  const $ = cheerio3.load(html);
   stripNoiseFromDom($, options);
-  const titleSelector = options.titleSelector || "h1, title";
-  let title = $(titleSelector).first().text().trim();
+  const h1Title = $("h1").first().text().trim();
+  const docTitle = $("title").text().trim();
+  let title = "";
+  if (options.titleSelector) {
+    title = $(options.titleSelector).first().text().trim();
+  } else {
+    title = docTitle || h1Title;
+  }
   if (!title) {
-    title = $("title").text().trim();
+    title = h1Title || docTitle;
   }
   const content = extractBestContentText($, options);
   const minChars = options.minExtractedContentLength ?? 50;
@@ -230,12 +458,27 @@ function extractPageFromHtml(url, html, options = {}) {
     }
   }
   const productMeta = extractProductMetadata(html);
-  const metadata = {
+  const cardMeta = resolvePageCardMetadata({
+    url,
+    title,
+    headingTitle: h1Title || void 0,
+    description,
+    imageUrl,
+    html,
     type,
+    hasProductPrice: productMeta.price != null
+  });
+  const metadata = {
+    type: cardMeta.type,
+    cardEligible: cardMeta.cardEligible,
+    cardPriority: cardMeta.cardPriority,
     ...title ? { title } : {},
+    ...cardMeta.displayTitle ? { displayTitle: cardMeta.displayTitle } : {},
     url,
     ...imageUrl ? { imageUrl } : {},
+    ...cardMeta.displayImageUrl ? { displayImageUrl: cardMeta.displayImageUrl } : {},
     ...description ? { description } : {},
+    ...cardMeta.displayDescription ? { displayDescription: cardMeta.displayDescription } : {},
     ...productMeta.price != null ? { price: productMeta.price } : {},
     ...productMeta.currency ? { currency: productMeta.currency } : {},
     ...productMeta.availability ? { availability: productMeta.availability } : {},
@@ -541,13 +784,18 @@ var WebRAGPlugin = class {
         plugin: this.name,
         contentCount: scoredResults.length,
         types: [...new Set(scoredResults.map((d) => d.metadata.type))],
-        topResults: scoredResults.slice(0, 5).map((doc) => ({
+        topResults: scoredResults.slice(0, 16).map((doc) => ({
           id: doc.id,
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
           imageUrl: doc.metadata.imageUrl,
           description: doc.metadata.description,
+          cardEligible: doc.metadata.cardEligible,
+          cardPriority: doc.metadata.cardPriority,
+          displayTitle: doc.metadata.displayTitle,
+          displayDescription: doc.metadata.displayDescription,
+          displayImageUrl: doc.metadata.displayImageUrl,
           ...doc.metadata.price != null ? { price: doc.metadata.price } : {},
           ...doc.metadata.currency ? { currency: doc.metadata.currency } : {},
           ...doc.metadata.availability ? { availability: doc.metadata.availability } : {},
@@ -1582,7 +1830,9 @@ var WebRAGPlugin = class {
       render: config.render,
       renderOptions: config.renderOptions,
       debug: config.debug,
-      crawlLedger: config.crawlLedger
+      crawlLedger: config.crawlLedger,
+      extractLinks: config.extractLinks,
+      maxLinksPerPage: config.maxLinksPerPage
     }, options);
     return {
       ...result,
@@ -1721,7 +1971,7 @@ var WebRAGPlugin = class {
     return await response.text();
   }
   extractInternalLinks(html, base, stripQueryParams) {
-    const $ = cheerio3.load(html);
+    const $ = cheerio4.load(html);
     const links = /* @__PURE__ */ new Set();
     $("a[href]").each((_, el) => {
       const href = ($(el).attr("href") || "").trim();
@@ -1738,6 +1988,22 @@ var WebRAGPlugin = class {
     });
     return Array.from(links);
   }
+  /**
+   * When `config.extractLinks` is set, parse same-origin internal links from a page's HTML so the
+   * caller can drive a resumable recursive (BFS) crawl without a separate discovery fetch. Returns
+   * undefined when disabled or on any parse error (link extraction must never fail a crawl).
+   */
+  extractLinksIfEnabled(url, html, config) {
+    if (!config.extractLinks) return void 0;
+    try {
+      const base = new URL(url);
+      const links = this.extractInternalLinks(html, base, config.stripQueryParams ?? false);
+      const cap = config.maxLinksPerPage ?? 200;
+      return links.length > cap ? links.slice(0, cap) : links;
+    } catch {
+      return void 0;
+    }
+  }
   /**
    * Ingest content from a list of URLs
    * 
@@ -1768,7 +2034,9 @@ var WebRAGPlugin = class {
       render: config.render,
       renderOptions: config.renderOptions,
       debug: config.debug,
-      crawlLedger: config.crawlLedger
+      crawlLedger: config.crawlLedger,
+      extractLinks: config.extractLinks,
+      maxLinksPerPage: config.maxLinksPerPage
     }, options);
   }
   /**
@@ -1887,7 +2155,7 @@ var WebRAGPlugin = class {
             }
           }
           try {
-            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2 } = await this.crawlPageSmart(url, config, timeout, {
+            const { doc, diag, bodyTextLengthHint: bodyTextLengthHint2, links } = await this.crawlPageSmart(url, config, timeout, {
               renderMode,
               renderOptions,
               minContentLength,
@@ -1921,7 +2189,8 @@ var WebRAGPlugin = class {
               bodyTextLengthHint: bodyTextLengthHint2,
               title: doc?.metadata?.title,
               docId: doc?.id,
-              error: diag?.errorMessage
+              error: diag?.errorMessage,
+              ...links ? { links } : {}
             });
             this.emitCrawlPage(config, {
               url,
@@ -2059,41 +2328,39 @@ var WebRAGPlugin = class {
     const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
     return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
   }
-  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed) {
+  diagFromRenderedAttempt(doc, bodyTextLengthHint2, renderFailure, blockedSuspected, modeOk, modeFailed, links) {
     if (blockedSuspected) {
       return {
         doc: null,
-        diag: { modeUsed: modeFailed, reason: "blocked_suspected" }
+        diag: { modeUsed: modeFailed, reason: "blocked_suspected" },
+        links
       };
     }
     if (renderFailure) {
       return {
         doc: null,
-        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure }
+        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure },
+        links
       };
     }
     return {
       doc,
       diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
-      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint2,
+      links
     };
   }
   async crawlPageSmart(url, config, timeout, ctx) {
     if (ctx.renderMode === true) {
-      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected } = await this.crawlPageRendered(
-        url,
-        config,
-        timeout,
-        ctx.renderOptions,
-        ctx.dbg
-      );
+      const { doc, bodyTextLengthHint: bodyTextLengthHint2, renderFailure, blockedSuspected, links } = await this.crawlPageRendered(url, config, timeout, ctx.renderOptions, ctx.dbg);
       return this.diagFromRenderedAttempt(
         doc,
         bodyTextLengthHint2,
         renderFailure,
         blockedSuspected,
         "render_ok",
-        "render_failed"
+        "render_failed",
+        links
       );
     }
     try {
@@ -2119,8 +2386,9 @@ var WebRAGPlugin = class {
       const html = await response.text();
       const doc = this.extractDocumentFromHtml(url, html, config);
       const staticHint = !doc ? this.bodyTextLengthHint(html, config) : void 0;
+      const staticLinks = this.extractLinksIfEnabled(url, html, config);
       if (doc && doc.content.length >= ctx.minContentLength) {
-        return { doc, diag: { modeUsed: "static_ok" } };
+        return { doc, diag: { modeUsed: "static_ok" }, links: staticLinks };
       }
       if (ctx.renderMode === "auto") {
         const shouldRender = this.looksLikeDynamicShell(html) || !doc || doc.content.length < ctx.minContentLength;
@@ -2134,7 +2402,8 @@ var WebRAGPlugin = class {
             doc: rendered,
             bodyTextLengthHint: rHint,
             renderFailure,
-            blockedSuspected
+            blockedSuspected,
+            links: renderedLinks
           } = await this.crawlPageRendered(
             url,
             config,
@@ -2149,7 +2418,9 @@ var WebRAGPlugin = class {
             renderFailure,
             blockedSuspected,
             "render_fallback_ok",
-            "render_fallback_failed"
+            "render_fallback_failed",
+            // Prefer links from the rendered DOM; fall back to the static HTML's links.
+            renderedLinks ?? staticLinks
           );
           if (!rendered && (renderFailure || blockedSuspected)) {
             fb.bodyTextLengthHint = staticHint ?? rHint;
@@ -2160,7 +2431,8 @@ var WebRAGPlugin = class {
       return {
         doc: null,
         diag: { modeUsed: "static_failed", reason: "too_small" },
-        bodyTextLengthHint: staticHint
+        bodyTextLengthHint: staticHint,
+        links: staticLinks
       };
     } catch (e) {
       throw e;
@@ -2208,6 +2480,7 @@ var WebRAGPlugin = class {
       const html = await page.content();
       const bodyTextLengthHint2 = this.bodyTextLengthHint(html, config);
       const doc = this.extractDocumentFromHtml(url, html, config);
+      const links = this.extractLinksIfEnabled(url, html, config);
       if (config.debug?.saveDir && config.debug?.enabled) {
         try {
           const saveDir = config.debug.saveDir;
@@ -2221,7 +2494,7 @@ var WebRAGPlugin = class {
           dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
         }
       }
-      return { doc, bodyTextLengthHint: bodyTextLengthHint2 };
+      return { doc, bodyTextLengthHint: bodyTextLengthHint2, links };
     } catch (e) {
       const msg = String(e?.message || e || "render_failed");
       const lower = msg.toLowerCase();
@@ -2587,8 +2860,12 @@ export {
   bodyTextLengthHint,
   extractPageFromHtml,
   extractProductMetadata,
+  hardExcludePage,
+  inferTypeFromUrl,
   normalizeAvailability,
   normalizeCurrency,
+  normalizeDisplayTitle,
   parsePrice,
+  resolvePageCardMetadata,
   urlToDocumentId
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snap-agent/rag-web",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "Web RAG plugin for SnapAgent SDK - Schema-agnostic content search via web crawling, CMS APIs, sitemaps, and RSS feeds",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
@@ -68,4 +68,3 @@
     "url": "https://github.com/vilo-hq/snap-agent/issues"
   }
 }
-