@snap-agent/rag-web 0.1.6 → 0.1.7

package/dist/index.d.mts

@@ -805,4 +805,31 @@ declare function parsePrice(value: unknown): number | undefined;
 declare function normalizeCurrency(value: unknown): string | undefined;
 declare function normalizeAvailability(value: unknown): string | undefined;
 
-export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, normalizeAvailability, normalizeCurrency, parsePrice, urlToDocumentId };
+/** Abstract page roles — vertical-agnostic. */
+type PageCardType = 'detail' | 'listing' | 'amenity' | 'promotion' | 'contact' | 'content' | 'blog' | 'system' | 'page';
+interface PageCardMetadataInput {
+    url: string;
+    title?: string;
+    /** Primary heading (h1) — preferred for displayTitle over the document title tag. */
+    headingTitle?: string;
+    description?: string;
+    imageUrl?: string;
+    html?: string;
+    /** Type already resolved from typeFromUrl / defaultType. */
+    type?: string;
+    hasProductPrice?: boolean;
+}
+interface PageCardMetadataResult {
+    type: PageCardType | string;
+    cardEligible: boolean;
+    cardPriority: number;
+    displayTitle?: string;
+    displayDescription?: string;
+    displayImageUrl?: string;
+}
+declare function normalizeDisplayTitle(title?: string): string | undefined;
+declare function hardExcludePage(url: string, title?: string): boolean;
+declare function inferTypeFromUrl(url: string): PageCardType | undefined;
+declare function resolvePageCardMetadata(input: PageCardMetadataInput): PageCardMetadataResult;
+
+export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type PageCardMetadataInput, type PageCardMetadataResult, type PageCardType, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, hardExcludePage, inferTypeFromUrl, normalizeAvailability, normalizeCurrency, normalizeDisplayTitle, parsePrice, resolvePageCardMetadata, urlToDocumentId };

package/dist/index.d.ts

@@ -805,4 +805,31 @@ declare function parsePrice(value: unknown): number | undefined;
 declare function normalizeCurrency(value: unknown): string | undefined;
 declare function normalizeAvailability(value: unknown): string | undefined;
 
-export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, normalizeAvailability, normalizeCurrency, parsePrice, urlToDocumentId };
+/** Abstract page roles — vertical-agnostic. */
+type PageCardType = 'detail' | 'listing' | 'amenity' | 'promotion' | 'contact' | 'content' | 'blog' | 'system' | 'page';
+interface PageCardMetadataInput {
+    url: string;
+    title?: string;
+    /** Primary heading (h1) — preferred for displayTitle over the document title tag. */
+    headingTitle?: string;
+    description?: string;
+    imageUrl?: string;
+    html?: string;
+    /** Type already resolved from typeFromUrl / defaultType. */
+    type?: string;
+    hasProductPrice?: boolean;
+}
+interface PageCardMetadataResult {
+    type: PageCardType | string;
+    cardEligible: boolean;
+    cardPriority: number;
+    displayTitle?: string;
+    displayDescription?: string;
+    displayImageUrl?: string;
+}
+declare function normalizeDisplayTitle(title?: string): string | undefined;
+declare function hardExcludePage(url: string, title?: string): boolean;
+declare function inferTypeFromUrl(url: string): PageCardType | undefined;
+declare function resolvePageCardMetadata(input: PageCardMetadataInput): PageCardMetadataResult;
+
+export { type BulkProgressCallback, type BulkProgressPhase, type BulkProgressUpdate, type CrawlLedgerDocument, type CrawlLedgerOptions, type CrawlLedgerPluginConfig, type CrawlLedgerStatus, type CrawlPageCallback, type CrawlPageEvent, type CrawlPageStatusEntry, type CrawlProgressCallback, type CrawlProgressPhase, type CrawlProgressUpdate, type CrawlResult, type DataTransform, type DebugOptions, type DrupalConfig, type HtmlPageExtractOptions, type HtmlPageExtractResult, type PageCardMetadataInput, type PageCardMetadataResult, type PageCardType, type ProductMetadata, type RSSConfig, type RenderOptions, type SanityConfig, type SinglePageConfig, type SitemapConfig, type StoredWebDocument, type StrapiConfig, type URLSource, type URLSourceAuth, type UrlListConfig, type WebDocument, type WebIngestResult, type WebRAGConfig, WebRAGPlugin, type WebURLIngestResult, type WebsiteCrawlConfig, type WordPressConfig, bodyTextLengthHint, extractPageFromHtml, extractProductMetadata, hardExcludePage, inferTypeFromUrl, normalizeAvailability, normalizeCurrency, normalizeDisplayTitle, parsePrice, resolvePageCardMetadata, urlToDocumentId };

package/dist/index.js

@@ -34,9 +34,13 @@ __export(index_exports, {
   bodyTextLengthHint: () => bodyTextLengthHint,
   extractPageFromHtml: () => extractPageFromHtml,
   extractProductMetadata: () => extractProductMetadata,
+  hardExcludePage: () => hardExcludePage,
+  inferTypeFromUrl: () => inferTypeFromUrl,
   normalizeAvailability: () => normalizeAvailability,
   normalizeCurrency: () => normalizeCurrency,
+  normalizeDisplayTitle: () => normalizeDisplayTitle,
   parsePrice: () => parsePrice,
+  resolvePageCardMetadata: () => resolvePageCardMetadata,
   urlToDocumentId: () => urlToDocumentId
 });
 module.exports = __toCommonJS(index_exports);
@@ -44,12 +48,12 @@ module.exports = __toCommonJS(index_exports);
 // src/WebRAGPlugin.ts
 var import_mongodb = require("mongodb");
 var import_openai = __toESM(require("openai"));
-var cheerio3 = __toESM(require("cheerio"));
+var cheerio4 = __toESM(require("cheerio"));
 var fs = __toESM(require("fs"));
 var path = __toESM(require("path"));
 
 // src/htmlPageExtract.ts
-var cheerio2 = __toESM(require("cheerio"));
+var cheerio3 = __toESM(require("cheerio"));
 
 // src/productMetadata.ts
 var cheerio = __toESM(require("cheerio"));
@@ -216,6 +220,228 @@ function normalizeAvailability(value) {
   return s.replace(/\s+/g, "") || void 0;
 }
 
+// src/pageCardMetadata.ts
+var cheerio2 = __toESM(require("cheerio"));
+var HARD_EXCLUDE_URL_RE = /(?:^|\/)(?:login|signin|sign-in|signup|sign-up|register|account|cart|checkout|admin|wp-admin|privacy|terms|legal|cookies|gdpr|thank|gracias|confirm|success|receipt|404|tag|tags|category|categories|author|archive|newsletter|careers|jobs)(?:\/|$|-|\.)/i;
+var HARD_EXCLUDE_TITLE_RE = /\b(?:login|sign\s*in|sign\s*up|privacy\s*policy|terms\s*(?:of\s*)?service|thank\s*you|gracias\s*por|admin|404|not\s*found)\b/i;
+var BLOG_URL_RE = /(?:^|\/)(?:blog|news|press|article|posts?)(?:\/|$)/i;
+var PROMOTION_URL_RE = /(?:^|\/)(?:offer|offers|sale|sales|promo|promotion|deal|deals|coupon|special-offer|buster)(?:\/|$|-|\.)/i;
+var PROMOTION_SLUG_RE = /(?:^|\/)[^/]*(?:-sale|-offer|-promo|-deal|-buster)(?:\/|$)/i;
+var ENTITY_DETAIL_PATH_RES = [
+  /\/projects\/[^/]+/i,
+  /\/project\/[^/]+/i,
+  /\/perspectives\/[^/]+/i,
+  /\/perspective\/[^/]+/i,
+  /\/portfolio\/[^/]+/i,
+  /\/case-stud(?:y|ies)\/[^/]+/i,
+  /\/insights?\/[^/]+/i,
+  /\/people\/[^/]+/i,
+  /\/person\/[^/]+/i,
+  /\/team-members?\/[^/]+/i,
+  /\/members?\/[^/]+/i,
+  /\/staff\/[^/]+/i,
+  /\/experts?\/[^/]+/i,
+  /\/authors?\/[^/]+/i,
+  /\/leadership\/[^/]+/i,
+  /\/biograph(?:y|ies)\/[^/]+/i
+];
+var DETAIL_URL_RE = /(?:^|\/)(?:product|products|item|items|p|room|rooms|suite|suites|habitacion|plan|plans|space|spaces|tour|tours|menu|project|perspective|person|team-member|team-members|staff|expert|case-study|author|biography)(?:\/|$)/i;
+var LISTING_URL_RE = /(?:^|\/)(?:catalog|catalogue|collection|collections|category|categories|shop|store|habitaciones|rooms|products|projects|perspectives|portfolio|people|team|members|insights|case-studies|thought-leadership)(?:\/|$)/i;
+var AMENITY_URL_RE = /(?:^|\/)(?:amenity|amenities|activity|activities|experience|experiences|service-page)(?:\/|$)/i;
+var CONTACT_URL_RE = /(?:^|\/)(?:contact|contacto|about|nosotros|faq|help|support)(?:\/|$)/i;
+var EN_DASH_SUFFIX_RE = /\s+[–—]\s+.+$/;
+var PIPE_SUFFIX_RE = /\s+\|\s+.+$/;
+var CARD_PRIORITY = {
+  detail: 10,
+  listing: 6,
+  amenity: 5,
+  promotion: 2,
+  contact: 1,
+  content: 1,
+  blog: 0,
+  system: 0,
+  page: 3
+};
+var CARD_ELIGIBLE_DEFAULT = {
+  detail: true,
+  listing: true,
+  amenity: true,
+  promotion: false,
+  contact: false,
+  content: false,
+  blog: false,
+  system: false,
+  page: false
+};
+var SCHEMA_TYPE_MAP = {
+  product: "detail",
+  service: "amenity",
+  hotelroom: "detail",
+  room: "detail",
+  apartment: "detail",
+  lodgingroom: "detail",
+  course: "detail",
+  event: "detail",
+  offer: "promotion",
+  person: "detail",
+  employee: "detail",
+  profilepage: "detail",
+  article: "detail",
+  newsarticle: "detail",
+  blogposting: "detail",
+  creativework: "detail"
+};
+function normalizeDisplayTitle(title) {
+  if (!title?.trim()) return title;
+  let t = title.trim();
+  for (let i = 0; i < 2; i++) {
+    const dash = t.match(EN_DASH_SUFFIX_RE);
+    if (dash && dash.index !== void 0 && dash.index >= 4) {
+      t = t.slice(0, dash.index).trim();
+      continue;
+    }
+    const pipe = t.match(PIPE_SUFFIX_RE);
+    if (pipe && pipe.index !== void 0 && pipe.index >= 8) {
+      t = t.slice(0, pipe.index).trim();
+      continue;
+    }
+    break;
+  }
+  return t || title.trim();
+}
+function hardExcludePage(url, title) {
+  const path2 = url.toLowerCase();
+  if (HARD_EXCLUDE_URL_RE.test(path2)) return true;
+  if (BLOG_URL_RE.test(path2)) return true;
+  if (title && HARD_EXCLUDE_TITLE_RE.test(title.toLowerCase())) return true;
+  try {
+    const u = new URL(url);
+    if (u.pathname === "/" || u.pathname === "") return true;
+  } catch {
+  }
+  return false;
+}
+function inferTypeFromUrl(url) {
+  const path2 = url.toLowerCase();
+  if (PROMOTION_URL_RE.test(path2) || PROMOTION_SLUG_RE.test(path2)) return "promotion";
+  if (CONTACT_URL_RE.test(path2)) return "contact";
+  if (ENTITY_DETAIL_PATH_RES.some((re) => re.test(path2))) return "detail";
+  if (AMENITY_URL_RE.test(path2)) return "amenity";
+  if (DETAIL_URL_RE.test(path2)) return "detail";
+  if (LISTING_URL_RE.test(path2)) return "listing";
+  if (BLOG_URL_RE.test(path2)) return "blog";
+  return void 0;
+}
+function collectJsonLdNodes2(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function schemaTypeName(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  const raw = types[0];
+  if (raw == null) return "";
+  const s = String(raw).toLowerCase();
+  const slash = s.lastIndexOf("/");
+  return slash >= 0 ? s.slice(slash + 1) : s;
+}
+function inferTypeFromSchema(html) {
+  const $ = cheerio2.load(html);
+  for (const el of $('script[type="application/ld+json"]').toArray()) {
+    const raw = $(el).html()?.trim();
+    if (!raw) continue;
+    try {
+      const parsed = JSON.parse(raw);
+      for (const node of collectJsonLdNodes2(parsed)) {
+        const name = schemaTypeName(node);
+        if (SCHEMA_TYPE_MAP[name]) return SCHEMA_TYPE_MAP[name];
+        if (name === "product" || node.offers != null) return "detail";
+      }
+    } catch {
+    }
+  }
+  const ogType = $('meta[property="og:type"]').attr("content")?.toLowerCase();
+  if (ogType === "product") return "detail";
+  return void 0;
+}
+function normalizePageType(raw) {
+  if (!raw) return "page";
+  const lower = raw.toLowerCase();
+  const known = [
+    "detail",
+    "listing",
+    "amenity",
+    "promotion",
+    "contact",
+    "content",
+    "blog",
+    "system",
+    "page"
+  ];
+  if (known.includes(lower)) return lower;
+  if (lower === "room" || lower === "product") return "detail";
+  if (lower === "offer" || lower === "sale") return "promotion";
+  return raw;
+}
+function resolveDisplayTitle(input) {
+  const heading = input.headingTitle?.trim();
+  if (heading) return normalizeDisplayTitle(heading);
+  return normalizeDisplayTitle(input.title);
+}
+function resolvePageCardMetadata(input) {
+  const title = input.title?.trim();
+  const url = input.url;
+  const displayTitle = resolveDisplayTitle(input);
+  if (hardExcludePage(url, title)) {
+    return {
+      type: "system",
+      cardEligible: false,
+      cardPriority: 0,
+      displayTitle,
+      displayDescription: input.description,
+      displayImageUrl: input.imageUrl
+    };
+  }
+  let type = normalizePageType(input.type);
+  if (type === "page" && input.html) {
+    const fromSchema = inferTypeFromSchema(input.html);
+    if (fromSchema) type = fromSchema;
+  }
+  if (type === "page") {
+    const fromUrl = inferTypeFromUrl(url);
+    if (fromUrl) type = fromUrl;
+  }
+  if (input.hasProductPrice && type === "page") {
+    type = "detail";
+  }
+  const typeKey = String(type).toLowerCase();
+  let cardEligible = CARD_ELIGIBLE_DEFAULT[typeKey] ?? false;
+  let cardPriority = CARD_PRIORITY[typeKey] ?? 3;
+  if (cardEligible && PROMOTION_URL_RE.test(url.toLowerCase())) {
+    cardEligible = false;
+  }
+  return {
+    type,
+    cardEligible,
+    cardPriority,
+    displayTitle,
+    displayDescription: input.description?.trim() || void 0,
+    displayImageUrl: input.imageUrl
+  };
+}
+
 // src/htmlPageExtract.ts
 var DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
 var DEFAULT_REMOVE_SELECTORS = [
@@ -238,17 +464,23 @@ function cleanContent(text) {
   return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
 }
 function bodyTextLengthHint(html, options = {}) {
-  const $ = cheerio2.load(html);
+  const $ = cheerio3.load(html);
   stripNoiseFromDom($, options);
   return cleanContent($("body").text().trim()).length;
 }
 function extractPageFromHtml(url, html, options = {}) {
-  const $ = cheerio2.load(html);
+  const $ = cheerio3.load(html);
   stripNoiseFromDom($, options);
-  const titleSelector = options.titleSelector || "h1, title";
-  let title = $(titleSelector).first().text().trim();
+  const h1Title = $("h1").first().text().trim();
+  const docTitle = $("title").text().trim();
+  let title = "";
+  if (options.titleSelector) {
+    title = $(options.titleSelector).first().text().trim();
+  } else {
+    title = docTitle || h1Title;
+  }
   if (!title) {
-    title = $("title").text().trim();
+    title = h1Title || docTitle;
   }
   const content = extractBestContentText($, options);
   const minChars = options.minExtractedContentLength ?? 50;
@@ -273,12 +505,27 @@ function extractPageFromHtml(url, html, options = {}) {
     }
   }
   const productMeta = extractProductMetadata(html);
-  const metadata = {
+  const cardMeta = resolvePageCardMetadata({
+    url,
+    title,
+    headingTitle: h1Title || void 0,
+    description,
+    imageUrl,
+    html,
     type,
+    hasProductPrice: productMeta.price != null
+  });
+  const metadata = {
+    type: cardMeta.type,
+    cardEligible: cardMeta.cardEligible,
+    cardPriority: cardMeta.cardPriority,
     ...title ? { title } : {},
+    ...cardMeta.displayTitle ? { displayTitle: cardMeta.displayTitle } : {},
     url,
     ...imageUrl ? { imageUrl } : {},
+    ...cardMeta.displayImageUrl ? { displayImageUrl: cardMeta.displayImageUrl } : {},
     ...description ? { description } : {},
+    ...cardMeta.displayDescription ? { displayDescription: cardMeta.displayDescription } : {},
     ...productMeta.price != null ? { price: productMeta.price } : {},
     ...productMeta.currency ? { currency: productMeta.currency } : {},
     ...productMeta.availability ? { availability: productMeta.availability } : {},
@@ -584,13 +831,18 @@ var WebRAGPlugin = class {
         plugin: this.name,
         contentCount: scoredResults.length,
         types: [...new Set(scoredResults.map((d) => d.metadata.type))],
-        topResults: scoredResults.slice(0, 5).map((doc) => ({
+        topResults: scoredResults.slice(0, 16).map((doc) => ({
           id: doc.id,
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
           imageUrl: doc.metadata.imageUrl,
           description: doc.metadata.description,
+          cardEligible: doc.metadata.cardEligible,
+          cardPriority: doc.metadata.cardPriority,
+          displayTitle: doc.metadata.displayTitle,
+          displayDescription: doc.metadata.displayDescription,
+          displayImageUrl: doc.metadata.displayImageUrl,
           ...doc.metadata.price != null ? { price: doc.metadata.price } : {},
           ...doc.metadata.currency ? { currency: doc.metadata.currency } : {},
           ...doc.metadata.availability ? { availability: doc.metadata.availability } : {},
@@ -1764,7 +2016,7 @@ var WebRAGPlugin = class {
     return await response.text();
   }
   extractInternalLinks(html, base, stripQueryParams) {
-    const $ = cheerio3.load(html);
+    const $ = cheerio4.load(html);
     const links = /* @__PURE__ */ new Set();
     $("a[href]").each((_, el) => {
       const href = ($(el).attr("href") || "").trim();
@@ -2631,8 +2883,12 @@ var WebRAGPlugin = class {
   bodyTextLengthHint,
   extractPageFromHtml,
   extractProductMetadata,
+  hardExcludePage,
+  inferTypeFromUrl,
   normalizeAvailability,
   normalizeCurrency,
+  normalizeDisplayTitle,
   parsePrice,
+  resolvePageCardMetadata,
   urlToDocumentId
 });

package/dist/index.mjs

@@ -1,12 +1,12 @@
 // src/WebRAGPlugin.ts
 import { MongoClient } from "mongodb";
 import OpenAI from "openai";
-import * as cheerio3 from "cheerio";
+import * as cheerio4 from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
 
 // src/htmlPageExtract.ts
-import * as cheerio2 from "cheerio";
+import * as cheerio3 from "cheerio";
 
 // src/productMetadata.ts
 import * as cheerio from "cheerio";
@@ -173,6 +173,228 @@ function normalizeAvailability(value) {
   return s.replace(/\s+/g, "") || void 0;
 }
 
+// src/pageCardMetadata.ts
+import * as cheerio2 from "cheerio";
+var HARD_EXCLUDE_URL_RE = /(?:^|\/)(?:login|signin|sign-in|signup|sign-up|register|account|cart|checkout|admin|wp-admin|privacy|terms|legal|cookies|gdpr|thank|gracias|confirm|success|receipt|404|tag|tags|category|categories|author|archive|newsletter|careers|jobs)(?:\/|$|-|\.)/i;
+var HARD_EXCLUDE_TITLE_RE = /\b(?:login|sign\s*in|sign\s*up|privacy\s*policy|terms\s*(?:of\s*)?service|thank\s*you|gracias\s*por|admin|404|not\s*found)\b/i;
+var BLOG_URL_RE = /(?:^|\/)(?:blog|news|press|article|posts?)(?:\/|$)/i;
+var PROMOTION_URL_RE = /(?:^|\/)(?:offer|offers|sale|sales|promo|promotion|deal|deals|coupon|special-offer|buster)(?:\/|$|-|\.)/i;
+var PROMOTION_SLUG_RE = /(?:^|\/)[^/]*(?:-sale|-offer|-promo|-deal|-buster)(?:\/|$)/i;
+var ENTITY_DETAIL_PATH_RES = [
+  /\/projects\/[^/]+/i,
+  /\/project\/[^/]+/i,
+  /\/perspectives\/[^/]+/i,
+  /\/perspective\/[^/]+/i,
+  /\/portfolio\/[^/]+/i,
+  /\/case-stud(?:y|ies)\/[^/]+/i,
+  /\/insights?\/[^/]+/i,
+  /\/people\/[^/]+/i,
+  /\/person\/[^/]+/i,
+  /\/team-members?\/[^/]+/i,
+  /\/members?\/[^/]+/i,
+  /\/staff\/[^/]+/i,
+  /\/experts?\/[^/]+/i,
+  /\/authors?\/[^/]+/i,
+  /\/leadership\/[^/]+/i,
+  /\/biograph(?:y|ies)\/[^/]+/i
+];
+var DETAIL_URL_RE = /(?:^|\/)(?:product|products|item|items|p|room|rooms|suite|suites|habitacion|plan|plans|space|spaces|tour|tours|menu|project|perspective|person|team-member|team-members|staff|expert|case-study|author|biography)(?:\/|$)/i;
+var LISTING_URL_RE = /(?:^|\/)(?:catalog|catalogue|collection|collections|category|categories|shop|store|habitaciones|rooms|products|projects|perspectives|portfolio|people|team|members|insights|case-studies|thought-leadership)(?:\/|$)/i;
+var AMENITY_URL_RE = /(?:^|\/)(?:amenity|amenities|activity|activities|experience|experiences|service-page)(?:\/|$)/i;
+var CONTACT_URL_RE = /(?:^|\/)(?:contact|contacto|about|nosotros|faq|help|support)(?:\/|$)/i;
+var EN_DASH_SUFFIX_RE = /\s+[–—]\s+.+$/;
+var PIPE_SUFFIX_RE = /\s+\|\s+.+$/;
+var CARD_PRIORITY = {
+  detail: 10,
+  listing: 6,
+  amenity: 5,
+  promotion: 2,
+  contact: 1,
+  content: 1,
+  blog: 0,
+  system: 0,
+  page: 3
+};
+var CARD_ELIGIBLE_DEFAULT = {
+  detail: true,
+  listing: true,
+  amenity: true,
+  promotion: false,
+  contact: false,
+  content: false,
+  blog: false,
+  system: false,
+  page: false
+};
+var SCHEMA_TYPE_MAP = {
+  product: "detail",
+  service: "amenity",
+  hotelroom: "detail",
+  room: "detail",
+  apartment: "detail",
+  lodgingroom: "detail",
+  course: "detail",
+  event: "detail",
+  offer: "promotion",
+  person: "detail",
+  employee: "detail",
+  profilepage: "detail",
+  article: "detail",
+  newsarticle: "detail",
+  blogposting: "detail",
+  creativework: "detail"
+};
+function normalizeDisplayTitle(title) {
+  if (!title?.trim()) return title;
+  let t = title.trim();
+  for (let i = 0; i < 2; i++) {
+    const dash = t.match(EN_DASH_SUFFIX_RE);
+    if (dash && dash.index !== void 0 && dash.index >= 4) {
+      t = t.slice(0, dash.index).trim();
+      continue;
+    }
+    const pipe = t.match(PIPE_SUFFIX_RE);
+    if (pipe && pipe.index !== void 0 && pipe.index >= 8) {
+      t = t.slice(0, pipe.index).trim();
+      continue;
+    }
+    break;
+  }
+  return t || title.trim();
+}
+function hardExcludePage(url, title) {
+  const path2 = url.toLowerCase();
+  if (HARD_EXCLUDE_URL_RE.test(path2)) return true;
+  if (BLOG_URL_RE.test(path2)) return true;
+  if (title && HARD_EXCLUDE_TITLE_RE.test(title.toLowerCase())) return true;
+  try {
+    const u = new URL(url);
+    if (u.pathname === "/" || u.pathname === "") return true;
+  } catch {
+  }
+  return false;
+}
+function inferTypeFromUrl(url) {
+  const path2 = url.toLowerCase();
+  if (PROMOTION_URL_RE.test(path2) || PROMOTION_SLUG_RE.test(path2)) return "promotion";
+  if (CONTACT_URL_RE.test(path2)) return "contact";
+  if (ENTITY_DETAIL_PATH_RES.some((re) => re.test(path2))) return "detail";
+  if (AMENITY_URL_RE.test(path2)) return "amenity";
+  if (DETAIL_URL_RE.test(path2)) return "detail";
+  if (LISTING_URL_RE.test(path2)) return "listing";
+  if (BLOG_URL_RE.test(path2)) return "blog";
+  return void 0;
+}
+function collectJsonLdNodes2(data) {
+  const nodes = [];
+  const visit = (value) => {
+    if (value == null) return;
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (typeof value !== "object") return;
+    const obj = value;
+    nodes.push(obj);
+    if (obj["@graph"]) visit(obj["@graph"]);
+  };
+  visit(data);
+  return nodes;
+}
+function schemaTypeName(node) {
+  const type = node["@type"];
+  const types = Array.isArray(type) ? type : type != null ? [type] : [];
+  const raw = types[0];
+  if (raw == null) return "";
+  const s = String(raw).toLowerCase();
+  const slash = s.lastIndexOf("/");
+  return slash >= 0 ? s.slice(slash + 1) : s;
+}
+function inferTypeFromSchema(html) {
+  const $ = cheerio2.load(html);
+  for (const el of $('script[type="application/ld+json"]').toArray()) {
+    const raw = $(el).html()?.trim();
+    if (!raw) continue;
+    try {
+      const parsed = JSON.parse(raw);
+      for (const node of collectJsonLdNodes2(parsed)) {
+        const name = schemaTypeName(node);
+        if (SCHEMA_TYPE_MAP[name]) return SCHEMA_TYPE_MAP[name];
+        if (name === "product" || node.offers != null) return "detail";
+      }
+    } catch {
+    }
+  }
+  const ogType = $('meta[property="og:type"]').attr("content")?.toLowerCase();
+  if (ogType === "product") return "detail";
+  return void 0;
+}
+function normalizePageType(raw) {
+  if (!raw) return "page";
+  const lower = raw.toLowerCase();
+  const known = [
+    "detail",
+    "listing",
+    "amenity",
+    "promotion",
+    "contact",
+    "content",
+    "blog",
+    "system",
+    "page"
+  ];
+  if (known.includes(lower)) return lower;
+  if (lower === "room" || lower === "product") return "detail";
+  if (lower === "offer" || lower === "sale") return "promotion";
+  return raw;
+}
+function resolveDisplayTitle(input) {
+  const heading = input.headingTitle?.trim();
+  if (heading) return normalizeDisplayTitle(heading);
+  return normalizeDisplayTitle(input.title);
+}
+function resolvePageCardMetadata(input) {
+  const title = input.title?.trim();
+  const url = input.url;
+  const displayTitle = resolveDisplayTitle(input);
+  if (hardExcludePage(url, title)) {
+    return {
+      type: "system",
+      cardEligible: false,
+      cardPriority: 0,
+      displayTitle,
+      displayDescription: input.description,
+      displayImageUrl: input.imageUrl
+    };
+  }
+  let type = normalizePageType(input.type);
+  if (type === "page" && input.html) {
+    const fromSchema = inferTypeFromSchema(input.html);
+    if (fromSchema) type = fromSchema;
+  }
+  if (type === "page") {
+    const fromUrl = inferTypeFromUrl(url);
+    if (fromUrl) type = fromUrl;
+  }
+  if (input.hasProductPrice && type === "page") {
+    type = "detail";
+  }
+  const typeKey = String(type).toLowerCase();
+  let cardEligible = CARD_ELIGIBLE_DEFAULT[typeKey] ?? false;
+  let cardPriority = CARD_PRIORITY[typeKey] ?? 3;
+  if (cardEligible && PROMOTION_URL_RE.test(url.toLowerCase())) {
+    cardEligible = false;
+  }
+  return {
+    type,
+    cardEligible,
+    cardPriority,
+    displayTitle,
+    displayDescription: input.description?.trim() || void 0,
+    displayImageUrl: input.imageUrl
+  };
+}
+
 // src/htmlPageExtract.ts
 var DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
 var DEFAULT_REMOVE_SELECTORS = [
@@ -195,17 +417,23 @@ function cleanContent(text) {
   return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
 }
 function bodyTextLengthHint(html, options = {}) {
-  const $ = cheerio2.load(html);
+  const $ = cheerio3.load(html);
   stripNoiseFromDom($, options);
   return cleanContent($("body").text().trim()).length;
 }
 function extractPageFromHtml(url, html, options = {}) {
-  const $ = cheerio2.load(html);
+  const $ = cheerio3.load(html);
   stripNoiseFromDom($, options);
-  const titleSelector = options.titleSelector || "h1, title";
-  let title = $(titleSelector).first().text().trim();
+  const h1Title = $("h1").first().text().trim();
+  const docTitle = $("title").text().trim();
+  let title = "";
+  if (options.titleSelector) {
+    title = $(options.titleSelector).first().text().trim();
+  } else {
+    title = docTitle || h1Title;
+  }
   if (!title) {
-    title = $("title").text().trim();
+    title = h1Title || docTitle;
   }
   const content = extractBestContentText($, options);
   const minChars = options.minExtractedContentLength ?? 50;
@@ -230,12 +458,27 @@ function extractPageFromHtml(url, html, options = {}) {
     }
   }
   const productMeta = extractProductMetadata(html);
-  const metadata = {
+  const cardMeta = resolvePageCardMetadata({
+    url,
+    title,
+    headingTitle: h1Title || void 0,
+    description,
+    imageUrl,
+    html,
     type,
+    hasProductPrice: productMeta.price != null
+  });
+  const metadata = {
+    type: cardMeta.type,
+    cardEligible: cardMeta.cardEligible,
+    cardPriority: cardMeta.cardPriority,
     ...title ? { title } : {},
+    ...cardMeta.displayTitle ? { displayTitle: cardMeta.displayTitle } : {},
     url,
     ...imageUrl ? { imageUrl } : {},
+    ...cardMeta.displayImageUrl ? { displayImageUrl: cardMeta.displayImageUrl } : {},
     ...description ? { description } : {},
+    ...cardMeta.displayDescription ? { displayDescription: cardMeta.displayDescription } : {},
     ...productMeta.price != null ? { price: productMeta.price } : {},
     ...productMeta.currency ? { currency: productMeta.currency } : {},
     ...productMeta.availability ? { availability: productMeta.availability } : {},
@@ -541,13 +784,18 @@ var WebRAGPlugin = class {
         plugin: this.name,
         contentCount: scoredResults.length,
         types: [...new Set(scoredResults.map((d) => d.metadata.type))],
-        topResults: scoredResults.slice(0, 5).map((doc) => ({
+        topResults: scoredResults.slice(0, 16).map((doc) => ({
           id: doc.id,
           type: doc.metadata.type,
           title: doc.metadata.title,
           url: doc.metadata.url,
           imageUrl: doc.metadata.imageUrl,
           description: doc.metadata.description,
+          cardEligible: doc.metadata.cardEligible,
+          cardPriority: doc.metadata.cardPriority,
+          displayTitle: doc.metadata.displayTitle,
+          displayDescription: doc.metadata.displayDescription,
+          displayImageUrl: doc.metadata.displayImageUrl,
           ...doc.metadata.price != null ? { price: doc.metadata.price } : {},
           ...doc.metadata.currency ? { currency: doc.metadata.currency } : {},
           ...doc.metadata.availability ? { availability: doc.metadata.availability } : {},
@@ -1721,7 +1969,7 @@ var WebRAGPlugin = class {
     return await response.text();
   }
   extractInternalLinks(html, base, stripQueryParams) {
-    const $ = cheerio3.load(html);
+    const $ = cheerio4.load(html);
     const links = /* @__PURE__ */ new Set();
     $("a[href]").each((_, el) => {
       const href = ($(el).attr("href") || "").trim();
@@ -2587,8 +2835,12 @@ export {
   bodyTextLengthHint,
   extractPageFromHtml,
   extractProductMetadata,
+  hardExcludePage,
+  inferTypeFromUrl,
   normalizeAvailability,
   normalizeCurrency,
+  normalizeDisplayTitle,
   parsePrice,
+  resolvePageCardMetadata,
   urlToDocumentId
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snap-agent/rag-web",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "description": "Web RAG plugin for SnapAgent SDK - Schema-agnostic content search via web crawling, CMS APIs, sitemaps, and RSS feeds",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",