aeorank 3.0.2 → 3.1.0

package/dist/{chunk-OCLAIHX6.js → chunk-RYV25AUV.js}

@@ -1,6 +1,7 @@
 // src/full-site-crawler.ts
 var RESOURCE_EXTENSIONS = /\.(js|css|png|jpg|jpeg|gif|svg|ico|pdf|xml|txt|woff|woff2|ttf|eot|mp4|mp3|webp|avif|zip|gz|tar|json)$/i;
-var SKIP_PATH_PATTERNS = /^\/(api|wp-admin|wp-json|static|assets|_next|auth|login|signup|cart|checkout|admin|feed|xmlrpc)\b/i;
+var SKIP_PATH_PATTERNS = /^\/(api|wp-admin|wp-json|wp-content|wp-includes|static|assets|_next|auth|login|signup|cart|checkout|admin|feed|xmlrpc|tag|tags|author|authors|category|categories|attachment|embed|trackback|comments|search|print|amp)\b/i;
+var SKIP_URL_PATTERNS = /\/page\/\d+\/?$|[?&](s|replytocom|p|preview|share|like|amp)=/i;
 function parseRobotsTxt(robotsText) {
   const lines = robotsText.split("\n");
   const rules = { disallow: [], allow: [] };
@@ -141,6 +142,7 @@ function extractInternalLinks(html, domain) {
       if (RESOURCE_EXTENSIONS.test(path)) continue;
       if (SKIP_PATH_PATTERNS.test(path)) continue;
       const normalized = parsed.origin + path.replace(/\/+$/, "") + parsed.search;
+      if (SKIP_URL_PATTERNS.test(normalized)) continue;
       urls.add(normalized);
     } catch {
       continue;
@@ -299,4 +301,4 @@ export {
   inferCategory,
   crawlFullSite
 };
-//# sourceMappingURL=chunk-OCLAIHX6.js.map
+//# sourceMappingURL=chunk-RYV25AUV.js.map

package/dist/chunk-RYV25AUV.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/full-site-crawler.ts"],"sourcesContent":["/**\n * Full-site crawler for deep AEO audits.\n * BFS crawl that discovers all internal pages up to a configurable limit.\n */\n\nimport type { FetchResult, SiteData, PageCategory } from './site-crawler.js';\n\n// ─── Types ──────────────────────────────────────────────────────────────────\n\nexport interface CrawlOptions {\n  /** Maximum pages to fetch (default 200) */\n  maxPages?: number;\n  /** Per-page fetch timeout in ms (default 10000) */\n  timeoutMs?: number;\n  /** Parallel fetches (default 5) */\n  concurrency?: number;\n  /** Honor robots.txt Disallow rules (default true) */\n  respectRobots?: boolean;\n  /** Include asset files — skipped by default */\n  includeAssets?: boolean;\n  /** Called after each batch with per-URL results */\n  onProgress?: (event: CrawlProgressEvent) => void;\n}\n\nexport interface CrawlProgressEvent {\n  /** URLs attempted in this batch */\n  urls: string[];\n  /** Whether each URL succeeded */\n  results: Array<{ url: string; ok: boolean; status?: number }>;\n  /** Total fetched so far */\n  fetched: number;\n  /** Total discovered so far */\n  discovered: number;\n  /** Max pages limit */\n  maxPages: number;\n}\n\nexport interface CrawlResult {\n  pages: FetchResult[];\n  discoveredUrls: string[];\n  fetchedUrls: string[];\n  skippedUrls: string[];\n  elapsed: number;\n}\n\n// ─── Resource file extensions to skip ────────────────────────────────────────\n\nconst RESOURCE_EXTENSIONS = /\\.(js|css|png|jpg|jpeg|gif|svg|ico|pdf|xml|txt|woff|woff2|ttf|eot|mp4|mp3|webp|avif|zip|gz|tar|json)$/i;\n\nconst SKIP_PATH_PATTERNS = /^\\/(api|wp-admin|wp-json|wp-content|wp-includes|static|assets|_next|auth|login|signup|cart|checkout|admin|feed|xmlrpc|tag|tags|author|authors|category|categories|attachment|embed|trackback|comments|search|print|amp)\\b/i;\n\n/** WordPress pagination and query-string junk */\nconst SKIP_URL_PATTERNS = /\\/page\\/\\d+\\/?$|[?&](s|replytocom|p|preview|share|like|amp)=/i;\n\n// ─── Robots.txt parsing ─────────────────────────────────────────────────────\n\ninterface RobotsRules {\n  disallow: string[];\n  allow: string[];\n}\n\nexport function parseRobotsTxt(robotsText: string): RobotsRules {\n  const lines = robotsText.split('\\n');\n  const rules: RobotsRules = { disallow: [], allow: [] };\n\n  // Collect rules for User-agent: * and User-agent: AEO-Visibility-Bot\n  let inRelevantSection = false;\n\n  for (const rawLine of lines) {\n    const line = rawLine.trim();\n    if (!line || line.startsWith('#')) continue;\n\n    const uaMatch = line.match(/^user-agent:\\s*(.+)/i);\n    if (uaMatch) {\n      const agent = uaMatch[1].trim().toLowerCase();\n      inRelevantSection = agent === '*' || agent === 'aeo-visibility-bot';\n      continue;\n    }\n\n    if (!inRelevantSection) continue;\n\n    const disallowMatch = line.match(/^disallow:\\s*(.*)/i);\n    if (disallowMatch) {\n      const path = disallowMatch[1].trim();\n      if (path) rules.disallow.push(path);\n      continue;\n    }\n\n    const allowMatch = line.match(/^allow:\\s*(.*)/i);\n    if (allowMatch) {\n      const path = allowMatch[1].trim();\n      if (path) rules.allow.push(path);\n    }\n  }\n\n  return rules;\n}\n\nexport function isDisallowedByRobots(urlPath: string, rules: RobotsRules): boolean {\n  // Check allow rules first — more specific (longer) rules take precedence\n  let longestAllow = 0;\n  let longestDisallow = 0;\n\n  for (const pattern of rules.allow) {\n    if (urlPath.startsWith(pattern) && pattern.length > longestAllow) {\n      longestAllow = pattern.length;\n    }\n  }\n\n  for (const pattern of rules.disallow) {\n    if (urlPath.startsWith(pattern) && pattern.length > longestDisallow) {\n      longestDisallow = pattern.length;\n    }\n  }\n\n  // More specific (longer) rule wins; if equal length, allow wins\n  if (longestAllow === 0 && longestDisallow === 0) return false;\n  return longestDisallow > longestAllow;\n}\n\n// ─── Fetch helper (matches multi-page-fetcher.ts fetchPage) ──────────────────\n\nasync function fetchPage(url: string, timeoutMs = 10000): Promise<FetchResult | null> {\n  try {\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(timeoutMs),\n      headers: { 'User-Agent': 'AEO-Visibility-Bot/1.0' },\n      redirect: 'follow',\n    });\n    if (res.status !== 200) return null;\n    const text = await res.text();\n    if (text.length < 200) return null;\n    return { text: text.slice(0, 500_000), status: res.status, finalUrl: res.url };\n  } catch {\n    return null;\n  }\n}\n\nasync function fetchSitemapXml(url: string, timeoutMs = 10000): Promise<string | null> {\n  try {\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(timeoutMs),\n      headers: { 'User-Agent': 'AEO-Visibility-Bot/1.0' },\n      redirect: 'follow',\n    });\n    if (res.status !== 200) return null;\n    return await res.text();\n  } catch {\n    return null;\n  }\n}\n\n// ─── Sitemap parsing ────────────────────────────────────────────────────────\n\n/**\n * Extract all page URLs from sitemap XML (handles sitemapindex with sub-sitemaps).\n * Filters to same domain only, skips resource files.\n */\nexport async function extractAllUrlsFromSitemap(\n  sitemapText: string,\n  domain: string,\n  timeoutMs = 10000,\n): Promise<string[]> {\n  const cleanDomain = domain.replace(/^www\\./, '').toLowerCase();\n  const urls = new Set<string>();\n\n  // Check for sitemapindex — fetch sub-sitemaps\n  const subSitemapLocs = sitemapText.match(/<sitemap>[\\s\\S]*?<loc>([^<]+)<\\/loc>[\\s\\S]*?<\\/sitemap>/gi) || [];\n  if (subSitemapLocs.length > 0) {\n    const subUrls: string[] = [];\n    for (const block of subSitemapLocs) {\n      const locMatch = block.match(/<loc>([^<]+)<\\/loc>/i);\n      if (locMatch) subUrls.push(locMatch[1].trim());\n    }\n\n    // Fetch sub-sitemaps in parallel (limit to 10)\n    const fetches = subUrls.slice(0, 10).map(u => fetchSitemapXml(u, timeoutMs));\n    const results = await Promise.all(fetches);\n    for (const text of results) {\n      if (text) {\n        extractLocsFromXml(text, cleanDomain, urls);\n      }\n    }\n  }\n\n  // Also extract <url><loc> from the main sitemap text (could be a regular sitemap)\n  extractLocsFromXml(sitemapText, cleanDomain, urls);\n\n  return Array.from(urls);\n}\n\nfunction extractLocsFromXml(xml: string, cleanDomain: string, urls: Set<string>): void {\n  const locMatches = xml.match(/<url>[\\s\\S]*?<loc>([^<]+)<\\/loc>[\\s\\S]*?<\\/url>/gi) || [];\n  for (const block of locMatches) {\n    const locMatch = block.match(/<loc>([^<]+)<\\/loc>/i);\n    if (!locMatch) continue;\n    const url = locMatch[1].trim();\n\n    try {\n      const parsed = new URL(url);\n      const urlDomain = parsed.hostname.replace(/^www\\./, '').toLowerCase();\n      if (urlDomain !== cleanDomain) continue;\n      if (RESOURCE_EXTENSIONS.test(parsed.pathname)) continue;\n      urls.add(url);\n    } catch {\n      continue;\n    }\n  }\n}\n\n// ─── Internal link extraction ───────────────────────────────────────────────\n\n/**\n * Extract ALL internal links from HTML (not just nav).\n * Returns deduplicated full URLs for the same domain.\n */\nexport function extractInternalLinks(html: string, domain: string): string[] {\n  const cleanDomain = domain.replace(/^www\\./, '').toLowerCase();\n  const hrefMatches = html.match(/href=\"([^\"]*)\"/gi) || [];\n  const urls = new Set<string>();\n\n  for (const match of hrefMatches) {\n    const href = match.match(/href=\"([^\"]*)\"/i)?.[1];\n    if (!href || !href.trim()) continue;\n\n    let fullUrl: string;\n\n    if (href.startsWith('//')) {\n      fullUrl = `https:${href}`;\n    } else if (href.startsWith('/')) {\n      // Skip fragment-only, query-only, and anchor links\n      if (href === '/' || href.startsWith('/#')) continue;\n      fullUrl = `https://${domain}${href}`;\n    } else if (href.startsWith('http')) {\n      fullUrl = href;\n    } else if (href.startsWith('#') || href.startsWith('?') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) {\n      continue;\n    } else {\n      // Relative path\n      fullUrl = `https://${domain}/${href}`;\n    }\n\n    try {\n      const parsed = new URL(fullUrl);\n      const linkDomain = parsed.hostname.replace(/^www\\./, '').toLowerCase();\n      if (linkDomain !== cleanDomain) continue;\n\n      // Strip hash and normalize\n      parsed.hash = '';\n      const path = parsed.pathname;\n\n      if (path === '/' || path === '') continue;\n      if (RESOURCE_EXTENSIONS.test(path)) continue;\n      if (SKIP_PATH_PATTERNS.test(path)) continue;\n\n      // Normalize: strip trailing slash\n      const normalized = parsed.origin + path.replace(/\\/+$/, '') + parsed.search;\n      if (SKIP_URL_PATTERNS.test(normalized)) continue;\n      urls.add(normalized);\n    } catch {\n      continue;\n    }\n  }\n\n  return Array.from(urls);\n}\n\n// ─── Category inference ─────────────────────────────────────────────────────\n\nconst CATEGORY_PATTERNS: Array<[RegExp, PageCategory]> = [\n  [/\\/([^/]*-?)?(blog|articles?|posts?|news|insights|guides)\\b/i, 'blog'],\n  [/\\/(about|about-us|company|who-we-are)\\b/i, 'about'],\n  [/\\/(pricing|plans|packages)\\b/i, 'pricing'],\n  [/\\/(services?|features?|solutions?|products?|what-we-do|offerings?)\\b/i, 'services'],\n  [/\\/(contact|contact-us|get-in-touch)\\b/i, 'contact'],\n  [/\\/(team|our-team|authors?|people|leadership|staff)\\b/i, 'team'],\n  [/\\/(resources?|resource-center|library|downloads?)\\b/i, 'resources'],\n  [/\\/(docs?|documentation|help|help-center|support|knowledge-base)\\b/i, 'docs'],\n  [/\\/(case-stud\\w*|cases|customers?|success-stor\\w*|testimonials?)\\b/i, 'cases'],\n  [/\\/(faq|frequently-asked|questions)\\b/i, 'faq'],\n];\n\n/**\n * Infer PageCategory from URL path patterns.\n */\nexport function inferCategory(url: string): PageCategory {\n  try {\n    const path = new URL(url).pathname;\n    for (const [pattern, category] of CATEGORY_PATTERNS) {\n      if (pattern.test(path)) return category;\n    }\n  } catch {\n    // Fall through to default\n  }\n  return 'content';\n}\n\n// ─── Main crawler ───────────────────────────────────────────────────────────\n\n/**\n * BFS crawl of a site, discovering all internal pages up to maxPages.\n * Seeds from sitemap URLs + homepage internal links.\n * Skips URLs already in siteData.blogSample and homepage.\n */\nexport async function crawlFullSite(\n  siteData: SiteData,\n  options?: CrawlOptions,\n): Promise<CrawlResult> {\n  const startTime = Date.now();\n  const maxPages = options?.maxPages ?? 200;\n  const timeoutMs = options?.timeoutMs ?? 10000;\n  const concurrency = options?.concurrency ?? 5;\n  const respectRobots = options?.respectRobots ?? true;\n\n  const pages: FetchResult[] = [];\n  const discoveredUrls = new Set<string>();\n  const fetchedUrls = new Set<string>();\n  const skippedUrls = new Set<string>();\n  const visited = new Set<string>();\n\n  // Parse robots.txt rules\n  let robotsRules: RobotsRules = { disallow: [], allow: [] };\n  if (respectRobots && siteData.robotsTxt?.text) {\n    robotsRules = parseRobotsTxt(siteData.robotsTxt.text);\n  }\n\n  const baseUrl = `${siteData.protocol}://${siteData.domain}`;\n\n  // Mark already-fetched URLs as visited\n  visited.add(normalizeUrl(baseUrl));\n  visited.add(normalizeUrl(baseUrl + '/'));\n  if (siteData.blogSample) {\n    for (const page of siteData.blogSample) {\n      if (page.finalUrl) visited.add(normalizeUrl(page.finalUrl));\n    }\n  }\n\n  // Seed the queue from sitemap\n  const queue: string[] = [];\n  if (siteData.sitemapXml?.text) {\n    const sitemapUrls = await extractAllUrlsFromSitemap(\n      siteData.sitemapXml.text,\n      siteData.domain,\n      timeoutMs,\n    );\n    for (const url of sitemapUrls) {\n      const norm = normalizeUrl(url);\n      if (!visited.has(norm)) {\n        discoveredUrls.add(url);\n        if (!queue.includes(url)) queue.push(url);\n      }\n    }\n  }\n\n  // Seed from homepage internal links\n  if (siteData.homepage?.text) {\n    const homeLinks = extractInternalLinks(siteData.homepage.text, siteData.domain);\n    for (const url of homeLinks) {\n      const norm = normalizeUrl(url);\n      if (!visited.has(norm) && !discoveredUrls.has(url)) {\n        discoveredUrls.add(url);\n        if (!queue.includes(url)) queue.push(url);\n      }\n    }\n  }\n\n  // BFS loop\n  while (queue.length > 0 && fetchedUrls.size < maxPages) {\n    // Take a batch\n    const batchSize = Math.min(concurrency, maxPages - fetchedUrls.size, queue.length);\n    const batch: string[] = [];\n\n    while (batch.length < batchSize && queue.length > 0) {\n      const url = queue.shift()!;\n      const norm = normalizeUrl(url);\n\n      if (visited.has(norm)) continue;\n      visited.add(norm);\n\n      // Check robots.txt\n      if (respectRobots) {\n        try {\n          const path = new URL(url).pathname;\n          if (isDisallowedByRobots(path, robotsRules)) {\n            skippedUrls.add(url);\n            continue;\n          }\n        } catch {\n          // Skip malformed URLs\n          continue;\n        }\n      }\n\n      batch.push(url);\n    }\n\n    if (batch.length === 0) continue;\n\n    // Fetch batch in parallel\n    const fetchResults = await Promise.all(batch.map(url => fetchPage(url, timeoutMs)));\n\n    const batchResults: Array<{ url: string; ok: boolean; status?: number }> = [];\n\n    for (let i = 0; i < fetchResults.length; i++) {\n      const result = fetchResults[i];\n      const url = batch[i];\n      fetchedUrls.add(url);\n      batchResults.push({ url, ok: !!result, status: result?.status });\n\n      if (!result) continue;\n\n      result.category = inferCategory(url);\n      pages.push(result);\n\n      // Extract new internal links from fetched page\n      const newLinks = extractInternalLinks(result.text, siteData.domain);\n      for (const link of newLinks) {\n        const norm = normalizeUrl(link);\n        if (!visited.has(norm) && !discoveredUrls.has(link)) {\n          discoveredUrls.add(link);\n          queue.push(link);\n        }\n      }\n    }\n\n    // Notify caller of per-batch progress\n    if (options?.onProgress) {\n      options.onProgress({\n        urls: batch,\n        results: batchResults,\n        fetched: fetchedUrls.size,\n        discovered: discoveredUrls.size,\n        maxPages,\n      });\n    }\n  }\n\n  // Any remaining queued URLs count as discovered but skipped\n  for (const url of queue) {\n    if (!fetchedUrls.has(url)) {\n      skippedUrls.add(url);\n    }\n  }\n\n  return {\n    pages,\n    discoveredUrls: Array.from(discoveredUrls),\n    fetchedUrls: Array.from(fetchedUrls),\n    skippedUrls: Array.from(skippedUrls),\n    elapsed: Math.round((Date.now() - startTime) / 100) / 10,\n  };\n}\n\nfunction normalizeUrl(url: string): string {\n  try {\n    const parsed = new URL(url);\n    // Normalize: lowercase host, strip trailing slash, strip hash\n    return (parsed.origin + parsed.pathname.replace(/\\/+$/, '') + parsed.search).toLowerCase();\n  } catch {\n    return url.toLowerCase();\n  }\n}\n"],"mappings":";AA+CA,IAAM,sBAAsB;AAE5B,IAAM,qBAAqB;AAG3B,IAAM,oBAAoB;AASnB,SAAS,eAAe,YAAiC;AAC9D,QAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,QAAM,QAAqB,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,EAAE;AAGrD,MAAI,oBAAoB;AAExB,aAAW,WAAW,OAAO;AAC3B,UAAM,OAAO,QAAQ,KAAK;AAC1B,QAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,EAAG;AAEnC,UAAM,UAAU,KAAK,MAAM,sBAAsB;AACjD,QAAI,SAAS;AACX,YAAM,QAAQ,QAAQ,CAAC,EAAE,KAAK,EAAE,YAAY;AAC5C,0BAAoB,UAAU,OAAO,UAAU;AAC/C;AAAA,IACF;AAEA,QAAI,CAAC,kBAAmB;AAExB,UAAM,gBAAgB,KAAK,MAAM,oBAAoB;AACrD,QAAI,eAAe;AACjB,YAAM,OAAO,cAAc,CAAC,EAAE,KAAK;AACnC,UAAI,KAAM,OAAM,SAAS,KAAK,IAAI;AAClC;AAAA,IACF;AAEA,UAAM,aAAa,KAAK,MAAM,iBAAiB;AAC/C,QAAI,YAAY;AACd,YAAM,OAAO,WAAW,CAAC,EAAE,KAAK;AAChC,UAAI,KAAM,OAAM,MAAM,KAAK,IAAI;AAAA,IACjC;AAAA,EACF;AAEA,SAAO;AACT;AAEO,SAAS,qBAAqB,SAAiB,OAA6B;AAEjF,MAAI,eAAe;AACnB,MAAI,kBAAkB;AAEtB,aAAW,WAAW,MAAM,OAAO;AACjC,QAAI,QAAQ,WAAW,OAAO,KAAK,QAAQ,SAAS,cAAc;AAChE,qBAAe,QAAQ;AAAA,IACzB;AAAA,EACF;AAEA,aAAW,WAAW,MAAM,UAAU;AACpC,QAAI,QAAQ,WAAW,OAAO,KAAK,QAAQ,SAAS,iBAAiB;AACnE,wBAAkB,QAAQ;AAAA,IAC5B;AAAA,EACF;AAGA,MAAI,iBAAiB,KAAK,oBAAoB,EAAG,QAAO;AACxD,SAAO,kBAAkB;AAC3B;AAIA,eAAe,UAAU,KAAa,YAAY,KAAoC;AACpF,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,SAAS;AAAA,MACrC,SAAS,EAAE,cAAc,yBAAyB;AAAA,MAClD,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,WAAW,IAAK,QAAO;AAC/B,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,QAAI,KAAK,SAAS,IAAK,QAAO;AAC9B,WAAO,EAAE,MAAM,KAAK,MAAM,GAAG,GAAO,GAAG,QAAQ,IAAI,QAAQ,UAAU,IAAI,IAAI;AAAA,EAC/E,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAe,gBAAgB,KAAa,YAAY,KAA+B;AACrF,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,SAAS;AAAA,MACrC,SAAS,EAAE,cAAc,yBAAyB;AAAA,MAClD,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,WAAW,IAAK,QAAO;AAC/B,WAAO,MAAM,IAAI,KAAK;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQA,eAAsB,0BACpB,aACA,QACA,YAAY,KACO;AACnB,QAAM,cAAc,OAAO,QAAQ,UAAU,EAAE,EAAE,YAAY;AAC7D,QAAM,OAAO,oBAAI,IAAY;AAG7B,QAAM,iBAAiB,YAAY,MAAM,2DAA2D,KAAK,CAAC;AAC1G,MAAI,eAAe,SAAS,GAAG;AAC7B,UAAM,UAAoB,CAAC;AAC3B,eAAW,SAAS,gBAAgB;AAClC,YAAM,WAAW,MAAM,MAAM,sBAAsB;AACnD,UAAI,SAAU,SAAQ,KAAK,SAAS,CAAC,EAAE,KAAK,CAAC;AAAA,IAC/C;AAGA,UAAM,UAAU,QAAQ,MAAM,GAAG,EAAE,EAAE,IAAI,OAAK,gBAAgB,GAAG,SAAS,CAAC;AAC3E,UAAM,UAAU,MAAM,QAAQ,IAAI,OAAO;AACzC,eAAW,QAAQ,SAAS;AAC1B,UAAI,MAAM;AACR,2BAAmB,MAAM,aAAa,IAAI;AAAA,MAC5C;AAAA,IACF;AAAA,EACF;AAGA,qBAAmB,aAAa,aAAa,IAAI;AAEjD,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,mBAAmB,KAAa,aAAqB,MAAyB;AACrF,QAAM,aAAa,IAAI,MAAM,mDAAmD,KAAK,CAAC;AACtF,aAAW,SAAS,YAAY;AAC9B,UAAM,WAAW,MAAM,MAAM,sBAAsB;AACnD,QAAI,CAAC,SAAU;AACf,UAAM,MAAM,SAAS,CAAC,EAAE,KAAK;AAE7B,QAAI;AACF,YAAM,SAAS,IAAI,IAAI,GAAG;AAC1B,YAAM,YAAY,OAAO,SAAS,QAAQ,UAAU,EAAE,EAAE,YAAY;AACpE,UAAI,cAAc,YAAa;AAC/B,UAAI,oBAAoB,KAAK,OAAO,QAAQ,EAAG;AAC/C,WAAK,IAAI,GAAG;AAAA,IACd,QAAQ;AACN;AAAA,IACF;AAAA,EACF;AACF;AAQO,SAAS,qBAAqB,MAAc,QAA0B;AAC3E,QAAM,cAAc,OAAO,QAAQ,UAAU,EAAE,EAAE,YAAY;AAC7D,QAAM,cAAc,KAAK,MAAM,kBAAkB,KAAK,CAAC;AACvD,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,SAAS,aAAa;AAC/B,UAAM,OAAO,MAAM,MAAM,iBAAiB,IAAI,CAAC;AAC/C,QAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAG;AAE3B,QAAI;AAEJ,QAAI,KAAK,WAAW,IAAI,GAAG;AACzB,gBAAU,SAAS,IAAI;AAAA,IACzB,WAAW,KAAK,WAAW,GAAG,GAAG;AAE/B,UAAI,SAAS,OAAO,KAAK,WAAW,IAAI,EAAG;AAC3C,gBAAU,WAAW,MAAM,GAAG,IAAI;AAAA,IACpC,WAAW,KAAK,WAAW,MAAM,GAAG;AAClC,gBAAU;AAAA,IACZ,WAAW,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,aAAa,GAAG;AAClJ;AAAA,IACF,OAAO;AAEL,gBAAU,WAAW,MAAM,IAAI,IAAI;AAAA,IACrC;AAEA,QAAI;AACF,YAAM,SAAS,IAAI,IAAI,OAAO;AAC9B,YAAM,aAAa,OAAO,SAAS,QAAQ,UAAU,EAAE,EAAE,YAAY;AACrE,UAAI,eAAe,YAAa;AAGhC,aAAO,OAAO;AACd,YAAM,OAAO,OAAO;AAEpB,UAAI,SAAS,OAAO,SAAS,GAAI;AACjC,UAAI,oBAAoB,KAAK,IAAI,EAAG;AACpC,UAAI,mBAAmB,KAAK,IAAI,EAAG;AAGnC,YAAM,aAAa,OAAO,SAAS,KAAK,QAAQ,QAAQ,EAAE,IAAI,OAAO;AACrE,UAAI,kBAAkB,KAAK,UAAU,EAAG;AACxC,WAAK,IAAI,UAAU;AAAA,IACrB,QAAQ;AACN;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,IAAM,oBAAmD;AAAA,EACvD,CAAC,+DAA+D,MAAM;AAAA,EACtE,CAAC,4CAA4C,OAAO;AAAA,EACpD,CAAC,iCAAiC,SAAS;AAAA,EAC3C,CAAC,yEAAyE,UAAU;AAAA,EACpF,CAAC,0CAA0C,SAAS;AAAA,EACpD,CAAC,yDAAyD,MAAM;AAAA,EAChE,CAAC,wDAAwD,WAAW;AAAA,EACpE,CAAC,sEAAsE,MAAM;AAAA,EAC7E,CAAC,sEAAsE,OAAO;AAAA,EAC9E,CAAC,yCAAyC,KAAK;AACjD;AAKO,SAAS,cAAc,KAA2B;AACvD,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,GAAG,EAAE;AAC1B,eAAW,CAAC,SAAS,QAAQ,KAAK,mBAAmB;AACnD,UAAI,QAAQ,KAAK,IAAI,EAAG,QAAO;AAAA,IACjC;AAAA,EACF,QAAQ;AAAA,EAER;AACA,SAAO;AACT;AASA,eAAsB,cACpB,UACA,SACsB;AACtB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,YAAY,SAAS,aAAa;AACxC,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,gBAAgB,SAAS,iBAAiB;AAEhD,QAAM,QAAuB,CAAC;AAC9B,QAAM,iBAAiB,oBAAI,IAAY;AACvC,QAAM,cAAc,oBAAI,IAAY;AACpC,QAAM,cAAc,oBAAI,IAAY;AACpC,QAAM,UAAU,oBAAI,IAAY;AAGhC,MAAI,cAA2B,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,EAAE;AACzD,MAAI,iBAAiB,SAAS,WAAW,MAAM;AAC7C,kBAAc,eAAe,SAAS,UAAU,IAAI;AAAA,EACtD;AAEA,QAAM,UAAU,GAAG,SAAS,QAAQ,MAAM,SAAS,MAAM;AAGzD,UAAQ,IAAI,aAAa,OAAO,CAAC;AACjC,UAAQ,IAAI,aAAa,UAAU,GAAG,CAAC;AACvC,MAAI,SAAS,YAAY;AACvB,eAAW,QAAQ,SAAS,YAAY;AACtC,UAAI,KAAK,SAAU,SAAQ,IAAI,aAAa,KAAK,QAAQ,CAAC;AAAA,IAC5D;AAAA,EACF;AAGA,QAAM,QAAkB,CAAC;AACzB,MAAI,SAAS,YAAY,MAAM;AAC7B,UAAM,cAAc,MAAM;AAAA,MACxB,SAAS,WAAW;AAAA,MACpB,SAAS;AAAA,MACT;AAAA,IACF;AACA,eAAW,OAAO,aAAa;AAC7B,YAAM,OAAO,aAAa,GAAG;AAC7B,UAAI,CAAC,QAAQ,IAAI,IAAI,GAAG;AACtB,uBAAe,IAAI,GAAG;AACtB,YAAI,CAAC,MAAM,SAAS,GAAG,EAAG,OAAM,KAAK,GAAG;AAAA,MAC1C;AAAA,IACF;AAAA,EACF;AAGA,MAAI,SAAS,UAAU,MAAM;AAC3B,UAAM,YAAY,qBAAqB,SAAS,SAAS,MAAM,SAAS,MAAM;AAC9E,eAAW,OAAO,WAAW;AAC3B,YAAM,OAAO,aAAa,GAAG;AAC7B,UAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,CAAC,eAAe,IAAI,GAAG,GAAG;AAClD,uBAAe,IAAI,GAAG;AACtB,YAAI,CAAC,MAAM,SAAS,GAAG,EAAG,OAAM,KAAK,GAAG;AAAA,MAC1C;AAAA,IACF;AAAA,EACF;AAGA,SAAO,MAAM,SAAS,KAAK,YAAY,OAAO,UAAU;AAEtD,UAAM,YAAY,KAAK,IAAI,aAAa,WAAW,YAAY,MAAM,MAAM,MAAM;AACjF,UAAM,QAAkB,CAAC;AAEzB,WAAO,MAAM,SAAS,aAAa,MAAM,SAAS,GAAG;AACnD,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,OAAO,aAAa,GAAG;AAE7B,UAAI,QAAQ,IAAI,IAAI,EAAG;AACvB,cAAQ,IAAI,IAAI;AAGhB,UAAI,eAAe;AACjB,YAAI;AACF,gBAAM,OAAO,IAAI,IAAI,GAAG,EAAE;AAC1B,cAAI,qBAAqB,MAAM,WAAW,GAAG;AAC3C,wBAAY,IAAI,GAAG;AACnB;AAAA,UACF;AAAA,QACF,QAAQ;AAEN;AAAA,QACF;AAAA,MACF;AAEA,YAAM,KAAK,GAAG;AAAA,IAChB;AAEA,QAAI,MAAM,WAAW,EAAG;AAGxB,UAAM,eAAe,MAAM,QAAQ,IAAI,MAAM,IAAI,SAAO,UAAU,KAAK,SAAS,CAAC,CAAC;AAElF,UAAM,eAAqE,CAAC;AAE5E,aAAS,IAAI,GAAG,IAAI,aAAa,QAAQ,KAAK;AAC5C,YAAM,SAAS,aAAa,CAAC;AAC7B,YAAM,MAAM,MAAM,CAAC;AACnB,kBAAY,IAAI,GAAG;AACnB,mBAAa,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC,QAAQ,QAAQ,QAAQ,OAAO,CAAC;AAE/D,UAAI,CAAC,OAAQ;AAEb,aAAO,WAAW,cAAc,GAAG;AACnC,YAAM,KAAK,MAAM;AAGjB,YAAM,WAAW,qBAAqB,OAAO,MAAM,SAAS,MAAM;AAClE,iBAAW,QAAQ,UAAU;AAC3B,cAAM,OAAO,aAAa,IAAI;AAC9B,YAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,CAAC,eAAe,IAAI,IAAI,GAAG;AACnD,yBAAe,IAAI,IAAI;AACvB,gBAAM,KAAK,IAAI;AAAA,QACjB;AAAA,MACF;AAAA,IACF;AAGA,QAAI,SAAS,YAAY;AACvB,cAAQ,WAAW;AAAA,QACjB,MAAM;AAAA,QACN,SAAS;AAAA,QACT,SAAS,YAAY;AAAA,QACrB,YAAY,eAAe;AAAA,QAC3B;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA,aAAW,OAAO,OAAO;AACvB,QAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,kBAAY,IAAI,GAAG;AAAA,IACrB;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,gBAAgB,MAAM,KAAK,cAAc;AAAA,IACzC,aAAa,MAAM,KAAK,WAAW;AAAA,IACnC,aAAa,MAAM,KAAK,WAAW;AAAA,IACnC,SAAS,KAAK,OAAO,KAAK,IAAI,IAAI,aAAa,GAAG,IAAI;AAAA,EACxD;AACF;AAEA,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,UAAM,SAAS,IAAI,IAAI,GAAG;AAE1B,YAAQ,OAAO,SAAS,OAAO,SAAS,QAAQ,QAAQ,EAAE,IAAI,OAAO,QAAQ,YAAY;AAAA,EAC3F,QAAQ;AACN,WAAO,IAAI,YAAY;AAAA,EACzB;AACF;","names":[]}

package/dist/cli.js

@@ -2556,6 +2556,234 @@ function checkImageContextAI(data) {
   }
   return { criterion: "image_context_ai", criterion_label: "Image Context for AI", score: Math.min(10, score), status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P2" };
 }
+var BOILERPLATE_RE = /\b(sign up|subscribe|get started|contact us|request a demo|free trial|book a call|schedule a|learn more|click here|follow us|share this|copyright|all rights reserved|privacy policy|terms of service)\b/i;
+function isBoilerplateParagraph(text) {
+  const words = text.split(/\s+/).length;
+  if (words < 20 && BOILERPLATE_RE.test(text)) return true;
+  if (/\b(cookie|gdpr|consent|opt.out)\b/i.test(text) && words < 30) return true;
+  return false;
+}
+function toShingles(text, n = 4) {
+  const words = text.split(/\s+/).filter((w) => w.length > 1);
+  const shingles = /* @__PURE__ */ new Set();
+  for (let i = 0; i <= words.length - n; i++) {
+    shingles.add(words.slice(i, i + n).join(" "));
+  }
+  return shingles;
+}
+function shingleSimilarity(a, b) {
+  if (a.size === 0 && b.size === 0) return 0;
+  let intersection = 0;
+  for (const s of a) {
+    if (b.has(s)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+function extractPageParagraphs(html) {
+  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
+  const pMatches = cleaned.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
+  return pMatches.map((p) => {
+    const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
+    return { text, shingles: toShingles(text) };
+  }).filter((p) => p.shingles.size >= 3 && !isBoilerplateParagraph(p.text));
+}
+function splitIntoSectionsWithParagraphs(html) {
+  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
+  const parts = cleaned.split(/(?=<h[23]\b[^>]*>)/i);
+  const sections = [];
+  for (const part of parts) {
+    const hMatch = part.match(/<h[23]\b[^>]*>([\s\S]*?)<\/h[23]>/i);
+    const heading = hMatch ? hMatch[1].replace(/<[^>]*>/g, "").trim() : "(intro)";
+    const pMatches = part.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
+    const paragraphs = pMatches.map((p) => {
+      const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
+      return { text, shingles: toShingles(text) };
+    }).filter((p) => p.shingles.size >= 3 && !isBoilerplateParagraph(p.text));
+    if (paragraphs.length > 0) sections.push({ heading, paragraphs });
+  }
+  return sections;
+}
+function findIntraPageDuplicates(html) {
+  const sections = splitIntoSectionsWithParagraphs(html);
+  if (sections.length < 2) return [];
+  const pairs = [];
+  for (let i = 0; i < sections.length; i++) {
+    for (let j = i + 1; j < sections.length; j++) {
+      let found = false;
+      for (const pA of sections[i].paragraphs) {
+        if (found) break;
+        for (const pB of sections[j].paragraphs) {
+          const sim = shingleSimilarity(pA.shingles, pB.shingles);
+          if (sim > 0.4) {
+            pairs.push({
+              headingA: sections[i].heading,
+              headingB: sections[j].heading,
+              similarity: Math.round(sim * 100),
+              sample: pA.text.slice(0, 80)
+            });
+            found = true;
+            break;
+          }
+        }
+      }
+    }
+  }
+  return pairs;
+}
+function checkDuplicateContent(data) {
+  const findings = [];
+  const pages = [];
+  if (data.homepage) {
+    pages.push({ html: data.homepage.text, url: data.homepage.finalUrl || `https://${data.domain}/` });
+  }
+  if (data.blogSample) {
+    for (const page of data.blogSample) {
+      pages.push({ html: page.text, url: page.finalUrl || "" });
+    }
+  }
+  if (pages.length === 0) {
+    findings.push({ severity: "critical", detail: "No pages available for duplicate content analysis" });
+    return { criterion: "duplicate_content", criterion_label: "Duplicate Content Blocks", score: 0, status: "fail", findings, fix_priority: "P1" };
+  }
+  let totalDupPages = 0;
+  let totalDupPairs = 0;
+  const dupDetails = [];
+  for (const page of pages) {
+    const pairs = findIntraPageDuplicates(page.html);
+    if (pairs.length > 0) {
+      totalDupPages++;
+      totalDupPairs += pairs.length;
+      dupDetails.push({ url: page.url, pairs });
+    }
+  }
+  const dupRatio = totalDupPages / pages.length;
+  let score;
+  if (totalDupPairs === 0) {
+    score = 10;
+    findings.push({ severity: "info", detail: `${pages.length} pages analyzed - no duplicate content blocks detected` });
+  } else if (dupRatio <= 0.05 && totalDupPairs <= 2) {
+    score = 9;
+    findings.push({ severity: "info", detail: `${totalDupPairs} duplicate block pair(s) on ${totalDupPages} page(s) - minor` });
+  } else if (dupRatio <= 0.1) {
+    score = 7;
+    findings.push({ severity: "low", detail: `${totalDupPairs} duplicate block pair(s) across ${totalDupPages} page(s)`, fix: "Rewrite duplicate sections to provide unique content in each" });
+  } else if (dupRatio <= 0.2) {
+    score = 5;
+    findings.push({ severity: "medium", detail: `${totalDupPages} pages (${Math.round(dupRatio * 100)}%) contain duplicate content blocks`, fix: "Rewrite or remove repeated text blocks - LLMs may flag this as low-quality content" });
+  } else if (dupRatio <= 0.4) {
+    score = 3;
+    findings.push({ severity: "medium", detail: `${totalDupPages} pages (${Math.round(dupRatio * 100)}%) have significant duplicate content`, fix: "Widespread duplicate blocks reduce content authority - rewrite each section with unique angles" });
+  } else {
+    score = 0;
+    findings.push({ severity: "high", detail: `${totalDupPages} pages (${Math.round(dupRatio * 100)}%) contain duplicate content blocks`, fix: "Severe content duplication across the site - LLMs will likely reduce citation authority" });
+  }
+  for (const dup of dupDetails.slice(0, 3)) {
+    const shortUrl = dup.url.slice(0, 60);
+    for (const pair of dup.pairs.slice(0, 2)) {
+      findings.push({
+        severity: "low",
+        detail: `${shortUrl}: '${pair.headingA}' and '${pair.headingB}' share ${pair.similarity}% similar text ("${pair.sample}...")`,
+        fix: `Rewrite one of these sections to eliminate duplicate content`
+      });
+    }
+  }
+  return { criterion: "duplicate_content", criterion_label: "Duplicate Content Blocks", score, status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P1" };
+}
+function checkCrossPageDuplication(data) {
+  const findings = [];
+  const pages = [];
+  if (data.homepage) {
+    pages.push({ url: data.homepage.finalUrl || `https://${data.domain}/`, paragraphs: extractPageParagraphs(data.homepage.text) });
+  }
+  if (data.blogSample) {
+    for (const page of data.blogSample) {
+      pages.push({ url: page.finalUrl || "", paragraphs: extractPageParagraphs(page.text) });
+    }
+  }
+  if (pages.length <= 1) {
+    findings.push({ severity: "info", detail: "Not enough pages to assess cross-page duplication" });
+    return { criterion: "cross_page_duplication", criterion_label: "Cross-Page Duplicate Content", score: 5, status: "partial", findings, fix_priority: "P3" };
+  }
+  const paragraphPageCount = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    const seen = /* @__PURE__ */ new Set();
+    for (const p of page.paragraphs) {
+      const fp = [...p.shingles].slice(0, 5).join("|");
+      if (!seen.has(fp)) {
+        seen.add(fp);
+        paragraphPageCount.set(fp, (paragraphPageCount.get(fp) || 0) + 1);
+      }
+    }
+  }
+  const boilerplateThreshold = Math.max(3, pages.length * 0.4);
+  const siteBoilerprints = /* @__PURE__ */ new Set();
+  for (const [fp, count] of paragraphPageCount) {
+    if (count >= boilerplateThreshold) siteBoilerprints.add(fp);
+  }
+  const crossDupPairs = [];
+  for (let i = 0; i < pages.length; i++) {
+    for (let j = i + 1; j < pages.length; j++) {
+      let dupCount = 0;
+      let sample = "";
+      for (const pA of pages[i].paragraphs) {
+        const fpA = [...pA.shingles].slice(0, 5).join("|");
+        if (siteBoilerprints.has(fpA)) continue;
+        for (const pB of pages[j].paragraphs) {
+          const sim = shingleSimilarity(pA.shingles, pB.shingles);
+          if (sim > 0.4) {
+            dupCount++;
+            if (!sample) sample = pA.text.slice(0, 80);
+            break;
+          }
+        }
+      }
+      if (dupCount >= 2) {
+        crossDupPairs.push({
+          urlA: pages[i].url.slice(0, 60),
+          urlB: pages[j].url.slice(0, 60),
+          dupCount,
+          sample
+        });
+      }
+    }
+  }
+  const affectedUrls = /* @__PURE__ */ new Set();
+  for (const pair of crossDupPairs) {
+    affectedUrls.add(pair.urlA);
+    affectedUrls.add(pair.urlB);
+  }
+  const affectedRatio = pages.length > 0 ? affectedUrls.size / pages.length : 0;
+  const totalDupParagraphs = crossDupPairs.reduce((s, p) => s + p.dupCount, 0);
+  let score;
+  if (crossDupPairs.length === 0) {
+    score = 10;
+    findings.push({ severity: "info", detail: `${pages.length} pages analyzed - no cross-page content duplication detected` });
+  } else if (affectedRatio <= 0.05 && totalDupParagraphs <= 4) {
+    score = 9;
+    findings.push({ severity: "info", detail: `${totalDupParagraphs} shared paragraph(s) across ${affectedUrls.size} page(s) - minor` });
+  } else if (affectedRatio <= 0.1) {
+    score = 7;
+    findings.push({ severity: "low", detail: `${totalDupParagraphs} shared paragraphs across ${affectedUrls.size} pages`, fix: "Rewrite shared content so each page provides a unique perspective" });
+  } else if (affectedRatio <= 0.2) {
+    score = 5;
+    findings.push({ severity: "medium", detail: `${affectedUrls.size} pages (${Math.round(affectedRatio * 100)}%) share duplicate paragraphs`, fix: "Significant cross-page duplication - AI engines may only index one version" });
+  } else if (affectedRatio <= 0.4) {
+    score = 3;
+    findings.push({ severity: "medium", detail: `${affectedUrls.size} pages (${Math.round(affectedRatio * 100)}%) contain shared content blocks`, fix: "Widespread copy-paste content across pages reduces overall site authority" });
+  } else {
+    score = 0;
+    findings.push({ severity: "high", detail: `${affectedUrls.size} pages (${Math.round(affectedRatio * 100)}%) share duplicate content`, fix: "Severe cross-page duplication - AI engines will likely ignore redundant pages entirely" });
+  }
+  for (const pair of crossDupPairs.slice(0, 3)) {
+    findings.push({
+      severity: "low",
+      detail: `${pair.dupCount} shared paragraph(s): ${pair.urlA} \u2194 ${pair.urlB} ("${pair.sample}...")`,
+      fix: "Rewrite shared paragraphs so each page has unique content"
+    });
+  }
+  return { criterion: "cross_page_duplication", criterion_label: "Cross-Page Duplicate Content", score, status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P1" };
+}
 function auditSiteFromData(data) {
   const topicCoherence = checkTopicCoherence(data);
   const cannibalization = checkContentCannibalization(data, topicCoherence.score);
@@ -2594,7 +2822,10 @@ function auditSiteFromData(data) {
     checkEvidencePackaging(data),
     checkEntityDisambiguation(data),
     checkExtractionFriction(data),
-    checkImageContextAI(data)
+    checkImageContextAI(data),
+    // V3 criteria (#35-#36)
+    checkDuplicateContent(data),
+    checkCrossPageDuplication(data)
   ];
 }
 
@@ -2612,11 +2843,11 @@ var WEIGHTS = {
   // Information density per page
   direct_answer_density: 0.05,
   // Direct answers to queries
-  qa_content_format: 0.05,
+  qa_content_format: 0.04,
   // Answer-shaped content structure
-  query_answer_alignment: 0.05,
+  query_answer_alignment: 0.04,
   // Relevance to actual AI queries
-  faq_section: 0.04,
+  faq_section: 0.03,
   // Structured Q&A pairs
   // ─── Content Organization (~30%) ──────────────────────────────────────────
   // HOW easily AI engines can extract and trust your content.
@@ -2664,8 +2895,13 @@ var WEIGHTS = {
   // Clear entity boundaries
   extraction_friction: 0.02,
   // Sentence length, voice, jargon
-  image_context_ai: 0.01
+  image_context_ai: 0.01,
   // Figure/figcaption, alt text quality
+  // ─── V3 Criteria ────────────────────────────────────────────────────────
+  duplicate_content: 0.05,
+  // Duplicate text blocks within pages
+  cross_page_duplication: 0.03
+  // Same paragraphs copied across pages
 };
 function calculateOverallScore(criteria) {
   let totalWeight = 0;
@@ -2778,7 +3014,9 @@ var PILLARS = {
     "fact_density",
     "citation_ready_writing",
     "answer_first_placement",
-    "evidence_packaging"
+    "evidence_packaging",
+    "duplicate_content",
+    "cross_page_duplication"
   ],
   "Content Structure": [
     "direct_answer_density",
@@ -2843,6 +3081,8 @@ var CLIENT_NAMES = {
   image_context_ai: "Image Context for AI",
   schema_coverage: "Schema Coverage",
   speakable_schema: "Speakable Schema",
+  duplicate_content: "Duplicate Content Blocks",
+  cross_page_duplication: "Cross-Page Duplicate Content",
   content_cannibalization: "Content Cannibalization",
   llms_txt: "llms.txt File",
   robots_txt: "robots.txt for AI",
@@ -2860,10 +3100,12 @@ var PILLAR_WEIGHTS = {
   citation_ready_writing: 0.04,
   answer_first_placement: 0.03,
   evidence_packaging: 0.03,
+  duplicate_content: 0.05,
+  cross_page_duplication: 0.03,
   direct_answer_density: 0.05,
-  qa_content_format: 0.05,
-  query_answer_alignment: 0.05,
-  faq_section: 0.04,
+  qa_content_format: 0.04,
+  query_answer_alignment: 0.04,
+  faq_section: 0.03,
   table_list_extractability: 0.03,
   definition_patterns: 0.02,
   entity_disambiguation: 0.02,
@@ -2896,6 +3138,8 @@ var CRITERION_EFFORT = {
   citation_ready_writing: "Medium",
   answer_first_placement: "Medium",
   evidence_packaging: "Medium",
+  duplicate_content: "Medium",
+  cross_page_duplication: "Medium",
   direct_answer_density: "Medium",
   qa_content_format: "Medium",
   query_answer_alignment: "Medium",
@@ -2951,6 +3195,8 @@ var FIX_DESCRIPTIONS = {
   image_context_ai: "Wrap images in <figure>/<figcaption> with descriptive alt text.",
   schema_coverage: "Extend structured data to inner pages (articles, services, products).",
   speakable_schema: "Add SpeakableSpecification schema for voice assistant compatibility.",
+  duplicate_content: "Rewrite duplicate text blocks so each section provides unique value.",
+  cross_page_duplication: "Rewrite shared paragraphs across pages so each page has unique content.",
   content_cannibalization: "Consolidate overlapping pages or differentiate titles and H1 headings.",
   llms_txt: "Create a /llms.txt file describing your site for AI engines.",
   robots_txt: "Update robots.txt to explicitly allow AI crawlers.",
@@ -3046,7 +3292,9 @@ var CRITERION_LABELS = {
   "Evidence Packaging": "Evidence Packaging",
   "Entity Disambiguation": "Entity Disambiguation",
   "Extraction Friction Score": "Extraction Friction Score",
-  "Image Context for AI": "Image Context for AI"
+  "Image Context for AI": "Image Context for AI",
+  "Duplicate Content Blocks": "Duplicate Content Blocks",
+  "Cross-Page Duplicate Content": "Cross-Page Duplicate Content"
 };
 function scoreToStatus(score) {
   if (score === 0) return "MISSING";
@@ -3141,9 +3389,9 @@ var CRITERION_WEIGHTS = {
   content_depth: 0.07,
   fact_density: 0.06,
   direct_answer_density: 0.05,
-  qa_content_format: 0.05,
-  query_answer_alignment: 0.05,
-  faq_section: 0.04,
+  qa_content_format: 0.04,
+  query_answer_alignment: 0.04,
+  faq_section: 0.03,
   // Content Organization (~30%)
   entity_consistency: 0.05,
   internal_linking: 0.04,
@@ -3172,7 +3420,10 @@ var CRITERION_WEIGHTS = {
   evidence_packaging: 0.03,
   entity_disambiguation: 0.02,
   extraction_friction: 0.02,
-  image_context_ai: 0.01
+  image_context_ai: 0.01,
+  // V3 Criteria
+  duplicate_content: 0.05,
+  cross_page_duplication: 0.03
 };
 var OPPORTUNITY_TEMPLATES = {
   llms_txt: {
@@ -3295,6 +3546,16 @@ var OPPORTUNITY_TEMPLATES = {
     effort: "Medium",
     description: "Ensure every question-format heading (H2/H3) is followed by a direct answer paragraph. This pattern is ideal for AI engine snippet extraction."
   },
+  duplicate_content: {
+    name: "Fix Duplicate Content Blocks",
+    effort: "Medium",
+    description: "Sections within pages contain identical or near-identical text. LLMs may flag this as low-quality or thin content, reducing citation authority. Rewrite duplicate blocks with unique angles."
+  },
+  cross_page_duplication: {
+    name: "Eliminate Cross-Page Duplicate Content",
+    effort: "Medium",
+    description: "The same paragraphs appear on multiple pages. AI engines may only index one version and ignore the rest. Rewrite shared content so each page offers a unique perspective."
+  },
   content_cannibalization: {
     name: "Resolve Content Cannibalization",
     effort: "Medium",
@@ -3703,9 +3964,9 @@ var PAGE_CRITERIA = {
   original_data: { weight: 0.1, label: "Original Data & Expert Content" },
   fact_density: { weight: 0.06, label: "Fact & Data Density" },
   direct_answer_density: { weight: 0.05, label: "Direct Answer Paragraphs" },
-  qa_content_format: { weight: 0.05, label: "Q&A Content Format" },
-  query_answer_alignment: { weight: 0.05, label: "Query-Answer Alignment" },
-  faq_section: { weight: 0.04, label: "FAQ Section Content" },
+  qa_content_format: { weight: 0.04, label: "Q&A Content Format" },
+  query_answer_alignment: { weight: 0.04, label: "Query-Answer Alignment" },
+  faq_section: { weight: 0.03, label: "FAQ Section Content" },
   // Content Organization
   content_freshness: { weight: 0.04, label: "Content Freshness Signals" },
   schema_markup: { weight: 0.03, label: "Schema.org Structured Data" },
@@ -3722,7 +3983,8 @@ var PAGE_CRITERIA = {
   evidence_packaging: { weight: 0.03, label: "Evidence Packaging" },
   entity_disambiguation: { weight: 0.02, label: "Entity Disambiguation" },
   extraction_friction: { weight: 0.02, label: "Extraction Friction Score" },
-  image_context_ai: { weight: 0.01, label: "Image Context for AI" }
+  image_context_ai: { weight: 0.01, label: "Image Context for AI" },
+  duplicate_content: { weight: 0.05, label: "Duplicate Content Blocks" }
 };
 function extractJsonLdBlocks(html) {
   const blocks = [];
@@ -4171,6 +4433,90 @@ function scoreImageContextAI(html) {
   if (contextualImages.length > 0) score += 3;
   return cap(score, 10);
 }
+var BOILERPLATE_PATTERNS = /\b(sign up|subscribe|get started|contact us|request a demo|free trial|book a call|schedule a|learn more|click here|follow us|share this|copyright|all rights reserved|privacy policy|terms of service)\b/i;
+function isBoilerplate(text) {
+  const words = text.split(/\s+/).length;
+  if (words < 20 && BOILERPLATE_PATTERNS.test(text)) return true;
+  if (/\b(cookie|gdpr|consent|opt.out)\b/i.test(text) && words < 30) return true;
+  return false;
+}
+function scoreDuplicateContent(html) {
+  return scoreDuplicateContentDetailed(html).score;
+}
+function scoreDuplicateContentDetailed(html) {
+  const sections = extractSectionsWithParagraphs(html);
+  if (sections.length < 2) return { score: 10, duplicates: [] };
+  const totalParagraphs = sections.reduce((sum, s) => sum + s.paragraphs.length, 0);
+  const duplicates = [];
+  let dupParagraphCount = 0;
+  for (let i = 0; i < sections.length; i++) {
+    for (let j = i + 1; j < sections.length; j++) {
+      for (const pA of sections[i].paragraphs) {
+        for (const pB of sections[j].paragraphs) {
+          const sim = shingleJaccard(pA.shingles, pB.shingles);
+          if (sim > 0.4) {
+            dupParagraphCount++;
+            duplicates.push({
+              headingA: sections[i].heading,
+              headingB: sections[j].heading,
+              similarity: Math.round(sim * 100),
+              sample: pA.text.slice(0, 80)
+            });
+            break;
+          }
+        }
+      }
+    }
+  }
+  if (dupParagraphCount === 0) return { score: 10, duplicates: [] };
+  const dupRatio = totalParagraphs > 0 ? dupParagraphCount / totalParagraphs : 0;
+  let score;
+  if (dupParagraphCount === 1 && dupRatio <= 0.05) {
+    score = 6;
+  } else if (dupParagraphCount === 1) {
+    score = 4;
+  } else if (dupParagraphCount === 2) {
+    score = 2;
+  } else {
+    score = 0;
+  }
+  return { score, duplicates };
+}
+function extractSectionsWithParagraphs(html) {
+  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
+  const parts = cleaned.split(/(?=<h[23]\b[^>]*>)/i);
+  const sections = [];
+  for (const part of parts) {
+    const headingMatch = part.match(/<h[23]\b[^>]*>([\s\S]*?)<\/h[23]>/i);
+    const heading = headingMatch ? headingMatch[1].replace(/<[^>]*>/g, "").trim() : "(intro)";
+    const pMatches = part.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
+    const paragraphs = pMatches.map((p) => {
+      const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
+      return { text, shingles: buildShingles(text, 4) };
+    }).filter((p) => p.shingles.size >= 3 && !isBoilerplate(p.text));
+    if (paragraphs.length > 0) {
+      sections.push({ heading, paragraphs });
+    }
+  }
+  return sections;
+}
+function buildShingles(text, n) {
+  const words = text.split(/\s+/).filter((w) => w.length > 1);
+  const shingles = /* @__PURE__ */ new Set();
+  for (let i = 0; i <= words.length - n; i++) {
+    shingles.add(words.slice(i, i + n).join(" "));
+  }
+  return shingles;
+}
+function shingleJaccard(a, b) {
+  if (a.size === 0 && b.size === 0) return 0;
+  let intersection = 0;
+  for (const s of a) {
+    if (b.has(s)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
 var SCORING_FUNCTIONS = {
   schema_markup: scoreSchemaMarkup,
   qa_content_format: scoreQAFormat,
@@ -4191,7 +4537,8 @@ var SCORING_FUNCTIONS = {
   evidence_packaging: scoreEvidencePackaging,
   entity_disambiguation: scoreEntityDisambiguation,
   extraction_friction: scoreExtractionFriction,
-  image_context_ai: scoreImageContextAI
+  image_context_ai: scoreImageContextAI,
+  duplicate_content: scoreDuplicateContent
 };
 function scorePage(html, url) {
   let totalWeight = 0;
@@ -4205,6 +4552,11 @@ function scorePage(html, url) {
     totalWeight += weight;
   }
   let aeoScore = totalWeight === 0 ? 0 : Math.round(weightedSum / totalWeight);
+  const dupScore = criterionScores.find((c) => c.criterion === "duplicate_content")?.score ?? 10;
+  if (dupScore <= 6) {
+    const dupCap = 35 + dupScore * 5;
+    aeoScore = Math.min(aeoScore, dupCap);
+  }
   const scoreCapped = aeoScore > 75;
   if (scoreCapped) aeoScore = 75;
   return { aeoScore, criterionScores, scoreCapped };
@@ -4410,6 +4762,15 @@ function checkHasCitationReadyContent(html) {
   }
   return null;
 }
+function checkDuplicateContentBlocks(html) {
+  const { score, duplicates } = scoreDuplicateContentDetailed(html);
+  if (score <= 6 && duplicates.length > 0) {
+    const first = duplicates[0];
+    const label = duplicates.length === 1 ? `Duplicate content: '${first.headingA}' and '${first.headingB}' share ${first.similarity}% similar text ("${first.sample}...")` : `${duplicates.length} duplicate blocks found (e.g. '${first.headingA}' and '${first.headingB}' \u2014 "${first.sample}...")`;
+    return { check: "duplicate-content", label, severity: score <= 3 ? "error" : "warning" };
+  }
+  return null;
+}
 function analyzePage(html, url, category) {
   const title = extractTitle(html);
   const textContent = getTextContent2(html);
@@ -4428,7 +4789,8 @@ function analyzePage(html, url, category) {
     checkImagesMissingAlt(html),
     checkNoInternalLinks(html, url),
     checkNoAnswerBlock(html),
-    checkNoEvidence(html, url)
+    checkNoEvidence(html, url),
+    checkDuplicateContentBlocks(html)
   ];
   for (const result of issueChecks) {
     if (result) issues.push(result);
@@ -4496,7 +4858,7 @@ async function audit(domain, options) {
     }
   }
   if (options?.fullCrawl) {
-    const { crawlFullSite } = await import("./full-site-crawler-5AYKCZQY.js");
+    const { crawlFullSite } = await import("./full-site-crawler-OBECS7AT.js");
     const crawlResult = await crawlFullSite(siteData, {
       maxPages: options.maxPages ?? 200,
       concurrency: options.concurrency ?? 5
@@ -5060,6 +5422,10 @@ function printSummary(result) {
       const issueLabel = issueCount === 0 ? "0 issues" : issueCount === 1 ? "1 issue" : `${issueCount} issues`;
       const aeoLabel = page.aeoScore != null ? ` [AEO: ${page.aeoScore}]` : "";
       log(`    ${cat.padEnd(10)} ${page.url.padEnd(50)} ${issueLabel}${aeoLabel}`);
+      const dupIssue = page.issues.find((i) => i.check === "duplicate-content");
+      if (dupIssue) {
+        log(`              \u26A0 ${dupIssue.label}`);
+      }
     }
     const scored = result.pagesReviewed.filter((p) => p.aeoScore != null);
     if (scored.length > 0) {