aeorank 1.6.0 → 2.0.0

package/dist/{full-site-crawler-F7J2HRL4.js → full-site-crawler-FQYO46YV.js}

@@ -151,7 +151,7 @@ function extractInternalLinks(html, domain) {
   return Array.from(urls);
 }
 var CATEGORY_PATTERNS = [
-  [/\/(blog|articles?|posts?|news|insights|guides)\b/i, "blog"],
+  [/\/([^/]*-?)?(blog|articles?|posts?|news|insights|guides)\b/i, "blog"],
   [/\/(about|about-us|company|who-we-are)\b/i, "about"],
   [/\/(pricing|plans|packages)\b/i, "pricing"],
   [/\/(services?|features?|solutions?|products?|what-we-do|offerings?)\b/i, "services"],
@@ -289,4 +289,4 @@ export {
   isDisallowedByRobots,
   parseRobotsTxt
 };
-//# sourceMappingURL=full-site-crawler-F7J2HRL4.js.map
+//# sourceMappingURL=full-site-crawler-FQYO46YV.js.map

package/dist/full-site-crawler-FQYO46YV.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/full-site-crawler.ts"],"sourcesContent":["/**\n * Full-site crawler for deep AEO audits.\n * BFS crawl that discovers all internal pages up to a configurable limit.\n */\n\nimport type { FetchResult, SiteData, PageCategory } from './site-crawler.js';\n\n// ─── Types ──────────────────────────────────────────────────────────────────\n\nexport interface CrawlOptions {\n  /** Maximum pages to fetch (default 200) */\n  maxPages?: number;\n  /** Per-page fetch timeout in ms (default 10000) */\n  timeoutMs?: number;\n  /** Parallel fetches (default 5) */\n  concurrency?: number;\n  /** Honor robots.txt Disallow rules (default true) */\n  respectRobots?: boolean;\n  /** Include asset files — skipped by default */\n  includeAssets?: boolean;\n}\n\nexport interface CrawlResult {\n  pages: FetchResult[];\n  discoveredUrls: string[];\n  fetchedUrls: string[];\n  skippedUrls: string[];\n  elapsed: number;\n}\n\n// ─── Resource file extensions to skip ────────────────────────────────────────\n\nconst RESOURCE_EXTENSIONS = /\\.(js|css|png|jpg|jpeg|gif|svg|ico|pdf|xml|txt|woff|woff2|ttf|eot|mp4|mp3|webp|avif|zip|gz|tar|json)$/i;\n\nconst SKIP_PATH_PATTERNS = /^\\/(api|wp-admin|wp-json|static|assets|_next|auth|login|signup|cart|checkout|admin|feed|xmlrpc)\\b/i;\n\n// ─── Robots.txt parsing ─────────────────────────────────────────────────────\n\ninterface RobotsRules {\n  disallow: string[];\n  allow: string[];\n}\n\nexport function parseRobotsTxt(robotsText: string): RobotsRules {\n  const lines = robotsText.split('\\n');\n  const rules: RobotsRules = { disallow: [], allow: [] };\n\n  // Collect rules for User-agent: * and User-agent: AEO-Visibility-Bot\n  let inRelevantSection = false;\n\n  for (const rawLine of lines) {\n    const line = rawLine.trim();\n    if (!line || line.startsWith('#')) continue;\n\n    const uaMatch = line.match(/^user-agent:\\s*(.+)/i);\n    if (uaMatch) {\n      const agent = uaMatch[1].trim().toLowerCase();\n      inRelevantSection = agent === '*' || agent === 'aeo-visibility-bot';\n      continue;\n    }\n\n    if (!inRelevantSection) continue;\n\n    const disallowMatch = line.match(/^disallow:\\s*(.*)/i);\n    if (disallowMatch) {\n      const path = disallowMatch[1].trim();\n      if (path) rules.disallow.push(path);\n      continue;\n    }\n\n    const allowMatch = line.match(/^allow:\\s*(.*)/i);\n    if (allowMatch) {\n      const path = allowMatch[1].trim();\n      if (path) rules.allow.push(path);\n    }\n  }\n\n  return rules;\n}\n\nexport function isDisallowedByRobots(urlPath: string, rules: RobotsRules): boolean {\n  // Check allow rules first — more specific (longer) rules take precedence\n  let longestAllow = 0;\n  let longestDisallow = 0;\n\n  for (const pattern of rules.allow) {\n    if (urlPath.startsWith(pattern) && pattern.length > longestAllow) {\n      longestAllow = pattern.length;\n    }\n  }\n\n  for (const pattern of rules.disallow) {\n    if (urlPath.startsWith(pattern) && pattern.length > longestDisallow) {\n      longestDisallow = pattern.length;\n    }\n  }\n\n  // More specific (longer) rule wins; if equal length, allow wins\n  if (longestAllow === 0 && longestDisallow === 0) return false;\n  return longestDisallow > longestAllow;\n}\n\n// ─── Fetch helper (matches multi-page-fetcher.ts fetchPage) ──────────────────\n\nasync function fetchPage(url: string, timeoutMs = 10000): Promise<FetchResult | null> {\n  try {\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(timeoutMs),\n      headers: { 'User-Agent': 'AEO-Visibility-Bot/1.0' },\n      redirect: 'follow',\n    });\n    if (res.status !== 200) return null;\n    const text = await res.text();\n    if (text.length < 200) return null;\n    return { text: text.slice(0, 500_000), status: res.status, finalUrl: res.url };\n  } catch {\n    return null;\n  }\n}\n\nasync function fetchSitemapXml(url: string, timeoutMs = 10000): Promise<string | null> {\n  try {\n    const res = await fetch(url, {\n      signal: AbortSignal.timeout(timeoutMs),\n      headers: { 'User-Agent': 'AEO-Visibility-Bot/1.0' },\n      redirect: 'follow',\n    });\n    if (res.status !== 200) return null;\n    return await res.text();\n  } catch {\n    return null;\n  }\n}\n\n// ─── Sitemap parsing ────────────────────────────────────────────────────────\n\n/**\n * Extract all page URLs from sitemap XML (handles sitemapindex with sub-sitemaps).\n * Filters to same domain only, skips resource files.\n */\nexport async function extractAllUrlsFromSitemap(\n  sitemapText: string,\n  domain: string,\n  timeoutMs = 10000,\n): Promise<string[]> {\n  const cleanDomain = domain.replace(/^www\\./, '').toLowerCase();\n  const urls = new Set<string>();\n\n  // Check for sitemapindex — fetch sub-sitemaps\n  const subSitemapLocs = sitemapText.match(/<sitemap>[\\s\\S]*?<loc>([^<]+)<\\/loc>[\\s\\S]*?<\\/sitemap>/gi) || [];\n  if (subSitemapLocs.length > 0) {\n    const subUrls: string[] = [];\n    for (const block of subSitemapLocs) {\n      const locMatch = block.match(/<loc>([^<]+)<\\/loc>/i);\n      if (locMatch) subUrls.push(locMatch[1].trim());\n    }\n\n    // Fetch sub-sitemaps in parallel (limit to 10)\n    const fetches = subUrls.slice(0, 10).map(u => fetchSitemapXml(u, timeoutMs));\n    const results = await Promise.all(fetches);\n    for (const text of results) {\n      if (text) {\n        extractLocsFromXml(text, cleanDomain, urls);\n      }\n    }\n  }\n\n  // Also extract <url><loc> from the main sitemap text (could be a regular sitemap)\n  extractLocsFromXml(sitemapText, cleanDomain, urls);\n\n  return Array.from(urls);\n}\n\nfunction extractLocsFromXml(xml: string, cleanDomain: string, urls: Set<string>): void {\n  const locMatches = xml.match(/<url>[\\s\\S]*?<loc>([^<]+)<\\/loc>[\\s\\S]*?<\\/url>/gi) || [];\n  for (const block of locMatches) {\n    const locMatch = block.match(/<loc>([^<]+)<\\/loc>/i);\n    if (!locMatch) continue;\n    const url = locMatch[1].trim();\n\n    try {\n      const parsed = new URL(url);\n      const urlDomain = parsed.hostname.replace(/^www\\./, '').toLowerCase();\n      if (urlDomain !== cleanDomain) continue;\n      if (RESOURCE_EXTENSIONS.test(parsed.pathname)) continue;\n      urls.add(url);\n    } catch {\n      continue;\n    }\n  }\n}\n\n// ─── Internal link extraction ───────────────────────────────────────────────\n\n/**\n * Extract ALL internal links from HTML (not just nav).\n * Returns deduplicated full URLs for the same domain.\n */\nexport function extractInternalLinks(html: string, domain: string): string[] {\n  const cleanDomain = domain.replace(/^www\\./, '').toLowerCase();\n  const hrefMatches = html.match(/href=\"([^\"]*)\"/gi) || [];\n  const urls = new Set<string>();\n\n  for (const match of hrefMatches) {\n    const href = match.match(/href=\"([^\"]*)\"/i)?.[1];\n    if (!href || !href.trim()) continue;\n\n    let fullUrl: string;\n\n    if (href.startsWith('//')) {\n      fullUrl = `https:${href}`;\n    } else if (href.startsWith('/')) {\n      // Skip fragment-only, query-only, and anchor links\n      if (href === '/' || href.startsWith('/#')) continue;\n      fullUrl = `https://${domain}${href}`;\n    } else if (href.startsWith('http')) {\n      fullUrl = href;\n    } else if (href.startsWith('#') || href.startsWith('?') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) {\n      continue;\n    } else {\n      // Relative path\n      fullUrl = `https://${domain}/${href}`;\n    }\n\n    try {\n      const parsed = new URL(fullUrl);\n      const linkDomain = parsed.hostname.replace(/^www\\./, '').toLowerCase();\n      if (linkDomain !== cleanDomain) continue;\n\n      // Strip hash and normalize\n      parsed.hash = '';\n      const path = parsed.pathname;\n\n      if (path === '/' || path === '') continue;\n      if (RESOURCE_EXTENSIONS.test(path)) continue;\n      if (SKIP_PATH_PATTERNS.test(path)) continue;\n\n      // Normalize: strip trailing slash\n      const normalized = parsed.origin + path.replace(/\\/+$/, '') + parsed.search;\n      urls.add(normalized);\n    } catch {\n      continue;\n    }\n  }\n\n  return Array.from(urls);\n}\n\n// ─── Category inference ─────────────────────────────────────────────────────\n\nconst CATEGORY_PATTERNS: Array<[RegExp, PageCategory]> = [\n  [/\\/([^/]*-?)?(blog|articles?|posts?|news|insights|guides)\\b/i, 'blog'],\n  [/\\/(about|about-us|company|who-we-are)\\b/i, 'about'],\n  [/\\/(pricing|plans|packages)\\b/i, 'pricing'],\n  [/\\/(services?|features?|solutions?|products?|what-we-do|offerings?)\\b/i, 'services'],\n  [/\\/(contact|contact-us|get-in-touch)\\b/i, 'contact'],\n  [/\\/(team|our-team|authors?|people|leadership|staff)\\b/i, 'team'],\n  [/\\/(resources?|resource-center|library|downloads?)\\b/i, 'resources'],\n  [/\\/(docs?|documentation|help|help-center|support|knowledge-base)\\b/i, 'docs'],\n  [/\\/(case-stud\\w*|cases|customers?|success-stor\\w*|testimonials?)\\b/i, 'cases'],\n  [/\\/(faq|frequently-asked|questions)\\b/i, 'faq'],\n];\n\n/**\n * Infer PageCategory from URL path patterns.\n */\nexport function inferCategory(url: string): PageCategory {\n  try {\n    const path = new URL(url).pathname;\n    for (const [pattern, category] of CATEGORY_PATTERNS) {\n      if (pattern.test(path)) return category;\n    }\n  } catch {\n    // Fall through to default\n  }\n  return 'content';\n}\n\n// ─── Main crawler ───────────────────────────────────────────────────────────\n\n/**\n * BFS crawl of a site, discovering all internal pages up to maxPages.\n * Seeds from sitemap URLs + homepage internal links.\n * Skips URLs already in siteData.blogSample and homepage.\n */\nexport async function crawlFullSite(\n  siteData: SiteData,\n  options?: CrawlOptions,\n): Promise<CrawlResult> {\n  const startTime = Date.now();\n  const maxPages = options?.maxPages ?? 200;\n  const timeoutMs = options?.timeoutMs ?? 10000;\n  const concurrency = options?.concurrency ?? 5;\n  const respectRobots = options?.respectRobots ?? true;\n\n  const pages: FetchResult[] = [];\n  const discoveredUrls = new Set<string>();\n  const fetchedUrls = new Set<string>();\n  const skippedUrls = new Set<string>();\n  const visited = new Set<string>();\n\n  // Parse robots.txt rules\n  let robotsRules: RobotsRules = { disallow: [], allow: [] };\n  if (respectRobots && siteData.robotsTxt?.text) {\n    robotsRules = parseRobotsTxt(siteData.robotsTxt.text);\n  }\n\n  const baseUrl = `${siteData.protocol}://${siteData.domain}`;\n\n  // Mark already-fetched URLs as visited\n  visited.add(normalizeUrl(baseUrl));\n  visited.add(normalizeUrl(baseUrl + '/'));\n  if (siteData.blogSample) {\n    for (const page of siteData.blogSample) {\n      if (page.finalUrl) visited.add(normalizeUrl(page.finalUrl));\n    }\n  }\n\n  // Seed the queue from sitemap\n  const queue: string[] = [];\n  if (siteData.sitemapXml?.text) {\n    const sitemapUrls = await extractAllUrlsFromSitemap(\n      siteData.sitemapXml.text,\n      siteData.domain,\n      timeoutMs,\n    );\n    for (const url of sitemapUrls) {\n      const norm = normalizeUrl(url);\n      if (!visited.has(norm)) {\n        discoveredUrls.add(url);\n        if (!queue.includes(url)) queue.push(url);\n      }\n    }\n  }\n\n  // Seed from homepage internal links\n  if (siteData.homepage?.text) {\n    const homeLinks = extractInternalLinks(siteData.homepage.text, siteData.domain);\n    for (const url of homeLinks) {\n      const norm = normalizeUrl(url);\n      if (!visited.has(norm) && !discoveredUrls.has(url)) {\n        discoveredUrls.add(url);\n        if (!queue.includes(url)) queue.push(url);\n      }\n    }\n  }\n\n  // BFS loop\n  while (queue.length > 0 && fetchedUrls.size < maxPages) {\n    // Take a batch\n    const batchSize = Math.min(concurrency, maxPages - fetchedUrls.size, queue.length);\n    const batch: string[] = [];\n\n    while (batch.length < batchSize && queue.length > 0) {\n      const url = queue.shift()!;\n      const norm = normalizeUrl(url);\n\n      if (visited.has(norm)) continue;\n      visited.add(norm);\n\n      // Check robots.txt\n      if (respectRobots) {\n        try {\n          const path = new URL(url).pathname;\n          if (isDisallowedByRobots(path, robotsRules)) {\n            skippedUrls.add(url);\n            continue;\n          }\n        } catch {\n          // Skip malformed URLs\n          continue;\n        }\n      }\n\n      batch.push(url);\n    }\n\n    if (batch.length === 0) continue;\n\n    // Fetch batch in parallel\n    const results = await Promise.all(batch.map(url => fetchPage(url, timeoutMs)));\n\n    for (let i = 0; i < results.length; i++) {\n      const result = results[i];\n      const url = batch[i];\n      fetchedUrls.add(url);\n\n      if (!result) continue;\n\n      result.category = inferCategory(url);\n      pages.push(result);\n\n      // Extract new internal links from fetched page\n      const newLinks = extractInternalLinks(result.text, siteData.domain);\n      for (const link of newLinks) {\n        const norm = normalizeUrl(link);\n        if (!visited.has(norm) && !discoveredUrls.has(link)) {\n          discoveredUrls.add(link);\n          queue.push(link);\n        }\n      }\n    }\n  }\n\n  // Any remaining queued URLs count as discovered but skipped\n  for (const url of queue) {\n    if (!fetchedUrls.has(url)) {\n      skippedUrls.add(url);\n    }\n  }\n\n  return {\n    pages,\n    discoveredUrls: Array.from(discoveredUrls),\n    fetchedUrls: Array.from(fetchedUrls),\n    skippedUrls: Array.from(skippedUrls),\n    elapsed: Math.round((Date.now() - startTime) / 100) / 10,\n  };\n}\n\nfunction normalizeUrl(url: string): string {\n  try {\n    const parsed = new URL(url);\n    // Normalize: lowercase host, strip trailing slash, strip hash\n    return (parsed.origin + parsed.pathname.replace(/\\/+$/, '') + parsed.search).toLowerCase();\n  } catch {\n    return url.toLowerCase();\n  }\n}\n"],"mappings":";;;AAgCA,IAAM,sBAAsB;AAE5B,IAAM,qBAAqB;AASpB,SAAS,eAAe,YAAiC;AAC9D,QAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,QAAM,QAAqB,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,EAAE;AAGrD,MAAI,oBAAoB;AAExB,aAAW,WAAW,OAAO;AAC3B,UAAM,OAAO,QAAQ,KAAK;AAC1B,QAAI,CAAC,QAAQ,KAAK,WAAW,GAAG,EAAG;AAEnC,UAAM,UAAU,KAAK,MAAM,sBAAsB;AACjD,QAAI,SAAS;AACX,YAAM,QAAQ,QAAQ,CAAC,EAAE,KAAK,EAAE,YAAY;AAC5C,0BAAoB,UAAU,OAAO,UAAU;AAC/C;AAAA,IACF;AAEA,QAAI,CAAC,kBAAmB;AAExB,UAAM,gBAAgB,KAAK,MAAM,oBAAoB;AACrD,QAAI,eAAe;AACjB,YAAM,OAAO,cAAc,CAAC,EAAE,KAAK;AACnC,UAAI,KAAM,OAAM,SAAS,KAAK,IAAI;AAClC;AAAA,IACF;AAEA,UAAM,aAAa,KAAK,MAAM,iBAAiB;AAC/C,QAAI,YAAY;AACd,YAAM,OAAO,WAAW,CAAC,EAAE,KAAK;AAChC,UAAI,KAAM,OAAM,MAAM,KAAK,IAAI;AAAA,IACjC;AAAA,EACF;AAEA,SAAO;AACT;AAEO,SAAS,qBAAqB,SAAiB,OAA6B;AAEjF,MAAI,eAAe;AACnB,MAAI,kBAAkB;AAEtB,aAAW,WAAW,MAAM,OAAO;AACjC,QAAI,QAAQ,WAAW,OAAO,KAAK,QAAQ,SAAS,cAAc;AAChE,qBAAe,QAAQ;AAAA,IACzB;AAAA,EACF;AAEA,aAAW,WAAW,MAAM,UAAU;AACpC,QAAI,QAAQ,WAAW,OAAO,KAAK,QAAQ,SAAS,iBAAiB;AACnE,wBAAkB,QAAQ;AAAA,IAC5B;AAAA,EACF;AAGA,MAAI,iBAAiB,KAAK,oBAAoB,EAAG,QAAO;AACxD,SAAO,kBAAkB;AAC3B;AAIA,eAAe,UAAU,KAAa,YAAY,KAAoC;AACpF,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,SAAS;AAAA,MACrC,SAAS,EAAE,cAAc,yBAAyB;AAAA,MAClD,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,WAAW,IAAK,QAAO;AAC/B,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,QAAI,KAAK,SAAS,IAAK,QAAO;AAC9B,WAAO,EAAE,MAAM,KAAK,MAAM,GAAG,GAAO,GAAG,QAAQ,IAAI,QAAQ,UAAU,IAAI,IAAI;AAAA,EAC/E,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAe,gBAAgB,KAAa,YAAY,KAA+B;AACrF,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ,YAAY,QAAQ,SAAS;AAAA,MACrC,SAAS,EAAE,cAAc,yBAAyB;AAAA,MAClD,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,WAAW,IAAK,QAAO;AAC/B,WAAO,MAAM,IAAI,KAAK;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAQA,eAAsB,0BACpB,aACA,QACA,YAAY,KACO;AACnB,QAAM,cAAc,OAAO,QAAQ,UAAU,EAAE,EAAE,YAAY;AAC7D,QAAM,OAAO,oBAAI,IAAY;AAG7B,QAAM,iBAAiB,YAAY,MAAM,2DAA2D,KAAK,CAAC;AAC1G,MAAI,eAAe,SAAS,GAAG;AAC7B,UAAM,UAAoB,CAAC;AAC3B,eAAW,SAAS,gBAAgB;AAClC,YAAM,WAAW,MAAM,MAAM,sBAAsB;AACnD,UAAI,SAAU,SAAQ,KAAK,SAAS,CAAC,EAAE,KAAK,CAAC;AAAA,IAC/C;AAGA,UAAM,UAAU,QAAQ,MAAM,GAAG,EAAE,EAAE,IAAI,OAAK,gBAAgB,GAAG,SAAS,CAAC;AAC3E,UAAM,UAAU,MAAM,QAAQ,IAAI,OAAO;AACzC,eAAW,QAAQ,SAAS;AAC1B,UAAI,MAAM;AACR,2BAAmB,MAAM,aAAa,IAAI;AAAA,MAC5C;AAAA,IACF;AAAA,EACF;AAGA,qBAAmB,aAAa,aAAa,IAAI;AAEjD,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,mBAAmB,KAAa,aAAqB,MAAyB;AACrF,QAAM,aAAa,IAAI,MAAM,mDAAmD,KAAK,CAAC;AACtF,aAAW,SAAS,YAAY;AAC9B,UAAM,WAAW,MAAM,MAAM,sBAAsB;AACnD,QAAI,CAAC,SAAU;AACf,UAAM,MAAM,SAAS,CAAC,EAAE,KAAK;AAE7B,QAAI;AACF,YAAM,SAAS,IAAI,IAAI,GAAG;AAC1B,YAAM,YAAY,OAAO,SAAS,QAAQ,UAAU,EAAE,EAAE,YAAY;AACpE,UAAI,cAAc,YAAa;AAC/B,UAAI,oBAAoB,KAAK,OAAO,QAAQ,EAAG;AAC/C,WAAK,IAAI,GAAG;AAAA,IACd,QAAQ;AACN;AAAA,IACF;AAAA,EACF;AACF;AAQO,SAAS,qBAAqB,MAAc,QAA0B;AAC3E,QAAM,cAAc,OAAO,QAAQ,UAAU,EAAE,EAAE,YAAY;AAC7D,QAAM,cAAc,KAAK,MAAM,kBAAkB,KAAK,CAAC;AACvD,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,SAAS,aAAa;AAC/B,UAAM,OAAO,MAAM,MAAM,iBAAiB,IAAI,CAAC;AAC/C,QAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAG;AAE3B,QAAI;AAEJ,QAAI,KAAK,WAAW,IAAI,GAAG;AACzB,gBAAU,SAAS,IAAI;AAAA,IACzB,WAAW,KAAK,WAAW,GAAG,GAAG;AAE/B,UAAI,SAAS,OAAO,KAAK,WAAW,IAAI,EAAG;AAC3C,gBAAU,WAAW,MAAM,GAAG,IAAI;AAAA,IACpC,WAAW,KAAK,WAAW,MAAM,GAAG;AAClC,gBAAU;AAAA,IACZ,WAAW,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,GAAG,KAAK,KAAK,WAAW,SAAS,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,WAAW,aAAa,GAAG;AAClJ;AAAA,IACF,OAAO;AAEL,gBAAU,WAAW,MAAM,IAAI,IAAI;AAAA,IACrC;AAEA,QAAI;AACF,YAAM,SAAS,IAAI,IAAI,OAAO;AAC9B,YAAM,aAAa,OAAO,SAAS,QAAQ,UAAU,EAAE,EAAE,YAAY;AACrE,UAAI,eAAe,YAAa;AAGhC,aAAO,OAAO;AACd,YAAM,OAAO,OAAO;AAEpB,UAAI,SAAS,OAAO,SAAS,GAAI;AACjC,UAAI,oBAAoB,KAAK,IAAI,EAAG;AACpC,UAAI,mBAAmB,KAAK,IAAI,EAAG;AAGnC,YAAM,aAAa,OAAO,SAAS,KAAK,QAAQ,QAAQ,EAAE,IAAI,OAAO;AACrE,WAAK,IAAI,UAAU;AAAA,IACrB,QAAQ;AACN;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,IAAM,oBAAmD;AAAA,EACvD,CAAC,+DAA+D,MAAM;AAAA,EACtE,CAAC,4CAA4C,OAAO;AAAA,EACpD,CAAC,iCAAiC,SAAS;AAAA,EAC3C,CAAC,yEAAyE,UAAU;AAAA,EACpF,CAAC,0CAA0C,SAAS;AAAA,EACpD,CAAC,yDAAyD,MAAM;AAAA,EAChE,CAAC,wDAAwD,WAAW;AAAA,EACpE,CAAC,sEAAsE,MAAM;AAAA,EAC7E,CAAC,sEAAsE,OAAO;AAAA,EAC9E,CAAC,yCAAyC,KAAK;AACjD;AAKO,SAAS,cAAc,KAA2B;AACvD,MAAI;AACF,UAAM,OAAO,IAAI,IAAI,GAAG,EAAE;AAC1B,eAAW,CAAC,SAAS,QAAQ,KAAK,mBAAmB;AACnD,UAAI,QAAQ,KAAK,IAAI,EAAG,QAAO;AAAA,IACjC;AAAA,EACF,QAAQ;AAAA,EAER;AACA,SAAO;AACT;AASA,eAAsB,cACpB,UACA,SACsB;AACtB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,YAAY,SAAS,aAAa;AACxC,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,gBAAgB,SAAS,iBAAiB;AAEhD,QAAM,QAAuB,CAAC;AAC9B,QAAM,iBAAiB,oBAAI,IAAY;AACvC,QAAM,cAAc,oBAAI,IAAY;AACpC,QAAM,cAAc,oBAAI,IAAY;AACpC,QAAM,UAAU,oBAAI,IAAY;AAGhC,MAAI,cAA2B,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,EAAE;AACzD,MAAI,iBAAiB,SAAS,WAAW,MAAM;AAC7C,kBAAc,eAAe,SAAS,UAAU,IAAI;AAAA,EACtD;AAEA,QAAM,UAAU,GAAG,SAAS,QAAQ,MAAM,SAAS,MAAM;AAGzD,UAAQ,IAAI,aAAa,OAAO,CAAC;AACjC,UAAQ,IAAI,aAAa,UAAU,GAAG,CAAC;AACvC,MAAI,SAAS,YAAY;AACvB,eAAW,QAAQ,SAAS,YAAY;AACtC,UAAI,KAAK,SAAU,SAAQ,IAAI,aAAa,KAAK,QAAQ,CAAC;AAAA,IAC5D;AAAA,EACF;AAGA,QAAM,QAAkB,CAAC;AACzB,MAAI,SAAS,YAAY,MAAM;AAC7B,UAAM,cAAc,MAAM;AAAA,MACxB,SAAS,WAAW;AAAA,MACpB,SAAS;AAAA,MACT;AAAA,IACF;AACA,eAAW,OAAO,aAAa;AAC7B,YAAM,OAAO,aAAa,GAAG;AAC7B,UAAI,CAAC,QAAQ,IAAI,IAAI,GAAG;AACtB,uBAAe,IAAI,GAAG;AACtB,YAAI,CAAC,MAAM,SAAS,GAAG,EAAG,OAAM,KAAK,GAAG;AAAA,MAC1C;AAAA,IACF;AAAA,EACF;AAGA,MAAI,SAAS,UAAU,MAAM;AAC3B,UAAM,YAAY,qBAAqB,SAAS,SAAS,MAAM,SAAS,MAAM;AAC9E,eAAW,OAAO,WAAW;AAC3B,YAAM,OAAO,aAAa,GAAG;AAC7B,UAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,CAAC,eAAe,IAAI,GAAG,GAAG;AAClD,uBAAe,IAAI,GAAG;AACtB,YAAI,CAAC,MAAM,SAAS,GAAG,EAAG,OAAM,KAAK,GAAG;AAAA,MAC1C;AAAA,IACF;AAAA,EACF;AAGA,SAAO,MAAM,SAAS,KAAK,YAAY,OAAO,UAAU;AAEtD,UAAM,YAAY,KAAK,IAAI,aAAa,WAAW,YAAY,MAAM,MAAM,MAAM;AACjF,UAAM,QAAkB,CAAC;AAEzB,WAAO,MAAM,SAAS,aAAa,MAAM,SAAS,GAAG;AACnD,YAAM,MAAM,MAAM,MAAM;AACxB,YAAM,OAAO,aAAa,GAAG;AAE7B,UAAI,QAAQ,IAAI,IAAI,EAAG;AACvB,cAAQ,IAAI,IAAI;AAGhB,UAAI,eAAe;AACjB,YAAI;AACF,gBAAM,OAAO,IAAI,IAAI,GAAG,EAAE;AAC1B,cAAI,qBAAqB,MAAM,WAAW,GAAG;AAC3C,wBAAY,IAAI,GAAG;AACnB;AAAA,UACF;AAAA,QACF,QAAQ;AAEN;AAAA,QACF;AAAA,MACF;AAEA,YAAM,KAAK,GAAG;AAAA,IAChB;AAEA,QAAI,MAAM,WAAW,EAAG;AAGxB,UAAM,UAAU,MAAM,QAAQ,IAAI,MAAM,IAAI,SAAO,UAAU,KAAK,SAAS,CAAC,CAAC;AAE7E,aAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,YAAM,SAAS,QAAQ,CAAC;AACxB,YAAM,MAAM,MAAM,CAAC;AACnB,kBAAY,IAAI,GAAG;AAEnB,UAAI,CAAC,OAAQ;AAEb,aAAO,WAAW,cAAc,GAAG;AACnC,YAAM,KAAK,MAAM;AAGjB,YAAM,WAAW,qBAAqB,OAAO,MAAM,SAAS,MAAM;AAClE,iBAAW,QAAQ,UAAU;AAC3B,cAAM,OAAO,aAAa,IAAI;AAC9B,YAAI,CAAC,QAAQ,IAAI,IAAI,KAAK,CAAC,eAAe,IAAI,IAAI,GAAG;AACnD,yBAAe,IAAI,IAAI;AACvB,gBAAM,KAAK,IAAI;AAAA,QACjB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,aAAW,OAAO,OAAO;AACvB,QAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,kBAAY,IAAI,GAAG;AAAA,IACrB;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,gBAAgB,MAAM,KAAK,cAAc;AAAA,IACzC,aAAa,MAAM,KAAK,WAAW;AAAA,IACnC,aAAa,MAAM,KAAK,WAAW;AAAA,IACnC,SAAS,KAAK,OAAO,KAAK,IAAI,IAAI,aAAa,GAAG,IAAI;AAAA,EACxD;AACF;AAEA,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,UAAM,SAAS,IAAI,IAAI,GAAG;AAE1B,YAAQ,OAAO,SAAS,OAAO,SAAS,QAAQ,QAAQ,EAAE,IAAI,OAAO,QAAQ,YAAY;AAAA,EAC3F,QAAQ;AACN,WAAO,IAAI,YAAY;AAAA,EACzB;AACF;","names":[]}

package/dist/{full-site-crawler-VFARFR2C.js → full-site-crawler-UIOMKOZA.js}

@@ -5,7 +5,7 @@ import {
   inferCategory,
   isDisallowedByRobots,
   parseRobotsTxt
-} from "./chunk-3IJISYWT.js";
+} from "./chunk-PKJIKMLV.js";
 export {
   crawlFullSite,
   extractAllUrlsFromSitemap,
@@ -14,4 +14,4 @@ export {
   isDisallowedByRobots,
   parseRobotsTxt
 };
-//# sourceMappingURL=full-site-crawler-VFARFR2C.js.map
+//# sourceMappingURL=full-site-crawler-UIOMKOZA.js.map

package/dist/index.cjs

@@ -303,7 +303,7 @@ var init_full_site_crawler = __esm({
     RESOURCE_EXTENSIONS = /\.(js|css|png|jpg|jpeg|gif|svg|ico|pdf|xml|txt|woff|woff2|ttf|eot|mp4|mp3|webp|avif|zip|gz|tar|json)$/i;
     SKIP_PATH_PATTERNS = /^\/(api|wp-admin|wp-json|static|assets|_next|auth|login|signup|cart|checkout|admin|feed|xmlrpc)\b/i;
     CATEGORY_PATTERNS = [
-      [/\/(blog|articles?|posts?|news|insights|guides)\b/i, "blog"],
+      [/\/([^/]*-?)?(blog|articles?|posts?|news|insights|guides)\b/i, "blog"],
       [/\/(about|about-us|company|who-we-are)\b/i, "about"],
       [/\/(pricing|plans|packages)\b/i, "pricing"],
       [/\/(services?|features?|solutions?|products?|what-we-do|offerings?)\b/i, "services"],
@@ -552,7 +552,7 @@ async function prefetchSiteData(domain) {
         sitemapForBlog = subSitemap.text;
       }
     }
-    const blogUrls = extractBlogUrlsFromSitemap(sitemapForBlog, domain, 10);
+    const blogUrls = extractBlogUrlsFromSitemap(sitemapForBlog, domain, 50);
     if (blogUrls.length > 0) {
       const fetched = await Promise.all(blogUrls.map((url) => fetchText(url)));
       blogSample = fetched.filter(
@@ -909,15 +909,17 @@ function checkOriginalData(data) {
     findings.push({ severity: "critical", detail: "Could not fetch homepage" });
     return { criterion: "original_data", criterion_label: "Original Data & Expert Content", score: 0, status: "not_found", findings, fix_priority: "P2" };
   }
+  const allPages = [data.homepage, ...data.blogSample || []].filter(Boolean);
   const html = data.homepage.text;
-  const text = html.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ");
+  const allText = allPages.map((p) => p.text.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ")).join(" ");
+  const text = data.homepage.text.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ");
   let score = 0;
   const statPatterns = /\d+%|\d+\s*(patients|clients|customers|cases|years|professionals|specialists|companies|users|businesses|domains|audits)/i;
-  if (statPatterns.test(text)) {
+  if (statPatterns.test(allText)) {
     const researchContext = /\b(our\s+(?:study|analysis|research|data|survey|findings|report)|we\s+(?:surveyed|analyzed|studied|measured|tracked)|proprietary|methodology|original\s+research)\b/i;
-    if (researchContext.test(text)) {
+    if (researchContext.test(allText)) {
       score += 3;
-      findings.push({ severity: "info", detail: "Proprietary statistics with research context found on homepage" });
+      findings.push({ severity: "info", detail: "Proprietary statistics with research context found" });
     } else {
       score += 1;
       findings.push({ severity: "low", detail: 'Statistics found but without research context (e.g., "500+ clients")', fix: 'Add context about your methodology: "Our analysis of X found..." or "We surveyed Y..."' });
@@ -1432,20 +1434,24 @@ function checkFactDensity(data) {
     findings.push({ severity: "critical", detail: "Could not fetch homepage" });
     return { criterion: "fact_density", criterion_label: "Fact & Data Density", score: 0, status: "not_found", findings, fix_priority: "P2" };
   }
-  const text = data.homepage.text.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ");
+  const allPages = [data.homepage, ...data.blogSample || []].filter(Boolean);
+  const allText = allPages.map((p) => p.text.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ")).join(" ");
+  const text = allText;
+  const pageCount = allPages.length;
   let score = 0;
   const dataPoints = text.match(/\d+(?:\.\d+)?(?:\s*%|\s*\$|\s*USD|\s*EUR)/g) || [];
   const countPhrases = text.match(/\d+(?:,\d{3})*\+?\s+(?:users?|clients?|customers?|companies|businesses|patients?|members?|employees?|projects?|downloads?)/gi) || [];
   const totalDataPoints = dataPoints.length + countPhrases.length;
-  if (totalDataPoints >= 6) {
+  const avgPerPage = pageCount > 0 ? totalDataPoints / pageCount : 0;
+  if (avgPerPage >= 4) {
     score += 5;
-    findings.push({ severity: "info", detail: `${totalDataPoints} quantitative data points found on homepage` });
-  } else if (totalDataPoints >= 3) {
+    findings.push({ severity: "info", detail: `${totalDataPoints} quantitative data points found across ${pageCount} pages (avg ${avgPerPage.toFixed(1)}/page)` });
+  } else if (avgPerPage >= 2) {
     score += 3;
-    findings.push({ severity: "info", detail: `${totalDataPoints} quantitative data points found` });
+    findings.push({ severity: "info", detail: `${totalDataPoints} quantitative data points found across ${pageCount} pages` });
   } else if (totalDataPoints >= 1) {
     score += 1;
-    findings.push({ severity: "low", detail: `Only ${totalDataPoints} quantitative data point(s) found`, fix: "Add more specific numbers, percentages, and metrics to strengthen credibility" });
+    findings.push({ severity: "low", detail: `Only ${totalDataPoints} quantitative data point(s) found across ${pageCount} pages`, fix: "Add more specific numbers, percentages, and metrics to strengthen credibility" });
   } else {
     findings.push({ severity: "high", detail: "No quantitative data points found", fix: "Add specific statistics (percentages, counts, comparisons) that AI engines can cite" });
   }
@@ -1551,9 +1557,9 @@ function countRecentSitemapDates(sitemapText) {
     distinctRecentDays: recentDays.size
   };
 }
-var BLOG_PATH_PATTERNS = /\/(?:blog|articles?|insights?|guides?|resources?|news|posts?|learn|help|how-?to|tutorials?|case-stud|whitepapers?)\b/i;
+var BLOG_PATH_PATTERNS = /\/(?:[^/]*-?)?(?:blog|articles?|insights?|guides?|resources?|news|posts?|learn|help|how-?to|tutorials?|case-stud|whitepapers?)\b/i;
 var EXCLUDE_PATH_PATTERNS = /\/(?:tag|category|author|page|feed|wp-content|wp-admin|wp-json|cart|checkout|login|search|api|static|assets|_next)\b/i;
-function extractBlogUrlsFromSitemap(sitemapText, domain, limit = 5) {
+function extractBlogUrlsFromSitemap(sitemapText, domain, limit = 50) {
   const urlBlocks = sitemapText.match(/<url>([\s\S]*?)<\/url>/gi) || [];
   const candidates = [];
   const cleanDomain = domain.replace(/^www\./, "").toLowerCase();
@@ -1849,7 +1855,7 @@ function jaccardSimilarity(a, b) {
   const union = a.size + b.size - intersection;
   return union === 0 ? 0 : intersection / union;
 }
-function checkContentCannibalization(data) {
+function checkContentCannibalization(data, topicCoherenceScore) {
   const findings = [];
   if (!data.homepage) {
     findings.push({ severity: "critical", detail: "No homepage available for cannibalization analysis" });
@@ -1859,7 +1865,7 @@ function checkContentCannibalization(data) {
     { html: data.homepage.text, url: data.homepage.finalUrl || `https://${data.domain}/` }
   ];
   if (data.blogSample) {
-    for (const page of data.blogSample.slice(0, 5)) {
+    for (const page of data.blogSample) {
       pages.push({ html: page.text, url: page.finalUrl || "" });
     }
   }
@@ -1869,10 +1875,29 @@ function checkContentCannibalization(data) {
   }
   const pageTitles = pages.map((p) => ({ title: extractPageTitle(p.html), url: p.url }));
   const wordSets = pageTitles.map((p) => titleToWordSet(p.title));
+  const termPageCount = /* @__PURE__ */ new Map();
+  for (const ws of wordSets) {
+    for (const w of ws) {
+      termPageCount.set(w, (termPageCount.get(w) || 0) + 1);
+    }
+  }
+  const commonTermThreshold = Math.max(3, pages.length * 0.4);
+  const siteThemeTerms = /* @__PURE__ */ new Set();
+  for (const [term, count] of termPageCount) {
+    if (count >= commonTermThreshold) siteThemeTerms.add(term);
+  }
+  const filteredSets = wordSets.map((ws) => {
+    const filtered = /* @__PURE__ */ new Set();
+    for (const w of ws) {
+      if (!siteThemeTerms.has(w)) filtered.add(w);
+    }
+    return filtered;
+  });
   const cannibalPairs = [];
   for (let i = 0; i < pages.length; i++) {
     for (let j = i + 1; j < pages.length; j++) {
-      const sim = jaccardSimilarity(wordSets[i], wordSets[j]);
+      if (filteredSets[i].size === 0 && filteredSets[j].size === 0) continue;
+      const sim = jaccardSimilarity(filteredSets[i], filteredSets[j]);
       if (sim > 0.6) {
         cannibalPairs.push({
           urlA: pageTitles[i].url.slice(0, 60),
@@ -1882,23 +1907,39 @@ function checkContentCannibalization(data) {
       }
     }
   }
+  const cannibalUrls = /* @__PURE__ */ new Set();
+  for (const pair of cannibalPairs) {
+    cannibalUrls.add(pair.urlA);
+    cannibalUrls.add(pair.urlB);
+  }
+  const cannibalRatio = pages.length > 0 ? cannibalUrls.size / pages.length : 0;
   let score;
   if (cannibalPairs.length === 0) {
     score = 10;
     findings.push({ severity: "info", detail: `${pages.length} pages analyzed - no content cannibalization detected` });
-  } else if (cannibalPairs.length === 1) {
-    score = 8;
-    findings.push({ severity: "low", detail: `1 pair of pages with overlapping topics (${cannibalPairs[0].similarity}% similarity)`, fix: "Differentiate titles and H1 headings to reduce topic overlap" });
-  } else if (cannibalPairs.length === 2) {
+  } else if (cannibalRatio <= 0.05) {
+    score = 9;
+    findings.push({ severity: "info", detail: `${cannibalPairs.length} pair(s) of pages with minor topic overlap (${cannibalUrls.size}/${pages.length} pages affected)` });
+  } else if (cannibalRatio <= 0.1) {
+    score = 7;
+    findings.push({ severity: "low", detail: `${cannibalUrls.size} pages (${Math.round(cannibalRatio * 100)}%) have overlapping topics`, fix: "Differentiate titles and H1 headings to reduce topic overlap" });
+  } else if (cannibalRatio <= 0.2) {
     score = 5;
-    findings.push({ severity: "medium", detail: `${cannibalPairs.length} pairs of pages with overlapping topics`, fix: "Consolidate overlapping pages or differentiate their titles and content focus" });
+    findings.push({ severity: "medium", detail: `${cannibalUrls.size} pages (${Math.round(cannibalRatio * 100)}%) competing for overlapping topics`, fix: "Consolidate overlapping pages or differentiate their titles and content focus" });
+  } else if (cannibalRatio <= 0.4) {
+    score = 3;
+    findings.push({ severity: "medium", detail: `${cannibalUrls.size} pages (${Math.round(cannibalRatio * 100)}%) have significant content overlap`, fix: "Many pages compete for the same topics - consolidate or clearly differentiate them" });
   } else {
     score = 0;
-    findings.push({ severity: "high", detail: `${cannibalPairs.length} pairs of pages competing for the same topics`, fix: "Significant content overlap detected - consolidate or clearly differentiate competing pages" });
+    findings.push({ severity: "high", detail: `${cannibalUrls.size} pages (${Math.round(cannibalRatio * 100)}%) competing for the same topics`, fix: "Severe content cannibalization - consolidate overlapping pages or create clear topic differentiation" });
   }
   for (const pair of cannibalPairs.slice(0, 3)) {
     findings.push({ severity: "low", detail: `Overlap (${pair.similarity}%): ${pair.urlA} vs ${pair.urlB}` });
   }
+  if (topicCoherenceScore !== void 0 && topicCoherenceScore <= 4 && score >= 8) {
+    score = 6;
+    findings.push({ severity: "low", detail: "Low topic overlap but content lacks coherent focus - not a strong signal for AI authority", fix: "Focus content on fewer core topics to build topical authority that AI engines can identify" });
+  }
   return { criterion: "content_cannibalization", criterion_label: "Content Cannibalization", score, status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P1" };
 }
 function checkVisibleDateSignal(data) {
@@ -2124,7 +2165,233 @@ function extractRawDataSummary(data) {
     crawl_skipped: data.crawlStats?.skipped ?? 0
   };
 }
+function getPageTopicText(html) {
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+  const h1Match = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
+  return [
+    titleMatch?.[1] || "",
+    h1Match?.[1]?.replace(/<[^>]*>/g, "") || ""
+  ].join(" ").toLowerCase().trim();
+}
+function extractBigrams(text) {
+  const words = text.split(/[\s,.!?;:()\[\]{}"'\/&]+/).filter((w) => w.length > 2 && !STOP_WORDS.has(w) && !/^\d+$/.test(w));
+  const bigrams = [];
+  for (let i = 0; i < words.length - 1; i++) {
+    bigrams.push(words[i] + " " + words[i + 1]);
+  }
+  return bigrams;
+}
+function checkTopicCoherence(data) {
+  const findings = [];
+  if (!data.homepage) {
+    findings.push({ severity: "critical", detail: "Could not fetch homepage" });
+    return { criterion: "topic_coherence", criterion_label: "Topic Coherence", score: 0, status: "not_found", findings, fix_priority: "P0" };
+  }
+  if (!data.blogSample || data.blogSample.length < 3) {
+    findings.push({ severity: "info", detail: `Only ${data.blogSample?.length || 0} blog pages found - insufficient for topic coherence analysis` });
+    return { criterion: "topic_coherence", criterion_label: "Topic Coherence", score: 5, status: "partial", findings, fix_priority: "P2" };
+  }
+  const blogPages = data.blogSample;
+  const domainBase = data.domain.replace(/^www\./, "").replace(/\.(com|org|net|io|co|ai)$/i, "").toLowerCase();
+  const brandWords = /* @__PURE__ */ new Set();
+  brandWords.add(domainBase);
+  for (const part of domainBase.split(/[-_]/)) {
+    if (part.length > 2) brandWords.add(part);
+  }
+  const rawTermFreq = /* @__PURE__ */ new Map();
+  const pageTitleTexts = [];
+  for (const page of blogPages) {
+    const topicText = getPageTopicText(page.text);
+    pageTitleTexts.push(topicText);
+    const words = topicText.split(/[\s,.!?;:()\[\]{}"'\/&]+/).filter((w) => w.length > 2 && !STOP_WORDS.has(w) && !/^\d+$/.test(w));
+    const uniqueWords = new Set(words);
+    for (const w of uniqueWords) {
+      rawTermFreq.set(w, (rawTermFreq.get(w) || 0) + 1);
+    }
+  }
+  for (const [term, count] of rawTermFreq) {
+    if (count / blogPages.length >= 0.8 && domainBase.includes(term)) {
+      brandWords.add(term);
+    }
+  }
+  const termFreq = /* @__PURE__ */ new Map();
+  for (const page of blogPages) {
+    const topicText = getPageTopicText(page.text);
+    const words = topicText.split(/[\s,.!?;:()\[\]{}"'\/&]+/).filter((w) => w.length > 2 && !STOP_WORDS.has(w) && !/^\d+$/.test(w) && !brandWords.has(w));
+    const uniqueWords = new Set(words);
+    for (const w of uniqueWords) {
+      termFreq.set(w, (termFreq.get(w) || 0) + 1);
+    }
+  }
+  const sortedTerms = [...termFreq.entries()].sort((a, b) => b[1] - a[1]);
+  const topTerm = sortedTerms[0];
+  const bigramFreq = /* @__PURE__ */ new Map();
+  const pageBigrams = [];
+  for (const topicText of pageTitleTexts) {
+    const bigrams = extractBigrams(topicText).filter((bg) => !bg.split(" ").some((w) => brandWords.has(w)));
+    pageBigrams.push(bigrams);
+    const uniqueBigrams = new Set(bigrams);
+    for (const bg of uniqueBigrams) {
+      bigramFreq.set(bg, (bigramFreq.get(bg) || 0) + 1);
+    }
+  }
+  const sortedBigrams = [...bigramFreq.entries()].sort((a, b) => b[1] - a[1]);
+  const topBigram = sortedBigrams[0];
+  const significantBigrams = sortedBigrams.filter(([, count]) => count >= 2);
+  const clusterRoots = [];
+  const assigned = /* @__PURE__ */ new Set();
+  for (const [bg] of significantBigrams) {
+    if (assigned.has(bg)) continue;
+    clusterRoots.push(bg);
+    assigned.add(bg);
+    const [w1, w2] = bg.split(" ");
+    for (const [otherBg] of significantBigrams) {
+      if (assigned.has(otherBg)) continue;
+      if (otherBg.includes(w1) || otherBg.includes(w2)) {
+        assigned.add(otherBg);
+      }
+    }
+  }
+  const topicClusterCount = clusterRoots.length;
+  const dominantTerm = topTerm?.[0] || "";
+  const dominantTermCount = topTerm?.[1] || 0;
+  const focusRatio = blogPages.length > 0 ? dominantTermCount / blogPages.length : 0;
+  const dominantBigram = topBigram?.[0] || "";
+  const dominantBigramCount = topBigram?.[1] || 0;
+  const bigramFocusRatio = blogPages.length > 0 ? dominantBigramCount / blogPages.length : 0;
+  let score = 0;
+  const bestFocusRatio = Math.max(focusRatio, bigramFocusRatio);
+  if (bestFocusRatio >= 0.8) {
+    score += 7;
+  } else if (bestFocusRatio >= 0.6) {
+    score += 6;
+  } else if (bestFocusRatio >= 0.45) {
+    score += 5;
+  } else if (bestFocusRatio >= 0.3) {
+    score += 3;
+  } else if (bestFocusRatio >= 0.15) {
+    score += 2;
+  } else {
+    score += 1;
+  }
+  const clusterPenaltyReduced = focusRatio >= 0.7;
+  if (topicClusterCount <= 3) {
+    score += 3;
+    findings.push({ severity: "info", detail: `${topicClusterCount} topic cluster(s) - tightly focused content` });
+  } else if (topicClusterCount <= 6) {
+    score += clusterPenaltyReduced ? 2 : 1;
+    findings.push({ severity: "info", detail: `${topicClusterCount} topic clusters${clusterPenaltyReduced ? " within a focused niche" : " - moderately focused"}` });
+  } else if (topicClusterCount <= 10) {
+    score += clusterPenaltyReduced ? 1 : 0;
+    if (!clusterPenaltyReduced) {
+      findings.push({ severity: "low", detail: `${topicClusterCount} topic clusters - scattered content`, fix: "Reduce the number of distinct topics. Focus blog content on 2-3 core expertise areas." });
+    } else {
+      findings.push({ severity: "info", detail: `${topicClusterCount} topic clusters but strong core topic focus (${Math.round(focusRatio * 100)}%)` });
+    }
+  } else {
+    score += clusterPenaltyReduced ? 0 : -2;
+    if (!clusterPenaltyReduced) {
+      findings.push({ severity: "medium", detail: `${topicClusterCount} topic clusters - highly scattered content`, fix: "Content covers too many unrelated topics. AI engines cannot identify your expertise. Focus on your core niche." });
+    } else {
+      findings.push({ severity: "low", detail: `${topicClusterCount} topic clusters despite strong core topic focus`, fix: "Consider narrowing subtopics within your niche for even stronger AI visibility." });
+    }
+  }
+  score = Math.max(0, Math.min(10, score));
+  if (dominantTerm) {
+    const focusPct = Math.round(focusRatio * 100);
+    findings.push({ severity: "info", detail: `Dominant topic term: "${dominantTerm}" (${focusPct}% of ${blogPages.length} pages)` });
+  }
+  if (dominantBigram && dominantBigramCount >= 2) {
+    findings.push({ severity: "info", detail: `Dominant topic phrase: "${dominantBigram}" (${dominantBigramCount}/${blogPages.length} pages)` });
+  }
+  const offTopicExamples = [];
+  for (let i = 0; i < pageTitleTexts.length && offTopicExamples.length < 3; i++) {
+    if (dominantTerm && !pageTitleTexts[i].includes(dominantTerm)) {
+      const title = blogPages[i].text.match(/<title[^>]*>([^<]+)<\/title>/i)?.[1]?.trim();
+      if (title && title.length > 3) offTopicExamples.push(title.slice(0, 60));
+    }
+  }
+  if (offTopicExamples.length > 0 && score < 8) {
+    findings.push({ severity: "low", detail: `Off-topic examples: ${offTopicExamples.join("; ")}` });
+  }
+  return { criterion: "topic_coherence", criterion_label: "Topic Coherence", score, status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P0" };
+}
+function countWords(html) {
+  const text = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
+  return text.split(/\s+/).filter((w) => w.length > 0).length;
+}
+function countHeadings(html) {
+  const headings = html.match(/<h[2-6][^>]*>/gi) || [];
+  return headings.length;
+}
+function checkContentDepth(data, topicCoherenceScore) {
+  const findings = [];
+  if (!data.blogSample || data.blogSample.length < 2) {
+    findings.push({ severity: "info", detail: `Only ${data.blogSample?.length || 0} blog pages found - insufficient for depth analysis` });
+    return { criterion: "content_depth", criterion_label: "Content Depth", score: 3, status: "partial", findings, fix_priority: "P2" };
+  }
+  const blogPages = data.blogSample;
+  const wordCounts = blogPages.map((p) => countWords(p.text));
+  const headingCounts = blogPages.map((p) => countHeadings(p.text));
+  const avgWords = wordCounts.reduce((a, b) => a + b, 0) / wordCounts.length;
+  const avgHeadings = headingCounts.reduce((a, b) => a + b, 0) / headingCounts.length;
+  const deepPages = wordCounts.filter((w) => w >= 1e3).length;
+  const thinPages = wordCounts.filter((w) => w < 300).length;
+  const deepRatio = deepPages / blogPages.length;
+  const thinRatio = thinPages / blogPages.length;
+  let score = 0;
+  if (avgWords >= 2e3) {
+    score += 5;
+    findings.push({ severity: "info", detail: `Average ${Math.round(avgWords)} words per page across ${blogPages.length} pages - excellent depth` });
+  } else if (avgWords >= 1200) {
+    score += 4;
+    findings.push({ severity: "info", detail: `Average ${Math.round(avgWords)} words per page across ${blogPages.length} pages - good depth` });
+  } else if (avgWords >= 800) {
+    score += 3;
+    findings.push({ severity: "info", detail: `Average ${Math.round(avgWords)} words per page - moderate depth` });
+  } else if (avgWords >= 400) {
+    score += 2;
+    findings.push({ severity: "low", detail: `Average ${Math.round(avgWords)} words per page - shallow content`, fix: "Expand articles with more detail, examples, and expert analysis to build AI citation authority" });
+  } else {
+    score += 1;
+    findings.push({ severity: "medium", detail: `Average ${Math.round(avgWords)} words per page - very thin content`, fix: "Content is too thin for AI engines to cite. Aim for 1000+ words per article with structured sections." });
+  }
+  if (avgHeadings >= 8) {
+    score += 3;
+    findings.push({ severity: "info", detail: `Average ${avgHeadings.toFixed(1)} subheadings per page - well-structured` });
+  } else if (avgHeadings >= 5) {
+    score += 2;
+    findings.push({ severity: "info", detail: `Average ${avgHeadings.toFixed(1)} subheadings per page - decent structure` });
+  } else if (avgHeadings >= 2) {
+    score += 1;
+    findings.push({ severity: "low", detail: `Average ${avgHeadings.toFixed(1)} subheadings per page`, fix: "Add more H2/H3 headings to break content into extractable sections" });
+  } else {
+    findings.push({ severity: "medium", detail: `Average ${avgHeadings.toFixed(1)} subheadings per page - minimal structure`, fix: "Add question-format H2/H3 headings so AI engines can extract specific answers" });
+  }
+  if (deepRatio >= 0.5) {
+    score += 2;
+    findings.push({ severity: "info", detail: `${deepPages}/${blogPages.length} pages (${Math.round(deepRatio * 100)}%) have 1000+ words` });
+  } else if (deepRatio >= 0.25) {
+    score += 1;
+    findings.push({ severity: "info", detail: `${deepPages}/${blogPages.length} pages have 1000+ words` });
+  }
+  if (thinRatio >= 0.5) {
+    score = Math.max(0, score - 2);
+    findings.push({ severity: "medium", detail: `${thinPages}/${blogPages.length} pages (${Math.round(thinRatio * 100)}%) have under 300 words - high thin content ratio`, fix: "Remove or expand thin pages. Thin content dilutes site quality for AI engines." });
+  } else if (thinRatio >= 0.25) {
+    score = Math.max(0, score - 1);
+    findings.push({ severity: "low", detail: `${thinPages}/${blogPages.length} pages have under 300 words` });
+  }
+  let finalScore = Math.min(10, score);
+  if (topicCoherenceScore !== void 0 && topicCoherenceScore <= 4 && finalScore >= 8) {
+    finalScore = 7;
+    findings.push({ severity: "low", detail: "Deep content but low topic coherence - depth on scattered topics has reduced AI citation value", fix: "Focus content depth on your core expertise area for maximum AI visibility" });
+  }
+  return { criterion: "content_depth", criterion_label: "Content Depth", score: finalScore, status: finalScore >= 7 ? "pass" : finalScore >= 4 ? "partial" : "fail", findings, fix_priority: finalScore >= 7 ? "P3" : "P1" };
+}
 function auditSiteFromData(data) {
+  const topicCoherence = checkTopicCoherence(data);
+  const cannibalization = checkContentCannibalization(data, topicCoherence.score);
   return [
     checkLlmsTxt(data),
     checkSchemaMarkup(data),
@@ -2150,47 +2417,55 @@ function auditSiteFromData(data) {
     checkSchemaCoverage(data),
     checkSpeakableSchema(data),
     checkQueryAnswerAlignment(data),
-    checkContentCannibalization(data),
-    checkVisibleDateSignal(data)
+    cannibalization,
+    checkVisibleDateSignal(data),
+    topicCoherence,
+    checkContentDepth(data, topicCoherence.score)
   ];
 }
 
 // src/scoring.ts
 var WEIGHTS = {
-  // Original 10
-  llms_txt: 0.1,
-  schema_markup: 0.15,
-  qa_content_format: 0.15,
-  clean_html: 0.1,
-  entity_consistency: 0.1,
-  robots_txt: 0.05,
-  faq_section: 0.1,
-  original_data: 0.1,
-  internal_linking: 0.1,
-  semantic_html: 0.05,
-  // New 12
-  content_freshness: 0.07,
-  sitemap_completeness: 0.05,
-  rss_feed: 0.03,
-  table_list_extractability: 0.07,
-  definition_patterns: 0.04,
+  // ─── Core Content (high weight - these determine real AI citation quality) ──
+  qa_content_format: 0.12,
+  original_data: 0.12,
+  topic_coherence: 0.14,
+  // NEW v2.0: biggest predictor of AI citation quality
+  fact_density: 0.08,
   direct_answer_density: 0.07,
-  content_licensing: 0.04,
+  content_depth: 0.06,
+  // NEW v2.0: substantive content vs thin pages
+  // ─── Structure & Discovery (medium weight - technical readiness) ────────────
+  schema_markup: 0.08,
+  llms_txt: 0.08,
+  clean_html: 0.08,
+  entity_consistency: 0.08,
+  faq_section: 0.08,
+  internal_linking: 0.08,
+  // ─── Content Signals (moderate weight) ──────────────────────────────────────
+  content_freshness: 0.06,
+  table_list_extractability: 0.05,
+  query_answer_alignment: 0.06,
+  definition_patterns: 0.04,
   author_schema_depth: 0.04,
-  fact_density: 0.05,
-  canonical_url: 0.04,
-  content_velocity: 0.03,
-  schema_coverage: 0.03,
-  speakable_schema: 0.03,
-  query_answer_alignment: 0.08,
   content_cannibalization: 0.05,
-  visible_date_signal: 0.04
+  visible_date_signal: 0.04,
+  semantic_html: 0.04,
+  // ─── Plumbing (low weight - nice to have but not what drives citations) ─────
+  robots_txt: 0.03,
+  sitemap_completeness: 0.03,
+  content_velocity: 0.03,
+  rss_feed: 0.02,
+  content_licensing: 0.03,
+  canonical_url: 0.02,
+  schema_coverage: 0.02,
+  speakable_schema: 0.02
 };
 function calculateOverallScore(criteria) {
   let totalWeight = 0;
   let weightedSum = 0;
   for (const c of criteria) {
-    const weight = WEIGHTS[c.criterion] ?? 0.1;
+    const weight = WEIGHTS[c.criterion] ?? 0.05;
     weightedSum += c.score / 10 * weight * 100;
     totalWeight += weight;
   }
@@ -2326,7 +2601,9 @@ var CRITERION_LABELS = {
   "Speakable Schema": "Speakable Schema",
   "Query-Answer Alignment": "Query-Answer Alignment",
   "Content Cannibalization": "Content Cannibalization",
-  "Visible Date Signal": "Visible Date Signal"
+  "Visible Date Signal": "Visible Date Signal",
+  "Topic Coherence": "Topic Coherence",
+  "Content Depth": "Content Depth"
 };
 function scoreToStatus(score) {
   if (score === 0) return "MISSING";
@@ -3235,7 +3512,7 @@ function extractTitle(html) {
 function getTextContent2(html) {
   return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
 }
-function countWords(text) {
+function countWords2(text) {
   if (!text) return 0;
   return text.split(/\s+/).filter((w) => w.length > 0).length;
 }
@@ -3386,7 +3663,7 @@ function checkHasQuestionHeadings(html) {
 function analyzePage(html, url, category) {
   const title = extractTitle(html);
   const textContent = getTextContent2(html);
-  const wordCount = countWords(textContent);
+  const wordCount = countWords2(textContent);
   const issues = [];
   const strengths = [];
   const issueChecks = [
@@ -3539,7 +3816,7 @@ function extractTitle2(html) {
 function getTextContent3(html) {
   return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
 }
-function countWords2(text) {
+function countWords3(text) {
   if (!text) return 0;
   return text.split(/\s+/).filter((w) => w.length > 0).length;
 }
@@ -3687,7 +3964,7 @@ function buildLinkGraph(pages, domain, homepageUrl) {
     if (nodes.has(norm)) continue;
     const title = extractTitle2(page.text);
     const text = getTextContent3(page.text);
-    const wordCount = countWords2(text);
+    const wordCount = countWords3(text);
     nodes.set(norm, {
       url: norm,
       title,