aeorank 3.0.2 → 3.1.0

package/dist/index.js

@@ -3,7 +3,7 @@ import {
   extractAllUrlsFromSitemap,
   extractInternalLinks,
   inferCategory
-} from "./chunk-OCLAIHX6.js";
+} from "./chunk-RYV25AUV.js";
 
 // src/parked-domain.ts
 var PARKING_PATHS = ["/lander", "/parking", "/park", "/sedoparking"];
@@ -2558,6 +2558,234 @@ function checkImageContextAI(data) {
   }
   return { criterion: "image_context_ai", criterion_label: "Image Context for AI", score: Math.min(10, score), status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P2" };
 }
+var BOILERPLATE_RE = /\b(sign up|subscribe|get started|contact us|request a demo|free trial|book a call|schedule a|learn more|click here|follow us|share this|copyright|all rights reserved|privacy policy|terms of service)\b/i;
+function isBoilerplateParagraph(text) {
+  const words = text.split(/\s+/).length;
+  if (words < 20 && BOILERPLATE_RE.test(text)) return true;
+  if (/\b(cookie|gdpr|consent|opt.out)\b/i.test(text) && words < 30) return true;
+  return false;
+}
+function toShingles(text, n = 4) {
+  const words = text.split(/\s+/).filter((w) => w.length > 1);
+  const shingles = /* @__PURE__ */ new Set();
+  for (let i = 0; i <= words.length - n; i++) {
+    shingles.add(words.slice(i, i + n).join(" "));
+  }
+  return shingles;
+}
+function shingleSimilarity(a, b) {
+  if (a.size === 0 && b.size === 0) return 0;
+  let intersection = 0;
+  for (const s of a) {
+    if (b.has(s)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+function extractPageParagraphs(html) {
+  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
+  const pMatches = cleaned.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
+  return pMatches.map((p) => {
+    const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
+    return { text, shingles: toShingles(text) };
+  }).filter((p) => p.shingles.size >= 3 && !isBoilerplateParagraph(p.text));
+}
+function splitIntoSectionsWithParagraphs(html) {
+  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
+  const parts = cleaned.split(/(?=<h[23]\b[^>]*>)/i);
+  const sections = [];
+  for (const part of parts) {
+    const hMatch = part.match(/<h[23]\b[^>]*>([\s\S]*?)<\/h[23]>/i);
+    const heading = hMatch ? hMatch[1].replace(/<[^>]*>/g, "").trim() : "(intro)";
+    const pMatches = part.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
+    const paragraphs = pMatches.map((p) => {
+      const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
+      return { text, shingles: toShingles(text) };
+    }).filter((p) => p.shingles.size >= 3 && !isBoilerplateParagraph(p.text));
+    if (paragraphs.length > 0) sections.push({ heading, paragraphs });
+  }
+  return sections;
+}
+function findIntraPageDuplicates(html) {
+  const sections = splitIntoSectionsWithParagraphs(html);
+  if (sections.length < 2) return [];
+  const pairs = [];
+  for (let i = 0; i < sections.length; i++) {
+    for (let j = i + 1; j < sections.length; j++) {
+      let found = false;
+      for (const pA of sections[i].paragraphs) {
+        if (found) break;
+        for (const pB of sections[j].paragraphs) {
+          const sim = shingleSimilarity(pA.shingles, pB.shingles);
+          if (sim > 0.4) {
+            pairs.push({
+              headingA: sections[i].heading,
+              headingB: sections[j].heading,
+              similarity: Math.round(sim * 100),
+              sample: pA.text.slice(0, 80)
+            });
+            found = true;
+            break;
+          }
+        }
+      }
+    }
+  }
+  return pairs;
+}
+function checkDuplicateContent(data) {
+  const findings = [];
+  const pages = [];
+  if (data.homepage) {
+    pages.push({ html: data.homepage.text, url: data.homepage.finalUrl || `https://${data.domain}/` });
+  }
+  if (data.blogSample) {
+    for (const page of data.blogSample) {
+      pages.push({ html: page.text, url: page.finalUrl || "" });
+    }
+  }
+  if (pages.length === 0) {
+    findings.push({ severity: "critical", detail: "No pages available for duplicate content analysis" });
+    return { criterion: "duplicate_content", criterion_label: "Duplicate Content Blocks", score: 0, status: "fail", findings, fix_priority: "P1" };
+  }
+  let totalDupPages = 0;
+  let totalDupPairs = 0;
+  const dupDetails = [];
+  for (const page of pages) {
+    const pairs = findIntraPageDuplicates(page.html);
+    if (pairs.length > 0) {
+      totalDupPages++;
+      totalDupPairs += pairs.length;
+      dupDetails.push({ url: page.url, pairs });
+    }
+  }
+  const dupRatio = totalDupPages / pages.length;
+  let score;
+  if (totalDupPairs === 0) {
+    score = 10;
+    findings.push({ severity: "info", detail: `${pages.length} pages analyzed - no duplicate content blocks detected` });
+  } else if (dupRatio <= 0.05 && totalDupPairs <= 2) {
+    score = 9;
+    findings.push({ severity: "info", detail: `${totalDupPairs} duplicate block pair(s) on ${totalDupPages} page(s) - minor` });
+  } else if (dupRatio <= 0.1) {
+    score = 7;
+    findings.push({ severity: "low", detail: `${totalDupPairs} duplicate block pair(s) across ${totalDupPages} page(s)`, fix: "Rewrite duplicate sections to provide unique content in each" });
+  } else if (dupRatio <= 0.2) {
+    score = 5;
+    findings.push({ severity: "medium", detail: `${totalDupPages} pages (${Math.round(dupRatio * 100)}%) contain duplicate content blocks`, fix: "Rewrite or remove repeated text blocks - LLMs may flag this as low-quality content" });
+  } else if (dupRatio <= 0.4) {
+    score = 3;
+    findings.push({ severity: "medium", detail: `${totalDupPages} pages (${Math.round(dupRatio * 100)}%) have significant duplicate content`, fix: "Widespread duplicate blocks reduce content authority - rewrite each section with unique angles" });
+  } else {
+    score = 0;
+    findings.push({ severity: "high", detail: `${totalDupPages} pages (${Math.round(dupRatio * 100)}%) contain duplicate content blocks`, fix: "Severe content duplication across the site - LLMs will likely reduce citation authority" });
+  }
+  for (const dup of dupDetails.slice(0, 3)) {
+    const shortUrl = dup.url.slice(0, 60);
+    for (const pair of dup.pairs.slice(0, 2)) {
+      findings.push({
+        severity: "low",
+        detail: `${shortUrl}: '${pair.headingA}' and '${pair.headingB}' share ${pair.similarity}% similar text ("${pair.sample}...")`,
+        fix: `Rewrite one of these sections to eliminate duplicate content`
+      });
+    }
+  }
+  return { criterion: "duplicate_content", criterion_label: "Duplicate Content Blocks", score, status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P1" };
+}
+function checkCrossPageDuplication(data) {
+  const findings = [];
+  const pages = [];
+  if (data.homepage) {
+    pages.push({ url: data.homepage.finalUrl || `https://${data.domain}/`, paragraphs: extractPageParagraphs(data.homepage.text) });
+  }
+  if (data.blogSample) {
+    for (const page of data.blogSample) {
+      pages.push({ url: page.finalUrl || "", paragraphs: extractPageParagraphs(page.text) });
+    }
+  }
+  if (pages.length <= 1) {
+    findings.push({ severity: "info", detail: "Not enough pages to assess cross-page duplication" });
+    return { criterion: "cross_page_duplication", criterion_label: "Cross-Page Duplicate Content", score: 5, status: "partial", findings, fix_priority: "P3" };
+  }
+  const paragraphPageCount = /* @__PURE__ */ new Map();
+  for (const page of pages) {
+    const seen = /* @__PURE__ */ new Set();
+    for (const p of page.paragraphs) {
+      const fp = [...p.shingles].slice(0, 5).join("|");
+      if (!seen.has(fp)) {
+        seen.add(fp);
+        paragraphPageCount.set(fp, (paragraphPageCount.get(fp) || 0) + 1);
+      }
+    }
+  }
+  const boilerplateThreshold = Math.max(3, pages.length * 0.4);
+  const siteBoilerprints = /* @__PURE__ */ new Set();
+  for (const [fp, count] of paragraphPageCount) {
+    if (count >= boilerplateThreshold) siteBoilerprints.add(fp);
+  }
+  const crossDupPairs = [];
+  for (let i = 0; i < pages.length; i++) {
+    for (let j = i + 1; j < pages.length; j++) {
+      let dupCount = 0;
+      let sample = "";
+      for (const pA of pages[i].paragraphs) {
+        const fpA = [...pA.shingles].slice(0, 5).join("|");
+        if (siteBoilerprints.has(fpA)) continue;
+        for (const pB of pages[j].paragraphs) {
+          const sim = shingleSimilarity(pA.shingles, pB.shingles);
+          if (sim > 0.4) {
+            dupCount++;
+            if (!sample) sample = pA.text.slice(0, 80);
+            break;
+          }
+        }
+      }
+      if (dupCount >= 2) {
+        crossDupPairs.push({
+          urlA: pages[i].url.slice(0, 60),
+          urlB: pages[j].url.slice(0, 60),
+          dupCount,
+          sample
+        });
+      }
+    }
+  }
+  const affectedUrls = /* @__PURE__ */ new Set();
+  for (const pair of crossDupPairs) {
+    affectedUrls.add(pair.urlA);
+    affectedUrls.add(pair.urlB);
+  }
+  const affectedRatio = pages.length > 0 ? affectedUrls.size / pages.length : 0;
+  const totalDupParagraphs = crossDupPairs.reduce((s, p) => s + p.dupCount, 0);
+  let score;
+  if (crossDupPairs.length === 0) {
+    score = 10;
+    findings.push({ severity: "info", detail: `${pages.length} pages analyzed - no cross-page content duplication detected` });
+  } else if (affectedRatio <= 0.05 && totalDupParagraphs <= 4) {
+    score = 9;
+    findings.push({ severity: "info", detail: `${totalDupParagraphs} shared paragraph(s) across ${affectedUrls.size} page(s) - minor` });
+  } else if (affectedRatio <= 0.1) {
+    score = 7;
+    findings.push({ severity: "low", detail: `${totalDupParagraphs} shared paragraphs across ${affectedUrls.size} pages`, fix: "Rewrite shared content so each page provides a unique perspective" });
+  } else if (affectedRatio <= 0.2) {
+    score = 5;
+    findings.push({ severity: "medium", detail: `${affectedUrls.size} pages (${Math.round(affectedRatio * 100)}%) share duplicate paragraphs`, fix: "Significant cross-page duplication - AI engines may only index one version" });
+  } else if (affectedRatio <= 0.4) {
+    score = 3;
+    findings.push({ severity: "medium", detail: `${affectedUrls.size} pages (${Math.round(affectedRatio * 100)}%) contain shared content blocks`, fix: "Widespread copy-paste content across pages reduces overall site authority" });
+  } else {
+    score = 0;
+    findings.push({ severity: "high", detail: `${affectedUrls.size} pages (${Math.round(affectedRatio * 100)}%) share duplicate content`, fix: "Severe cross-page duplication - AI engines will likely ignore redundant pages entirely" });
+  }
+  for (const pair of crossDupPairs.slice(0, 3)) {
+    findings.push({
+      severity: "low",
+      detail: `${pair.dupCount} shared paragraph(s): ${pair.urlA} \u2194 ${pair.urlB} ("${pair.sample}...")`,
+      fix: "Rewrite shared paragraphs so each page has unique content"
+    });
+  }
+  return { criterion: "cross_page_duplication", criterion_label: "Cross-Page Duplicate Content", score, status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P1" };
+}
 function auditSiteFromData(data) {
   const topicCoherence = checkTopicCoherence(data);
   const cannibalization = checkContentCannibalization(data, topicCoherence.score);
@@ -2596,7 +2824,10 @@ function auditSiteFromData(data) {
     checkEvidencePackaging(data),
     checkEntityDisambiguation(data),
     checkExtractionFriction(data),
-    checkImageContextAI(data)
+    checkImageContextAI(data),
+    // V3 criteria (#35-#36)
+    checkDuplicateContent(data),
+    checkCrossPageDuplication(data)
   ];
 }
 async function auditSite(targetUrl) {
@@ -2620,11 +2851,11 @@ var WEIGHTS = {
   // Information density per page
   direct_answer_density: 0.05,
   // Direct answers to queries
-  qa_content_format: 0.05,
+  qa_content_format: 0.04,
   // Answer-shaped content structure
-  query_answer_alignment: 0.05,
+  query_answer_alignment: 0.04,
   // Relevance to actual AI queries
-  faq_section: 0.04,
+  faq_section: 0.03,
   // Structured Q&A pairs
   // ─── Content Organization (~30%) ──────────────────────────────────────────
   // HOW easily AI engines can extract and trust your content.
@@ -2672,8 +2903,13 @@ var WEIGHTS = {
   // Clear entity boundaries
   extraction_friction: 0.02,
   // Sentence length, voice, jargon
-  image_context_ai: 0.01
+  image_context_ai: 0.01,
   // Figure/figcaption, alt text quality
+  // ─── V3 Criteria ────────────────────────────────────────────────────────
+  duplicate_content: 0.05,
+  // Duplicate text blocks within pages
+  cross_page_duplication: 0.03
+  // Same paragraphs copied across pages
 };
 function calculateOverallScore(criteria) {
   let totalWeight = 0;
@@ -2803,7 +3039,9 @@ var PILLARS = {
     "fact_density",
     "citation_ready_writing",
     "answer_first_placement",
-    "evidence_packaging"
+    "evidence_packaging",
+    "duplicate_content",
+    "cross_page_duplication"
   ],
   "Content Structure": [
     "direct_answer_density",
@@ -2868,6 +3106,8 @@ var CLIENT_NAMES = {
   image_context_ai: "Image Context for AI",
   schema_coverage: "Schema Coverage",
   speakable_schema: "Speakable Schema",
+  duplicate_content: "Duplicate Content Blocks",
+  cross_page_duplication: "Cross-Page Duplicate Content",
   content_cannibalization: "Content Cannibalization",
   llms_txt: "llms.txt File",
   robots_txt: "robots.txt for AI",
@@ -2885,10 +3125,12 @@ var PILLAR_WEIGHTS = {
   citation_ready_writing: 0.04,
   answer_first_placement: 0.03,
   evidence_packaging: 0.03,
+  duplicate_content: 0.05,
+  cross_page_duplication: 0.03,
   direct_answer_density: 0.05,
-  qa_content_format: 0.05,
-  query_answer_alignment: 0.05,
-  faq_section: 0.04,
+  qa_content_format: 0.04,
+  query_answer_alignment: 0.04,
+  faq_section: 0.03,
   table_list_extractability: 0.03,
   definition_patterns: 0.02,
   entity_disambiguation: 0.02,
@@ -2921,6 +3163,8 @@ var CRITERION_EFFORT = {
   citation_ready_writing: "Medium",
   answer_first_placement: "Medium",
   evidence_packaging: "Medium",
+  duplicate_content: "Medium",
+  cross_page_duplication: "Medium",
   direct_answer_density: "Medium",
   qa_content_format: "Medium",
   query_answer_alignment: "Medium",
@@ -2976,6 +3220,8 @@ var FIX_DESCRIPTIONS = {
   image_context_ai: "Wrap images in <figure>/<figcaption> with descriptive alt text.",
   schema_coverage: "Extend structured data to inner pages (articles, services, products).",
   speakable_schema: "Add SpeakableSpecification schema for voice assistant compatibility.",
+  duplicate_content: "Rewrite duplicate text blocks so each section provides unique value.",
+  cross_page_duplication: "Rewrite shared paragraphs across pages so each page has unique content.",
   content_cannibalization: "Consolidate overlapping pages or differentiate titles and H1 headings.",
   llms_txt: "Create a /llms.txt file describing your site for AI engines.",
   robots_txt: "Update robots.txt to explicitly allow AI crawlers.",
@@ -3071,7 +3317,9 @@ var CRITERION_LABELS = {
   "Evidence Packaging": "Evidence Packaging",
   "Entity Disambiguation": "Entity Disambiguation",
   "Extraction Friction Score": "Extraction Friction Score",
-  "Image Context for AI": "Image Context for AI"
+  "Image Context for AI": "Image Context for AI",
+  "Duplicate Content Blocks": "Duplicate Content Blocks",
+  "Cross-Page Duplicate Content": "Cross-Page Duplicate Content"
 };
 function scoreToStatus(score) {
   if (score === 0) return "MISSING";
@@ -3166,9 +3414,9 @@ var CRITERION_WEIGHTS = {
   content_depth: 0.07,
   fact_density: 0.06,
   direct_answer_density: 0.05,
-  qa_content_format: 0.05,
-  query_answer_alignment: 0.05,
-  faq_section: 0.04,
+  qa_content_format: 0.04,
+  query_answer_alignment: 0.04,
+  faq_section: 0.03,
   // Content Organization (~30%)
   entity_consistency: 0.05,
   internal_linking: 0.04,
@@ -3197,7 +3445,10 @@ var CRITERION_WEIGHTS = {
   evidence_packaging: 0.03,
   entity_disambiguation: 0.02,
   extraction_friction: 0.02,
-  image_context_ai: 0.01
+  image_context_ai: 0.01,
+  // V3 Criteria
+  duplicate_content: 0.05,
+  cross_page_duplication: 0.03
 };
 var OPPORTUNITY_TEMPLATES = {
   llms_txt: {
@@ -3320,6 +3571,16 @@ var OPPORTUNITY_TEMPLATES = {
     effort: "Medium",
     description: "Ensure every question-format heading (H2/H3) is followed by a direct answer paragraph. This pattern is ideal for AI engine snippet extraction."
   },
+  duplicate_content: {
+    name: "Fix Duplicate Content Blocks",
+    effort: "Medium",
+    description: "Sections within pages contain identical or near-identical text. LLMs may flag this as low-quality or thin content, reducing citation authority. Rewrite duplicate blocks with unique angles."
+  },
+  cross_page_duplication: {
+    name: "Eliminate Cross-Page Duplicate Content",
+    effort: "Medium",
+    description: "The same paragraphs appear on multiple pages. AI engines may only index one version and ignore the rest. Rewrite shared content so each page offers a unique perspective."
+  },
   content_cannibalization: {
     name: "Resolve Content Cannibalization",
     effort: "Medium",
@@ -3728,9 +3989,9 @@ var PAGE_CRITERIA = {
   original_data: { weight: 0.1, label: "Original Data & Expert Content" },
   fact_density: { weight: 0.06, label: "Fact & Data Density" },
   direct_answer_density: { weight: 0.05, label: "Direct Answer Paragraphs" },
-  qa_content_format: { weight: 0.05, label: "Q&A Content Format" },
-  query_answer_alignment: { weight: 0.05, label: "Query-Answer Alignment" },
-  faq_section: { weight: 0.04, label: "FAQ Section Content" },
+  qa_content_format: { weight: 0.04, label: "Q&A Content Format" },
+  query_answer_alignment: { weight: 0.04, label: "Query-Answer Alignment" },
+  faq_section: { weight: 0.03, label: "FAQ Section Content" },
   // Content Organization
   content_freshness: { weight: 0.04, label: "Content Freshness Signals" },
   schema_markup: { weight: 0.03, label: "Schema.org Structured Data" },
@@ -3747,7 +4008,8 @@ var PAGE_CRITERIA = {
   evidence_packaging: { weight: 0.03, label: "Evidence Packaging" },
   entity_disambiguation: { weight: 0.02, label: "Entity Disambiguation" },
   extraction_friction: { weight: 0.02, label: "Extraction Friction Score" },
-  image_context_ai: { weight: 0.01, label: "Image Context for AI" }
+  image_context_ai: { weight: 0.01, label: "Image Context for AI" },
+  duplicate_content: { weight: 0.05, label: "Duplicate Content Blocks" }
 };
 function extractJsonLdBlocks(html) {
   const blocks = [];
@@ -4196,6 +4458,90 @@ function scoreImageContextAI(html) {
   if (contextualImages.length > 0) score += 3;
   return cap(score, 10);
 }
+var BOILERPLATE_PATTERNS = /\b(sign up|subscribe|get started|contact us|request a demo|free trial|book a call|schedule a|learn more|click here|follow us|share this|copyright|all rights reserved|privacy policy|terms of service)\b/i;
+function isBoilerplate(text) {
+  const words = text.split(/\s+/).length;
+  if (words < 20 && BOILERPLATE_PATTERNS.test(text)) return true;
+  if (/\b(cookie|gdpr|consent|opt.out)\b/i.test(text) && words < 30) return true;
+  return false;
+}
+function scoreDuplicateContent(html) {
+  return scoreDuplicateContentDetailed(html).score;
+}
+function scoreDuplicateContentDetailed(html) {
+  const sections = extractSectionsWithParagraphs(html);
+  if (sections.length < 2) return { score: 10, duplicates: [] };
+  const totalParagraphs = sections.reduce((sum, s) => sum + s.paragraphs.length, 0);
+  const duplicates = [];
+  let dupParagraphCount = 0;
+  for (let i = 0; i < sections.length; i++) {
+    for (let j = i + 1; j < sections.length; j++) {
+      for (const pA of sections[i].paragraphs) {
+        for (const pB of sections[j].paragraphs) {
+          const sim = shingleJaccard(pA.shingles, pB.shingles);
+          if (sim > 0.4) {
+            dupParagraphCount++;
+            duplicates.push({
+              headingA: sections[i].heading,
+              headingB: sections[j].heading,
+              similarity: Math.round(sim * 100),
+              sample: pA.text.slice(0, 80)
+            });
+            break;
+          }
+        }
+      }
+    }
+  }
+  if (dupParagraphCount === 0) return { score: 10, duplicates: [] };
+  const dupRatio = totalParagraphs > 0 ? dupParagraphCount / totalParagraphs : 0;
+  let score;
+  if (dupParagraphCount === 1 && dupRatio <= 0.05) {
+    score = 6;
+  } else if (dupParagraphCount === 1) {
+    score = 4;
+  } else if (dupParagraphCount === 2) {
+    score = 2;
+  } else {
+    score = 0;
+  }
+  return { score, duplicates };
+}
+function extractSectionsWithParagraphs(html) {
+  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
+  const parts = cleaned.split(/(?=<h[23]\b[^>]*>)/i);
+  const sections = [];
+  for (const part of parts) {
+    const headingMatch = part.match(/<h[23]\b[^>]*>([\s\S]*?)<\/h[23]>/i);
+    const heading = headingMatch ? headingMatch[1].replace(/<[^>]*>/g, "").trim() : "(intro)";
+    const pMatches = part.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
+    const paragraphs = pMatches.map((p) => {
+      const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
+      return { text, shingles: buildShingles(text, 4) };
+    }).filter((p) => p.shingles.size >= 3 && !isBoilerplate(p.text));
+    if (paragraphs.length > 0) {
+      sections.push({ heading, paragraphs });
+    }
+  }
+  return sections;
+}
+function buildShingles(text, n) {
+  const words = text.split(/\s+/).filter((w) => w.length > 1);
+  const shingles = /* @__PURE__ */ new Set();
+  for (let i = 0; i <= words.length - n; i++) {
+    shingles.add(words.slice(i, i + n).join(" "));
+  }
+  return shingles;
+}
+function shingleJaccard(a, b) {
+  if (a.size === 0 && b.size === 0) return 0;
+  let intersection = 0;
+  for (const s of a) {
+    if (b.has(s)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
 var SCORING_FUNCTIONS = {
   schema_markup: scoreSchemaMarkup,
   qa_content_format: scoreQAFormat,
@@ -4216,7 +4562,8 @@ var SCORING_FUNCTIONS = {
   evidence_packaging: scoreEvidencePackaging,
   entity_disambiguation: scoreEntityDisambiguation,
   extraction_friction: scoreExtractionFriction,
-  image_context_ai: scoreImageContextAI
+  image_context_ai: scoreImageContextAI,
+  duplicate_content: scoreDuplicateContent
 };
 function scorePage(html, url) {
   let totalWeight = 0;
@@ -4230,6 +4577,11 @@ function scorePage(html, url) {
     totalWeight += weight;
   }
   let aeoScore = totalWeight === 0 ? 0 : Math.round(weightedSum / totalWeight);
+  const dupScore = criterionScores.find((c) => c.criterion === "duplicate_content")?.score ?? 10;
+  if (dupScore <= 6) {
+    const dupCap = 35 + dupScore * 5;
+    aeoScore = Math.min(aeoScore, dupCap);
+  }
   const scoreCapped = aeoScore > 75;
   if (scoreCapped) aeoScore = 75;
   return { aeoScore, criterionScores, scoreCapped };
@@ -4449,6 +4801,15 @@ function checkHasCitationReadyContent(html) {
   }
   return null;
 }
+function checkDuplicateContentBlocks(html) {
+  const { score, duplicates } = scoreDuplicateContentDetailed(html);
+  if (score <= 6 && duplicates.length > 0) {
+    const first = duplicates[0];
+    const label = duplicates.length === 1 ? `Duplicate content: '${first.headingA}' and '${first.headingB}' share ${first.similarity}% similar text ("${first.sample}...")` : `${duplicates.length} duplicate blocks found (e.g. '${first.headingA}' and '${first.headingB}' \u2014 "${first.sample}...")`;
+    return { check: "duplicate-content", label, severity: score <= 3 ? "error" : "warning" };
+  }
+  return null;
+}
 function analyzePage(html, url, category) {
   const title = extractTitle(html);
   const textContent = getTextContent2(html);
@@ -4467,7 +4828,8 @@ function analyzePage(html, url, category) {
     checkImagesMissingAlt(html),
     checkNoInternalLinks(html, url),
     checkNoAnswerBlock(html),
-    checkNoEvidence(html, url)
+    checkNoEvidence(html, url),
+    checkDuplicateContentBlocks(html)
   ];
   for (const result of issueChecks) {
     if (result) issues.push(result);
@@ -4535,7 +4897,7 @@ async function audit(domain, options) {
     }
   }
   if (options?.fullCrawl) {
-    const { crawlFullSite: crawlFullSite2 } = await import("./full-site-crawler-BCJS67WQ.js");
+    const { crawlFullSite: crawlFullSite2 } = await import("./full-site-crawler-TQ35TB2X.js");
     const crawlResult = await crawlFullSite2(siteData, {
       maxPages: options.maxPages ?? 200,
       concurrency: options.concurrency ?? 5
@@ -4830,9 +5192,9 @@ var CRITERION_WEIGHTS2 = {
   content_depth: 0.07,
   fact_density: 0.06,
   direct_answer_density: 0.05,
-  qa_content_format: 0.05,
-  query_answer_alignment: 0.05,
-  faq_section: 0.04,
+  qa_content_format: 0.04,
+  query_answer_alignment: 0.04,
+  faq_section: 0.03,
   // Content Organization (~30%)
   entity_consistency: 0.05,
   internal_linking: 0.04,
@@ -4846,6 +5208,8 @@ var CRITERION_WEIGHTS2 = {
   clean_html: 0.02,
   // Technical Plumbing (~15%)
   content_cannibalization: 0.02,
+  duplicate_content: 0.05,
+  cross_page_duplication: 0.03,
   llms_txt: 0.02,
   robots_txt: 0.02,
   content_velocity: 0.02,
@@ -4890,7 +5254,9 @@ var PHASE_CONFIG = [
       "citation_ready_writing",
       "answer_first_placement",
       "evidence_packaging",
-      "entity_disambiguation"
+      "entity_disambiguation",
+      "duplicate_content",
+      "cross_page_duplication"
     ]
   },
   {
@@ -5770,6 +6136,66 @@ Summarization: yes`,
     }
     return fixes;
   },
+  duplicate_content: (c, pages) => {
+    if (c.score >= 10) return [];
+    const impact = impactFromScore(c.score);
+    const effort = effortForCriterion("duplicate_content", c.score);
+    const affected = getAffectedPages("duplicate_content", pages);
+    const sectionPairs = c.findings.filter((f) => f.detail.includes("' and '")).map((f) => {
+      const match = f.detail.match(/'([^']+)' and '([^']+)'/);
+      return match ? { a: match[1], b: match[2] } : null;
+    }).filter(Boolean);
+    const steps = [
+      "Identify sections with duplicate or near-identical text",
+      "Rewrite each section to provide a unique angle on the topic",
+      "Ensure each heading section adds new information for the reader"
+    ];
+    if (sectionPairs.length > 0) {
+      const pair = sectionPairs[0];
+      steps.unshift(`Start with '${pair.a}' and '${pair.b}' which share similar text`);
+    }
+    return [{
+      id: "fix-duplicate-content",
+      criterion: c.criterion_label,
+      criterionId: c.criterion,
+      title: "Fix duplicate content blocks",
+      description: "Sections within pages contain identical or near-identical text. LLMs may flag this as low-quality content, reducing the authority of the page.",
+      impact,
+      effort,
+      impactScore: 0,
+      category: "content",
+      steps,
+      successCriteria: "Each section within a page provides unique content",
+      affectedPages: affected,
+      pageCount: affected?.length
+    }];
+  },
+  cross_page_duplication: (c, pages) => {
+    if (c.score >= 10) return [];
+    const impact = impactFromScore(c.score);
+    const effort = effortForCriterion("cross_page_duplication", c.score);
+    const affected = getAffectedPages("cross_page_duplication", pages);
+    return [{
+      id: "fix-cross-page-duplication",
+      criterion: c.criterion_label,
+      criterionId: c.criterion,
+      title: "Eliminate cross-page duplicate content",
+      description: "The same paragraphs appear on multiple pages. AI engines may only index one version, wasting the others.",
+      impact,
+      effort,
+      impactScore: 0,
+      category: "content",
+      steps: [
+        "Identify paragraphs that are copy-pasted across multiple pages",
+        "Rewrite each instance to provide a unique angle relevant to that page",
+        "Move truly shared content to a single resource page and link to it",
+        "Use canonical tags if pages must share content"
+      ],
+      successCriteria: "Each page has unique body content with no copy-pasted paragraphs",
+      affectedPages: affected,
+      pageCount: affected?.length
+    }];
+  },
   visible_date_signal: (c, pages) => {
     if (c.score >= 10) return [];
     const impact = impactFromScore(c.score);