aeorank 3.1.0 → 3.1.1

package/dist/cli.js

@@ -76,6 +76,75 @@ function detectParkedDomain(bodySnippet) {
   return { isParked: false };
 }
 
+// src/duplicate-content.ts
+var BOILERPLATE_PATTERNS = /\b(sign up|subscribe|get started|contact us|request a demo|free trial|book a call|schedule a|learn more|click here|follow us|share this|copyright|all rights reserved|privacy policy|terms of service)\b/i;
+var MIN_SUBSTANTIVE_WORDS = 15;
+var MAX_METADATA_WORDS = 24;
+var MAX_METADATA_LABEL_WORDS = 4;
+function normalizeParagraphText(htmlFragment) {
+  return htmlFragment.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
+}
+function tokenize(text) {
+  return text.split(/\s+/).map((word) => word.replace(/^[^a-z0-9]+|[^a-z0-9]+$/gi, "")).filter((word) => word.length > 0);
+}
+function isBoilerplateParagraph(text, words) {
+  if (words < 20 && BOILERPLATE_PATTERNS.test(text)) return true;
+  if (/\b(cookie|gdpr|consent|opt.out)\b/i.test(text) && words < 30) return true;
+  return false;
+}
+function isMetadataParagraph(text, words) {
+  const labelMatch = text.match(/^([^:]{1,60}):\s+/);
+  if (!labelMatch) return false;
+  const labelWords = tokenize(labelMatch[1]).length;
+  return labelWords > 0 && labelWords <= MAX_METADATA_LABEL_WORDS && words <= MAX_METADATA_WORDS;
+}
+function buildShinglesFromTokens(words, n = 4) {
+  const shingles = /* @__PURE__ */ new Set();
+  for (let i = 0; i <= words.length - n; i++) {
+    shingles.add(words.slice(i, i + n).join(" "));
+  }
+  return shingles;
+}
+function createParagraph(htmlFragment) {
+  const text = normalizeParagraphText(htmlFragment);
+  const words = tokenize(text);
+  if (words.length < MIN_SUBSTANTIVE_WORDS) return null;
+  if (isBoilerplateParagraph(text, words.length)) return null;
+  if (isMetadataParagraph(text, words.length)) return null;
+  const shingles = buildShinglesFromTokens(words);
+  if (shingles.size < 3) return null;
+  return { text, shingles };
+}
+function stripNonContentHtml(html) {
+  return html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
+}
+function extractDuplicateContentParagraphs(html) {
+  const cleaned = stripNonContentHtml(html);
+  const matches = cleaned.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
+  return matches.map(createParagraph).filter((paragraph) => paragraph !== null);
+}
+function extractDuplicateContentSections(html) {
+  const cleaned = stripNonContentHtml(html);
+  const parts = cleaned.split(/(?=<h[23]\b[^>]*>)/i);
+  const sections = [];
+  for (const part of parts) {
+    const headingMatch = part.match(/<h[23]\b[^>]*>([\s\S]*?)<\/h[23]>/i);
+    const heading = headingMatch ? headingMatch[1].replace(/<[^>]*>/g, "").trim() : "(intro)";
+    const paragraphs = (part.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || []).map(createParagraph).filter((paragraph) => paragraph !== null);
+    if (paragraphs.length > 0) sections.push({ heading, paragraphs });
+  }
+  return sections;
+}
+function shingleJaccardSimilarity(a, b) {
+  if (a.size === 0 && b.size === 0) return 0;
+  let intersection = 0;
+  for (const shingle of a) {
+    if (b.has(shingle)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
 // src/site-crawler.ts
 async function fetchText(url) {
   try {
@@ -2556,56 +2625,8 @@ function checkImageContextAI(data) {
   }
   return { criterion: "image_context_ai", criterion_label: "Image Context for AI", score: Math.min(10, score), status: score >= 7 ? "pass" : score >= 4 ? "partial" : "fail", findings, fix_priority: score >= 7 ? "P3" : "P2" };
 }
-var BOILERPLATE_RE = /\b(sign up|subscribe|get started|contact us|request a demo|free trial|book a call|schedule a|learn more|click here|follow us|share this|copyright|all rights reserved|privacy policy|terms of service)\b/i;
-function isBoilerplateParagraph(text) {
-  const words = text.split(/\s+/).length;
-  if (words < 20 && BOILERPLATE_RE.test(text)) return true;
-  if (/\b(cookie|gdpr|consent|opt.out)\b/i.test(text) && words < 30) return true;
-  return false;
-}
-function toShingles(text, n = 4) {
-  const words = text.split(/\s+/).filter((w) => w.length > 1);
-  const shingles = /* @__PURE__ */ new Set();
-  for (let i = 0; i <= words.length - n; i++) {
-    shingles.add(words.slice(i, i + n).join(" "));
-  }
-  return shingles;
-}
-function shingleSimilarity(a, b) {
-  if (a.size === 0 && b.size === 0) return 0;
-  let intersection = 0;
-  for (const s of a) {
-    if (b.has(s)) intersection++;
-  }
-  const union = a.size + b.size - intersection;
-  return union === 0 ? 0 : intersection / union;
-}
-function extractPageParagraphs(html) {
-  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
-  const pMatches = cleaned.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
-  return pMatches.map((p) => {
-    const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
-    return { text, shingles: toShingles(text) };
-  }).filter((p) => p.shingles.size >= 3 && !isBoilerplateParagraph(p.text));
-}
-function splitIntoSectionsWithParagraphs(html) {
-  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
-  const parts = cleaned.split(/(?=<h[23]\b[^>]*>)/i);
-  const sections = [];
-  for (const part of parts) {
-    const hMatch = part.match(/<h[23]\b[^>]*>([\s\S]*?)<\/h[23]>/i);
-    const heading = hMatch ? hMatch[1].replace(/<[^>]*>/g, "").trim() : "(intro)";
-    const pMatches = part.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
-    const paragraphs = pMatches.map((p) => {
-      const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
-      return { text, shingles: toShingles(text) };
-    }).filter((p) => p.shingles.size >= 3 && !isBoilerplateParagraph(p.text));
-    if (paragraphs.length > 0) sections.push({ heading, paragraphs });
-  }
-  return sections;
-}
 function findIntraPageDuplicates(html) {
-  const sections = splitIntoSectionsWithParagraphs(html);
+  const sections = extractDuplicateContentSections(html);
   if (sections.length < 2) return [];
   const pairs = [];
   for (let i = 0; i < sections.length; i++) {
@@ -2614,7 +2635,7 @@ function findIntraPageDuplicates(html) {
       for (const pA of sections[i].paragraphs) {
         if (found) break;
         for (const pB of sections[j].paragraphs) {
-          const sim = shingleSimilarity(pA.shingles, pB.shingles);
+          const sim = shingleJaccardSimilarity(pA.shingles, pB.shingles);
           if (sim > 0.4) {
             pairs.push({
               headingA: sections[i].heading,
@@ -2694,11 +2715,11 @@ function checkCrossPageDuplication(data) {
   const findings = [];
   const pages = [];
   if (data.homepage) {
-    pages.push({ url: data.homepage.finalUrl || `https://${data.domain}/`, paragraphs: extractPageParagraphs(data.homepage.text) });
+    pages.push({ url: data.homepage.finalUrl || `https://${data.domain}/`, paragraphs: extractDuplicateContentParagraphs(data.homepage.text) });
   }
   if (data.blogSample) {
     for (const page of data.blogSample) {
-      pages.push({ url: page.finalUrl || "", paragraphs: extractPageParagraphs(page.text) });
+      pages.push({ url: page.finalUrl || "", paragraphs: extractDuplicateContentParagraphs(page.text) });
     }
   }
   if (pages.length <= 1) {
@@ -2730,7 +2751,7 @@ function checkCrossPageDuplication(data) {
         const fpA = [...pA.shingles].slice(0, 5).join("|");
         if (siteBoilerprints.has(fpA)) continue;
         for (const pB of pages[j].paragraphs) {
-          const sim = shingleSimilarity(pA.shingles, pB.shingles);
+          const sim = shingleJaccardSimilarity(pA.shingles, pB.shingles);
           if (sim > 0.4) {
             dupCount++;
             if (!sample) sample = pA.text.slice(0, 80);
@@ -4433,18 +4454,11 @@ function scoreImageContextAI(html) {
   if (contextualImages.length > 0) score += 3;
   return cap(score, 10);
 }
-var BOILERPLATE_PATTERNS = /\b(sign up|subscribe|get started|contact us|request a demo|free trial|book a call|schedule a|learn more|click here|follow us|share this|copyright|all rights reserved|privacy policy|terms of service)\b/i;
-function isBoilerplate(text) {
-  const words = text.split(/\s+/).length;
-  if (words < 20 && BOILERPLATE_PATTERNS.test(text)) return true;
-  if (/\b(cookie|gdpr|consent|opt.out)\b/i.test(text) && words < 30) return true;
-  return false;
-}
 function scoreDuplicateContent(html) {
   return scoreDuplicateContentDetailed(html).score;
 }
 function scoreDuplicateContentDetailed(html) {
-  const sections = extractSectionsWithParagraphs(html);
+  const sections = extractDuplicateContentSections(html);
   if (sections.length < 2) return { score: 10, duplicates: [] };
   const totalParagraphs = sections.reduce((sum, s) => sum + s.paragraphs.length, 0);
   const duplicates = [];
@@ -4453,7 +4467,7 @@ function scoreDuplicateContentDetailed(html) {
     for (let j = i + 1; j < sections.length; j++) {
       for (const pA of sections[i].paragraphs) {
         for (const pB of sections[j].paragraphs) {
-          const sim = shingleJaccard(pA.shingles, pB.shingles);
+          const sim = shingleJaccardSimilarity(pA.shingles, pB.shingles);
           if (sim > 0.4) {
             dupParagraphCount++;
             duplicates.push({
@@ -4482,41 +4496,6 @@ function scoreDuplicateContentDetailed(html) {
   }
   return { score, duplicates };
 }
-function extractSectionsWithParagraphs(html) {
-  const cleaned = html.replace(/<(script|style|nav|header|footer|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "").replace(/<aside\b[^>]*>[\s\S]*?<\/aside>/gi, "");
-  const parts = cleaned.split(/(?=<h[23]\b[^>]*>)/i);
-  const sections = [];
-  for (const part of parts) {
-    const headingMatch = part.match(/<h[23]\b[^>]*>([\s\S]*?)<\/h[23]>/i);
-    const heading = headingMatch ? headingMatch[1].replace(/<[^>]*>/g, "").trim() : "(intro)";
-    const pMatches = part.match(/<p\b[^>]*>([\s\S]*?)<\/p>/gi) || [];
-    const paragraphs = pMatches.map((p) => {
-      const text = p.replace(/<[^>]*>/g, " ").replace(/&\w+;/g, " ").replace(/\s+/g, " ").trim().toLowerCase();
-      return { text, shingles: buildShingles(text, 4) };
-    }).filter((p) => p.shingles.size >= 3 && !isBoilerplate(p.text));
-    if (paragraphs.length > 0) {
-      sections.push({ heading, paragraphs });
-    }
-  }
-  return sections;
-}
-function buildShingles(text, n) {
-  const words = text.split(/\s+/).filter((w) => w.length > 1);
-  const shingles = /* @__PURE__ */ new Set();
-  for (let i = 0; i <= words.length - n; i++) {
-    shingles.add(words.slice(i, i + n).join(" "));
-  }
-  return shingles;
-}
-function shingleJaccard(a, b) {
-  if (a.size === 0 && b.size === 0) return 0;
-  let intersection = 0;
-  for (const s of a) {
-    if (b.has(s)) intersection++;
-  }
-  const union = a.size + b.size - intersection;
-  return union === 0 ? 0 : intersection / union;
-}
 var SCORING_FUNCTIONS = {
   schema_markup: scoreSchemaMarkup,
   qa_content_format: scoreQAFormat,