aeorank 1.4.0 → 1.6.0

package/dist/cli.js

@@ -2504,12 +2504,345 @@ async function fetchMultiPageData(siteData, options) {
   return added;
 }
 
+// src/page-scorer.ts
+var PAGE_CRITERIA = {
+  schema_markup: { weight: 0.15, label: "Schema.org Structured Data" },
+  qa_content_format: { weight: 0.15, label: "Q&A Content Format" },
+  clean_html: { weight: 0.1, label: "Clean, Crawlable HTML" },
+  faq_section: { weight: 0.1, label: "FAQ Section Content" },
+  original_data: { weight: 0.1, label: "Original Data & Expert Content" },
+  query_answer_alignment: { weight: 0.08, label: "Query-Answer Alignment" },
+  content_freshness: { weight: 0.07, label: "Content Freshness Signals" },
+  table_list_extractability: { weight: 0.07, label: "Table & List Extractability" },
+  direct_answer_density: { weight: 0.07, label: "Direct Answer Paragraphs" },
+  semantic_html: { weight: 0.05, label: "Semantic HTML5 & Accessibility" },
+  fact_density: { weight: 0.05, label: "Fact & Data Density" },
+  definition_patterns: { weight: 0.04, label: "Definition Patterns" },
+  canonical_url: { weight: 0.04, label: "Canonical URL Strategy" },
+  visible_date_signal: { weight: 0.04, label: "Visible Date Signal" }
+};
+function extractJsonLdBlocks(html) {
+  const blocks = [];
+  const regex = /<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+  let match;
+  while ((match = regex.exec(html)) !== null) {
+    blocks.push(match[1]);
+  }
+  return blocks;
+}
+function extractTypesFromJsonLd(blocks) {
+  const types = /* @__PURE__ */ new Set();
+  for (const block of blocks) {
+    const typeMatches = block.match(/"@type"\s*:\s*"([^"]+)"/g) || [];
+    for (const m of typeMatches) {
+      const t = m.match(/"@type"\s*:\s*"([^"]+)"/);
+      if (t) types.add(t[1]);
+    }
+  }
+  return types;
+}
+function getTextContent(html) {
+  return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
+}
+function extractQuestionHeadings2(html) {
+  const headings = html.match(/<h[2-3][^>]*>([\s\S]*?)<\/h[2-3]>/gi) || [];
+  const questions = [];
+  for (const h of headings) {
+    const text = h.replace(/<[^>]*>/g, "").trim();
+    if (/\?$/.test(text) || /^(what|how|why|when|where|who|which|can|do|does|is|are|should|will)\b/i.test(text)) {
+      questions.push(text);
+    }
+  }
+  return questions;
+}
+function countAnsweredQuestions(html) {
+  const questions = extractQuestionHeadings2(html);
+  if (questions.length === 0) return { total: 0, answered: 0 };
+  let answered = 0;
+  for (const q of questions) {
+    const escaped = q.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const pattern = new RegExp(escaped + "[\\s\\S]*?</h[2-3]>\\s*<p[^>]*>([\\s\\S]*?)</p>", "i");
+    const match = html.match(pattern);
+    if (match && match[1].replace(/<[^>]*>/g, "").trim().length >= 20) {
+      answered++;
+    }
+  }
+  return { total: questions.length, answered };
+}
+function cap(value, max) {
+  return Math.min(value, max);
+}
+function scoreSchemaMarkup(html) {
+  const blocks = extractJsonLdBlocks(html);
+  if (blocks.length === 0) return 0;
+  let score = 3;
+  const types = extractTypesFromJsonLd(blocks);
+  const knownTypes = [
+    "Organization",
+    "LocalBusiness",
+    "Article",
+    "FAQPage",
+    "Product",
+    "WebPage",
+    "BreadcrumbList",
+    "HowTo",
+    "Person",
+    "WebSite",
+    "BlogPosting",
+    "Service"
+  ];
+  let knownCount = 0;
+  for (const t of types) {
+    if (knownTypes.includes(t)) knownCount++;
+  }
+  score += cap(knownCount * 2, 4);
+  if (types.has("Organization") || types.has("LocalBusiness")) score += 2;
+  if (types.has("FAQPage")) score += 1;
+  return cap(score, 10);
+}
+function scoreQAFormat(html) {
+  const questions = extractQuestionHeadings2(html);
+  let score = 0;
+  if (questions.length >= 10) score += 5;
+  else if (questions.length >= 3) score += 3;
+  else if (questions.length >= 1) score += 1;
+  const { answered } = countAnsweredQuestions(html);
+  if (answered >= 1) score += 3;
+  const h1Matches = html.match(/<h1[\s>]/gi) || [];
+  if (h1Matches.length === 1) score += 2;
+  return cap(score, 10);
+}
+function scoreCleanHtml(html) {
+  let score = 0;
+  const semantics = ["<main", "<article", "<section"];
+  let semCount = 0;
+  for (const tag of semantics) {
+    if (html.toLowerCase().includes(tag)) semCount++;
+  }
+  score += cap(semCount, 3);
+  const h1Matches = html.match(/<h1[\s>]/gi) || [];
+  if (h1Matches.length === 1) score += 2;
+  const text = getTextContent(html);
+  if (text.length > 500) score += 3;
+  const hasTitle = /<title[^>]*>[^<]+<\/title>/i.test(html);
+  const hasDesc = /<meta\s[^>]*name=["']description["'][^>]*content=["'][^"']+["']/i.test(html) || /<meta\s[^>]*content=["'][^"']+["'][^>]*name=["']description["']/i.test(html);
+  if (hasTitle && hasDesc) score += 2;
+  return cap(score, 10);
+}
+function scoreFaqSection(html) {
+  let score = 0;
+  const lowerHtml = html.toLowerCase();
+  if (/frequently\s*asked|faq/i.test(html)) score += 2;
+  const blocks = extractJsonLdBlocks(html);
+  const types = extractTypesFromJsonLd(blocks);
+  if (types.has("FAQPage")) score += 3;
+  const questions = extractQuestionHeadings2(html);
+  if (questions.length >= 10) score += 1;
+  if (/<details[\s>]/i.test(html) || /accordion|collapsible|toggle/i.test(lowerHtml)) score += 1;
+  return cap(score, 10);
+}
+function scoreOriginalData(html) {
+  let score = 0;
+  const text = getTextContent(html);
+  if (/\b(our (study|analysis|research|survey|data|findings))\b/i.test(text)) {
+    score += 3;
+  } else if (/\d+(\.\d+)?%|\$[\d,.]+|\b\d{1,3}(,\d{3})+\b/.test(text)) {
+    score += 1;
+  }
+  if (/\bcase\s+stud(y|ies)\b/i.test(text) && /\d+(\.\d+)?%|\$[\d,.]+/.test(text)) {
+    score += 3;
+  } else if (/\bcase\s+stud(y|ies)\b/i.test(text)) {
+    score += 1;
+  }
+  if (/\baccording\s+to\b|\bexpert|\b(Ph\.?D|MD|professor|analyst|researcher)\b/i.test(text)) {
+    score += 2;
+  }
+  if (/href=["'][^"']*\/blog\b/i.test(html)) {
+    score += 2;
+  }
+  return cap(score, 10);
+}
+function scoreQueryAnswerAlignment(html) {
+  const { total, answered } = countAnsweredQuestions(html);
+  if (total === 0) return 5;
+  const ratio = answered / total;
+  if (ratio >= 0.8) return 10;
+  if (ratio >= 0.5) return 7;
+  if (answered > 0) return 4;
+  return 0;
+}
+function scoreContentFreshness(html) {
+  let score = 0;
+  const blocks = extractJsonLdBlocks(html);
+  const allJsonLd = blocks.join(" ");
+  if (/datePublished|dateModified/i.test(allJsonLd)) score += 3;
+  const timeElements = html.match(/<time[\s>]/gi) || [];
+  if (timeElements.length >= 2) score += 3;
+  else if (timeElements.length === 1) score += 1;
+  if (/<meta\s[^>]*property=["']article:(published_time|modified_time)["']/i.test(html)) score += 2;
+  const currentYear = (/* @__PURE__ */ new Date()).getFullYear();
+  const yearPattern = new RegExp(`\\b(${currentYear}|${currentYear - 1})\\b`);
+  if (yearPattern.test(html)) score += 2;
+  return cap(score, 10);
+}
+function scoreTableListExtractability(html) {
+  let score = 0;
+  const tablesWithHeaders = html.match(/<table[\s\S]*?<th[\s>]/gi) || [];
+  if (tablesWithHeaders.length >= 2) score += 4;
+  else if (tablesWithHeaders.length === 1) score += 3;
+  if (tablesWithHeaders.length === 0 && /<table[\s>]/i.test(html)) score += 1;
+  if (/<ol[\s>]/i.test(html)) score += 2;
+  if (/<ul[\s>]/i.test(html)) score += 2;
+  const listItems = html.match(/<li[\s>]/gi) || [];
+  if (listItems.length >= 10) score += 1;
+  if (/<dl[\s>]/i.test(html)) score += 1;
+  return cap(score, 10);
+}
+function scoreDirectAnswerDensity(html) {
+  let score = 0;
+  const { answered } = countAnsweredQuestions(html);
+  if (answered >= 3) score += 6;
+  else if (answered >= 1) score += 3;
+  const paragraphs = html.match(/<p[^>]*>([\s\S]*?)<\/p>/gi) || [];
+  let snippetCount = 0;
+  for (const p of paragraphs) {
+    const text = p.replace(/<[^>]*>/g, "").trim();
+    const words = text.split(/\s+/).filter((w) => w.length > 0).length;
+    if (words >= 40 && words <= 150) snippetCount++;
+  }
+  if (snippetCount >= 3) score += 2;
+  else if (snippetCount >= 1) score += 1;
+  const directOpeners = getTextContent(html).match(/\b(yes|no|in short|the answer is|simply put|in summary)\b/gi) || [];
+  if (directOpeners.length >= 2) score += 2;
+  return cap(score, 10);
+}
+function scoreSemanticHtml(html) {
+  let score = 0;
+  const lowerHtml = html.toLowerCase();
+  const elements = ["<main", "<article", "<time", "<nav", "<header", "<footer"];
+  let count = 0;
+  for (const el of elements) {
+    if (lowerHtml.includes(el)) count++;
+  }
+  score += cap(Math.floor(count * 0.7), 4);
+  const imgTags = html.match(/<img\s[^>]*>/gi) || [];
+  if (imgTags.length > 0) {
+    let withAlt = 0;
+    for (const img of imgTags) {
+      if (/\salt=["'][^"']*["']/i.test(img)) withAlt++;
+    }
+    if (withAlt / imgTags.length >= 0.8) score += 2;
+  }
+  if (/<html[^>]*\slang=["'][^"']+["']/i.test(html)) score += 2;
+  if (/\baria-/i.test(html)) score += 2;
+  return cap(score, 10);
+}
+function scoreFactDensity(html) {
+  let score = 0;
+  const text = getTextContent(html);
+  const numericPatterns = text.match(/\d+(\.\d+)?%|\$[\d,.]+|\b\d{1,3}(,\d{3})+\b|\b\d+\s*(million|billion|thousand|users|customers|employees)\b/gi) || [];
+  if (numericPatterns.length >= 6) score += 5;
+  else if (numericPatterns.length >= 3) score += 3;
+  else if (numericPatterns.length >= 1) score += 1;
+  const years = /* @__PURE__ */ new Set();
+  const yearMatches = text.match(/\b(19|20)\d{2}\b/g) || [];
+  for (const y of yearMatches) years.add(y);
+  if (years.size >= 2) score += 2;
+  else if (years.size === 1) score += 1;
+  if (/\baccording to\b|\bsource:\s|\bcited\b|\breported by\b/i.test(text)) score += 2;
+  const units = text.match(/\b\d+\s*(kg|lb|miles|km|hours|minutes|days|months|years|GB|MB|TB)\b/gi) || [];
+  if (units.length >= 2) score += 1;
+  return cap(score, 10);
+}
+function scoreDefinitionPatterns(html) {
+  let score = 0;
+  const text = getTextContent(html);
+  const defPatterns = text.match(/\b(is a|is an|refers to|defined as|means that|also known as|abbreviated as)\b/gi) || [];
+  if (defPatterns.length >= 3) score += 5;
+  else if (defPatterns.length >= 1) score += 3;
+  const early = text.slice(0, 2e3);
+  if (/\b(is a|is an|refers to|defined as)\b/i.test(early)) score += 2;
+  if (/<dfn[\s>]/i.test(html) || /<abbr[\s>]/i.test(html)) score += 1;
+  if (/<dl[\s>]/i.test(html) || /glossary/i.test(html)) score += 2;
+  return cap(score, 10);
+}
+function scoreCanonicalUrl(html, url) {
+  let score = 0;
+  const canonicalMatch = html.match(/<link[^>]*rel=["']canonical["'][^>]*href=["']([^"']+)["']/i) || html.match(/<link[^>]*href=["']([^"']+)["'][^>]*rel=["']canonical["']/i);
+  if (!canonicalMatch) return 0;
+  score += 4;
+  const canonicalHref = canonicalMatch[1];
+  if (url) {
+    try {
+      const canonicalUrl = new URL(canonicalHref, url);
+      const pageUrl = new URL(url);
+      if (canonicalUrl.pathname === pageUrl.pathname && canonicalUrl.hostname === pageUrl.hostname) {
+        score += 3;
+      }
+    } catch {
+    }
+  }
+  if (canonicalHref.startsWith("https://")) score += 2;
+  const allCanonicals = html.match(/<link[^>]*rel=["']canonical["'][^>]*>/gi) || [];
+  if (allCanonicals.length === 1) score += 1;
+  return cap(score, 10);
+}
+function scoreVisibleDateSignal(html) {
+  let score = 0;
+  const timeWithDatetime = html.match(/<time[^>]*datetime=["'][^"']+["'][^>]*>[^<]+<\/time>/gi) || [];
+  if (timeWithDatetime.length > 0) score += 5;
+  const blocks = extractJsonLdBlocks(html);
+  const allJsonLd = blocks.join(" ");
+  if (/datePublished|dateModified/i.test(allJsonLd)) score += 3;
+  if (/<meta\s[^>]*property=["']article:(published_time|modified_time)["']/i.test(html)) score += 2;
+  const modifiedMatch = allJsonLd.match(/"dateModified"\s*:\s*"([^"]+)"/i);
+  if (modifiedMatch) {
+    try {
+      const modified = new Date(modifiedMatch[1]);
+      const daysDiff = (Date.now() - modified.getTime()) / (1e3 * 60 * 60 * 24);
+      if (daysDiff <= 180) score += 1;
+    } catch {
+    }
+  }
+  return cap(score, 10);
+}
+var SCORING_FUNCTIONS = {
+  schema_markup: scoreSchemaMarkup,
+  qa_content_format: scoreQAFormat,
+  clean_html: scoreCleanHtml,
+  faq_section: scoreFaqSection,
+  original_data: scoreOriginalData,
+  query_answer_alignment: scoreQueryAnswerAlignment,
+  content_freshness: scoreContentFreshness,
+  table_list_extractability: scoreTableListExtractability,
+  direct_answer_density: scoreDirectAnswerDensity,
+  semantic_html: scoreSemanticHtml,
+  fact_density: scoreFactDensity,
+  definition_patterns: scoreDefinitionPatterns,
+  canonical_url: scoreCanonicalUrl,
+  visible_date_signal: scoreVisibleDateSignal
+};
+function scorePage(html, url) {
+  let totalWeight = 0;
+  let weightedSum = 0;
+  const criterionScores = [];
+  for (const [criterion, { weight, label }] of Object.entries(PAGE_CRITERIA)) {
+    const fn = SCORING_FUNCTIONS[criterion];
+    const score = fn(html, url);
+    criterionScores.push({ criterion, criterion_label: label, score, weight });
+    weightedSum += score / 10 * weight * 100;
+    totalWeight += weight;
+  }
+  const aeoScore = totalWeight === 0 ? 0 : Math.round(weightedSum / totalWeight);
+  return { aeoScore, criterionScores };
+}
+
 // src/page-analyzer.ts
 function extractTitle(html) {
   const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
   return match ? match[1].replace(/\s+/g, " ").trim() : "";
 }
-function getTextContent(html) {
+function getTextContent2(html) {
   return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
 }
 function countWords(text) {
@@ -2662,7 +2995,7 @@ function checkHasQuestionHeadings(html) {
 }
 function analyzePage(html, url, category) {
   const title = extractTitle(html);
-  const textContent = getTextContent(html);
+  const textContent = getTextContent2(html);
   const wordCount = countWords(textContent);
   const issues = [];
   const strengths = [];
@@ -2688,7 +3021,8 @@ function analyzePage(html, url, category) {
   for (const result of strengthChecks) {
     if (result) strengths.push(result);
   }
-  return { url, title, category, wordCount, issues, strengths };
+  const { aeoScore, criterionScores } = scorePage(html, url);
+  return { url, title, category, wordCount, issues, strengths, aeoScore, criterionScores };
 }
 function analyzeAllPages(siteData) {
   const reviews = [];
@@ -2937,17 +3271,22 @@ function generateHtmlReport(result) {
         <td>${escapeHtml(opp.description)}</td>
       </tr>`;
   }).join("\n");
-  const pagesRows = (result.pagesReviewed || []).map((page) => {
+  const pagesReviewed = result.pagesReviewed || [];
+  const pagesRows = pagesReviewed.map((page) => {
     const issueCount = page.issues.length;
     const strengthCount = page.strengths.length;
+    const aeoDisplay = page.aeoScore != null ? `<span style="font-weight:600;color:${scoreColor(page.aeoScore)}">${page.aeoScore}</span>` : "-";
     return `<tr>
         <td>${escapeHtml(page.url)}</td>
         <td>${escapeHtml(page.category)}</td>
         <td>${page.wordCount}</td>
+        <td>${aeoDisplay}</td>
         <td>${issueCount}</td>
         <td>${strengthCount}</td>
       </tr>`;
   }).join("\n");
+  const scoredPages = pagesReviewed.filter((p) => p.aeoScore != null);
+  const avgPageScore = scoredPages.length > 0 ? Math.round(scoredPages.reduce((sum, p) => sum + p.aeoScore, 0) / scoredPages.length) : null;
   const now = (/* @__PURE__ */ new Date()).toISOString();
   return `<!DOCTYPE html>
 <html lang="en">
@@ -2983,10 +3322,11 @@ function generateHtmlReport(result) {
     </table>
     ` : ""}
 
-    ${(result.pagesReviewed || []).length > 0 ? `
-    <h2 class="section-title">Pages Reviewed (${(result.pagesReviewed || []).length})</h2>
+    ${pagesReviewed.length > 0 ? `
+    <h2 class="section-title">Pages Reviewed (${pagesReviewed.length})</h2>
+    ${avgPageScore != null ? `<div class="summary-box"><div class="summary-stat"><div class="num" style="color:${scoreColor(avgPageScore)}">${avgPageScore}</div><div class="label">Avg Page AEO Score</div></div></div>` : ""}
     <table>
-      <thead><tr><th>URL</th><th>Category</th><th>Words</th><th>Issues</th><th>Strengths</th></tr></thead>
+      <thead><tr><th>URL</th><th>Category</th><th>Words</th><th>AEO Score</th><th>Issues</th><th>Strengths</th></tr></thead>
       <tbody>${pagesRows}</tbody>
     </table>
     ` : ""}
@@ -3111,7 +3451,7 @@ function generateComparisonHtmlReport(result) {
 }
 
 // src/cli.ts
-var VERSION = "1.4.0";
+var VERSION = "1.5.0";
 function printHelp() {
   console.log(`
   aeorank - AI Engine Optimization audit
@@ -3221,7 +3561,27 @@ function printSummary(result) {
       const cat = page.category.charAt(0).toUpperCase() + page.category.slice(1);
       const issueCount = page.issues.length;
       const issueLabel = issueCount === 0 ? "0 issues" : issueCount === 1 ? "1 issue" : `${issueCount} issues`;
-      log(`    ${cat.padEnd(10)} ${page.url.padEnd(50)} ${issueLabel}`);
+      const aeoLabel = page.aeoScore != null ? ` [AEO: ${page.aeoScore}]` : "";
+      log(`    ${cat.padEnd(10)} ${page.url.padEnd(50)} ${issueLabel}${aeoLabel}`);
+    }
+    const scored = result.pagesReviewed.filter((p) => p.aeoScore != null);
+    if (scored.length > 0) {
+      const avg = Math.round(scored.reduce((sum, p) => sum + p.aeoScore, 0) / scored.length);
+      const sorted = [...scored].sort((a, b) => b.aeoScore - a.aeoScore);
+      const top = sorted[0];
+      const bottom = sorted[sorted.length - 1];
+      log("");
+      log(`    Average page AEO score: ${avg}/100`);
+      try {
+        log(`    Top:    ${new URL(top.url).pathname} (${top.aeoScore})`);
+      } catch {
+        log(`    Top:    ${top.url} (${top.aeoScore})`);
+      }
+      try {
+        log(`    Bottom: ${new URL(bottom.url).pathname} (${bottom.aeoScore})`);
+      } catch {
+        log(`    Bottom: ${bottom.url} (${bottom.aeoScore})`);
+      }
     }
     log("");
   }