@rankcli/agent-runtime 0.0.9 → 0.0.11

package/dist/chunk-3ZSCLNTW.mjs

@@ -0,0 +1,557 @@
+// src/analyzers/geo-analyzer.ts
+import * as cheerio from "cheerio";
+var AI_CRAWLERS = {
+  // OpenAI
+  GPTBot: { userAgent: "GPTBot", company: "OpenAI", purpose: "ChatGPT training & browsing" },
+  ChatGPTUser: { userAgent: "ChatGPT-User", company: "OpenAI", purpose: "ChatGPT browsing feature" },
+  OAI_SearchBot: { userAgent: "OAI-SearchBot", company: "OpenAI", purpose: "OpenAI search features" },
+  // Anthropic
+  ClaudeBot: { userAgent: "Claude-Web", company: "Anthropic", purpose: "Claude web access" },
+  anthropic_ai: { userAgent: "anthropic-ai", company: "Anthropic", purpose: "Claude training" },
+  // Perplexity
+  PerplexityBot: { userAgent: "PerplexityBot", company: "Perplexity", purpose: "Perplexity AI search" },
+  // Google AI
+  GoogleOther: { userAgent: "Google-Extended", company: "Google", purpose: "Bard/Gemini training" },
+  // Microsoft/Bing
+  Bingbot: { userAgent: "bingbot", company: "Microsoft", purpose: "Bing Chat/Copilot" },
+  // Meta
+  FacebookBot: { userAgent: "FacebookBot", company: "Meta", purpose: "Meta AI features" },
+  MetaAI: { userAgent: "meta-externalagent", company: "Meta", purpose: "Meta AI training" },
+  // Apple
+  Applebot: { userAgent: "Applebot-Extended", company: "Apple", purpose: "Apple Intelligence" },
+  // Amazon
+  Amazonbot: { userAgent: "Amazonbot", company: "Amazon", purpose: "Alexa/Amazon AI" },
+  // Others
+  YouBot: { userAgent: "YouBot", company: "You.com", purpose: "You.com AI search" },
+  CCBot: { userAgent: "CCBot", company: "Common Crawl", purpose: "Training data" },
+  cohere_ai: { userAgent: "cohere-ai", company: "Cohere", purpose: "Cohere training" }
+};
+function analyzeRobotsTxtForAI(robotsTxt) {
+  const allowed = [];
+  const blocked = [];
+  const recommendations = [];
+  const lines = robotsTxt.split("\n").map((l) => l.trim().toLowerCase());
+  let currentAgent = "*";
+  for (const line of lines) {
+    if (line.startsWith("user-agent:")) {
+      currentAgent = line.replace("user-agent:", "").trim();
+    } else if (line.startsWith("disallow:")) {
+      const path = line.replace("disallow:", "").trim();
+      if (path === "/" || path === "/*") {
+        for (const [name, info] of Object.entries(AI_CRAWLERS)) {
+          if (currentAgent === "*" || currentAgent.includes(info.userAgent.toLowerCase())) {
+            if (!blocked.includes(name)) blocked.push(name);
+          }
+        }
+      }
+    } else if (line.startsWith("allow:")) {
+      for (const [name, info] of Object.entries(AI_CRAWLERS)) {
+        if (currentAgent === "*" || currentAgent.includes(info.userAgent.toLowerCase())) {
+          if (!allowed.includes(name)) allowed.push(name);
+        }
+      }
+    }
+  }
+  for (const [name, info] of Object.entries(AI_CRAWLERS)) {
+    const hasExplicitRule = lines.some(
+      (l) => l.includes(info.userAgent.toLowerCase())
+    );
+    if (!hasExplicitRule && !blocked.includes(name) && !allowed.includes(name)) {
+      allowed.push(name);
+    }
+  }
+  if (blocked.includes("GPTBot")) {
+    recommendations.push("\u26A0\uFE0F GPTBot is blocked - your content won't appear in ChatGPT responses");
+  }
+  if (blocked.includes("PerplexityBot")) {
+    recommendations.push("\u26A0\uFE0F PerplexityBot is blocked - missing Perplexity AI search visibility");
+  }
+  if (blocked.includes("ClaudeBot")) {
+    recommendations.push("\u26A0\uFE0F Claude-Web is blocked - no visibility in Claude AI responses");
+  }
+  if (allowed.length === 0) {
+    recommendations.push("\u{1F6A8} All AI crawlers appear to be blocked - zero AI search visibility");
+  }
+  return { allowed, blocked, recommendations };
+}
+function detectRenderingMode(html) {
+  const $ = cheerio.load(html);
+  const signals = [];
+  const hasNextData = html.includes("__NEXT_DATA__");
+  const hasNuxtData = html.includes("__NUXT__");
+  const hasGatsbyData = html.includes("___gatsby");
+  const hasAstroIsland = html.includes("astro-island");
+  const bodyText = $("body").text().replace(/\s+/g, " ").trim();
+  const hasMinimalContent = bodyText.length < 200;
+  const hasRootDiv = $("#root, #app, #__next, #__nuxt").length > 0;
+  const hasReactRoot = $("[data-reactroot]").length > 0;
+  const hasPrerender = $('meta[name="prerender-status-code"]').length > 0 || $('meta[name="fragment"]').attr("content") === "!";
+  const paragraphs = $("p").length;
+  const headings = $("h1, h2, h3, h4, h5, h6").length;
+  const hasSubstantialContent = paragraphs > 2 || headings > 2;
+  let serverSideRendered = false;
+  let jsRenderingRequired = false;
+  if (hasNextData || hasNuxtData || hasGatsbyData || hasAstroIsland) {
+    serverSideRendered = true;
+    signals.push("Meta-framework SSR detected");
+  }
+  if (hasMinimalContent && hasRootDiv) {
+    jsRenderingRequired = true;
+    signals.push("SPA with minimal server content detected");
+  }
+  if (hasSubstantialContent) {
+    serverSideRendered = true;
+    signals.push("Substantial HTML content present");
+  }
+  if (hasReactRoot && !hasNextData && hasMinimalContent) {
+    jsRenderingRequired = true;
+    signals.push("Client-side React app detected");
+  }
+  return {
+    serverSideRendered,
+    jsRenderingRequired,
+    hasPrerendering: hasPrerender,
+    contentInHTML: hasSubstantialContent,
+    signals
+  };
+}
+function analyzeContentStructure(html) {
+  const $ = cheerio.load(html);
+  const jsonLdScripts = $('script[type="application/ld+json"]');
+  let hasStructuredData = jsonLdScripts.length > 0;
+  let hasFAQSchema = false;
+  let hasHowToSchema = false;
+  let hasArticleSchema = false;
+  let hasProductSchema = false;
+  jsonLdScripts.each((_, el) => {
+    try {
+      const content = $(el).html() || "";
+      if (content.includes('"FAQPage"') || content.includes('"@type":"FAQPage"')) {
+        hasFAQSchema = true;
+      }
+      if (content.includes('"HowTo"') || content.includes('"@type":"HowTo"')) {
+        hasHowToSchema = true;
+      }
+      if (content.includes('"Article"') || content.includes('"BlogPosting"') || content.includes('"NewsArticle"')) {
+        hasArticleSchema = true;
+      }
+      if (content.includes('"Product"')) {
+        hasProductSchema = true;
+      }
+    } catch {
+    }
+  });
+  const hasBreadcrumbs = $('[itemtype*="BreadcrumbList"], .breadcrumb, .breadcrumbs, nav[aria-label*="breadcrumb"]').length > 0 || html.includes('"BreadcrumbList"');
+  const h1Count = $("h1").length;
+  const h2Count = $("h2").length;
+  const h3Count = $("h3").length;
+  let headingHierarchy = "poor";
+  if (h1Count === 1 && h2Count >= 2 && h3Count >= 0) {
+    headingHierarchy = "good";
+  } else if (h1Count === 1 && h2Count >= 1) {
+    headingHierarchy = "needs-work";
+  }
+  const contentSections = $('section, article, .section, [class*="section"]').length;
+  const listsAndTables = $("ul, ol, table, dl").length;
+  return {
+    hasStructuredData,
+    hasFAQSchema,
+    hasHowToSchema,
+    hasArticleSchema,
+    hasProductSchema,
+    hasBreadcrumbs,
+    headingHierarchy,
+    contentSections,
+    listsAndTables
+  };
+}
+function analyzeCitationReadiness(html) {
+  const $ = cheerio.load(html);
+  const bodyText = $("body").text();
+  const hasCitations = $('cite, blockquote, [class*="citation"], [class*="reference"]').length > 0 || /\[\d+\]|\(\d{4}\)/.test(bodyText);
+  const externalLinks = $('a[href^="http"]').filter((_, el) => {
+    const href = $(el).attr("href") || "";
+    return !href.includes(new URL(href).hostname);
+  });
+  const hasExternalLinks = externalLinks.length > 0;
+  const hasStatistics = /\d+%|\d+\s*(million|billion|thousand)|increased by \d+|decreased by \d+/i.test(bodyText);
+  const hasQuotes = $("blockquote, q").length > 0 || /"[^"]{20,}"/.test(bodyText);
+  const hasAuthorInfo = $('[rel="author"], .author, [class*="author"], [itemtype*="Person"]').length > 0 || $('meta[name="author"]').length > 0;
+  const hasPublishDate = $('time[datetime], meta[property="article:published_time"], .publish-date, .date').length > 0;
+  const hasLastModified = $('meta[property="article:modified_time"], .updated, .modified').length > 0;
+  const trustSignals = [];
+  if (hasCitations) trustSignals.push("Citations present");
+  if (hasExternalLinks) trustSignals.push("External source links");
+  if (hasStatistics) trustSignals.push("Statistical data");
+  if (hasQuotes) trustSignals.push("Expert quotes");
+  if (hasAuthorInfo) trustSignals.push("Author attribution");
+  if (hasPublishDate) trustSignals.push("Publish date");
+  if (hasLastModified) trustSignals.push("Last modified date");
+  return {
+    hasCitations,
+    hasExternalLinks,
+    hasStatistics,
+    hasQuotes,
+    hasAuthorInfo,
+    hasPublishDate,
+    hasLastModified,
+    trustSignals
+  };
+}
+function analyzeEntityExtraction(html) {
+  const $ = cheerio.load(html);
+  const bodyText = $("body").text();
+  const definedTerms = [];
+  $("dfn, abbr[title]").each((_, el) => {
+    const term = $(el).text().trim();
+    if (term) definedTerms.push(term);
+  });
+  const hasDefinitions = /is defined as|refers to|means that|is the process of/i.test(bodyText) || $('dfn, .definition, [class*="definition"]').length > 0;
+  const hasComparisons = /compared to|versus|vs\.|unlike|similar to|difference between/i.test(bodyText) || $('table[class*="comparison"], .comparison, [class*="versus"]').length > 0;
+  let questionAnswerPairs = 0;
+  $('details, .faq-item, [class*="question"], [class*="accordion"]').each(() => {
+    questionAnswerPairs++;
+  });
+  $("h2, h3, h4").each((_, el) => {
+    const text = $(el).text();
+    if (/^(what|how|why|when|where|who|which|can|does|is|are|should|will)\s/i.test(text) || text.endsWith("?")) {
+      questionAnswerPairs++;
+    }
+  });
+  const namedEntities = [];
+  $("strong, b").each((_, el) => {
+    const text = $(el).text().trim();
+    if (text.length > 2 && text.length < 50 && /^[A-Z]/.test(text)) {
+      if (!namedEntities.includes(text)) namedEntities.push(text);
+    }
+  });
+  return {
+    namedEntities: namedEntities.slice(0, 20),
+    // Limit to top 20
+    definedTerms,
+    hasDefinitions,
+    hasComparisons,
+    questionAnswerPairs
+  };
+}
+function calculateLLMSignals(structure, citation, entity, html) {
+  const $ = cheerio.load(html);
+  let contentClarity = 0;
+  if (structure.headingHierarchy === "good") contentClarity += 30;
+  else if (structure.headingHierarchy === "needs-work") contentClarity += 15;
+  contentClarity += Math.min(structure.listsAndTables * 5, 25);
+  contentClarity += Math.min(structure.contentSections * 5, 20);
+  if (structure.hasBreadcrumbs) contentClarity += 10;
+  contentClarity += entity.hasDefinitions ? 15 : 0;
+  contentClarity = Math.min(contentClarity, 100);
+  let factDensity = 0;
+  if (citation.hasStatistics) factDensity += 25;
+  if (citation.hasCitations) factDensity += 20;
+  if (citation.hasExternalLinks) factDensity += 15;
+  if (citation.hasQuotes) factDensity += 15;
+  factDensity += Math.min(entity.questionAnswerPairs * 5, 25);
+  factDensity = Math.min(factDensity, 100);
+  let structureQuality = 0;
+  if (structure.hasStructuredData) structureQuality += 25;
+  if (structure.hasFAQSchema) structureQuality += 20;
+  if (structure.hasArticleSchema) structureQuality += 15;
+  if (structure.hasHowToSchema) structureQuality += 15;
+  if (structure.hasProductSchema) structureQuality += 10;
+  structureQuality += structure.headingHierarchy === "good" ? 15 : 0;
+  structureQuality = Math.min(structureQuality, 100);
+  let citationQuality = 0;
+  citationQuality += citation.trustSignals.length * 12;
+  if (citation.hasAuthorInfo) citationQuality += 15;
+  if (citation.hasPublishDate && citation.hasLastModified) citationQuality += 15;
+  citationQuality = Math.min(citationQuality, 100);
+  const overallLLMFriendliness = Math.round(
+    contentClarity * 0.25 + factDensity * 0.25 + structureQuality * 0.3 + citationQuality * 0.2
+  );
+  return {
+    contentClarity,
+    factDensity,
+    structureQuality,
+    citationQuality,
+    overallLLMFriendliness
+  };
+}
+function generateGEOIssues(aiAccess, structure, citation, entity, llmSignals, url) {
+  const issues = [];
+  if (aiAccess.blockedCrawlers.length > 0) {
+    issues.push({
+      code: "GEO_AI_CRAWLERS_BLOCKED",
+      severity: "critical",
+      category: "technical",
+      title: `${aiAccess.blockedCrawlers.length} AI crawlers blocked in robots.txt`,
+      description: `The following AI crawlers are blocked: ${aiAccess.blockedCrawlers.join(", ")}. This prevents your content from appearing in AI search results.`,
+      impact: "Your content will NOT appear in ChatGPT, Perplexity, or Claude responses",
+      howToFix: `Update robots.txt to allow AI crawlers:
+
+User-agent: GPTBot
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Claude-Web
+Allow: /`,
+      affectedUrls: [url]
+    });
+  }
+  if (aiAccess.jsRenderingRequired && !aiAccess.hasPrerendering) {
+    issues.push({
+      code: "GEO_JS_RENDERING_REQUIRED",
+      severity: "critical",
+      category: "technical",
+      title: "JavaScript rendering required - AI crawlers see blank page",
+      description: "Your site requires JavaScript to render content. Most AI crawlers (GPTBot, PerplexityBot) do not execute JavaScript and will see a blank page.",
+      impact: "Zero visibility in AI search - crawlers cannot access your content",
+      howToFix: "Implement Server-Side Rendering (SSR) or Static Site Generation (SSG). For React: use Next.js. For Vue: use Nuxt. Alternatively, implement a prerendering service.",
+      affectedUrls: [url]
+    });
+  }
+  if (!structure.hasStructuredData) {
+    issues.push({
+      code: "GEO_NO_STRUCTURED_DATA",
+      severity: "warning",
+      category: "technical",
+      title: "No JSON-LD structured data found",
+      description: "Structured data helps AI systems understand your content's context, entities, and relationships.",
+      impact: "AI systems may misunderstand or skip your content",
+      howToFix: "Add JSON-LD structured data. At minimum, include Organization, WebPage, and Article/Product schemas.",
+      affectedUrls: [url]
+    });
+  }
+  if (!structure.hasFAQSchema && entity.questionAnswerPairs > 0) {
+    issues.push({
+      code: "GEO_MISSING_FAQ_SCHEMA",
+      severity: "warning",
+      category: "technical",
+      title: "Q&A content without FAQPage schema",
+      description: `Found ${entity.questionAnswerPairs} question-answer pairs but no FAQPage schema. FAQ schema is excellent for AI search visibility.`,
+      impact: "Missing opportunity for featured snippets and AI citations",
+      howToFix: "Add FAQPage schema markup for your Q&A content. This dramatically increases chances of being cited by AI.",
+      affectedUrls: [url]
+    });
+  }
+  if (structure.headingHierarchy === "poor") {
+    issues.push({
+      code: "GEO_POOR_HEADING_STRUCTURE",
+      severity: "warning",
+      category: "content",
+      title: "Poor heading hierarchy hurts AI understanding",
+      description: "AI systems use heading structure to understand content organization. Your page lacks a clear H1 \u2192 H2 \u2192 H3 hierarchy.",
+      impact: "AI may struggle to extract key topics and relationships",
+      howToFix: "Structure content with one H1 (main topic), multiple H2s (subtopics), and H3s (details). Use question-format headings where appropriate.",
+      affectedUrls: [url]
+    });
+  }
+  if (!citation.hasCitations && !citation.hasExternalLinks) {
+    issues.push({
+      code: "GEO_NO_CITATIONS",
+      severity: "warning",
+      category: "content",
+      title: "No citations or source references",
+      description: "AI systems prioritize content with verifiable sources and citations. Your page lacks external references.",
+      impact: "Lower trust score in AI ranking - may not be cited as authoritative",
+      howToFix: "Add citations, link to authoritative sources, include statistics with references, and add expert quotes.",
+      affectedUrls: [url]
+    });
+  }
+  if (!citation.hasAuthorInfo) {
+    issues.push({
+      code: "GEO_NO_AUTHOR_INFO",
+      severity: "info",
+      category: "content",
+      title: "Missing author attribution",
+      description: "Author information helps establish E-E-A-T (Experience, Expertise, Authoritativeness, Trustworthiness) for AI systems.",
+      impact: "Lower credibility score in AI ranking algorithms",
+      howToFix: "Add author byline, link to author bio/profile, and consider adding author schema markup.",
+      affectedUrls: [url]
+    });
+  }
+  if (!citation.hasPublishDate || !citation.hasLastModified) {
+    issues.push({
+      code: "GEO_NO_DATES",
+      severity: "info",
+      category: "content",
+      title: "Missing publish/update dates",
+      description: "AI systems consider content freshness. Pages without dates may be considered outdated.",
+      impact: "May be deprioritized for time-sensitive queries",
+      howToFix: 'Add visible publish date and "last updated" date. Include datePublished and dateModified in schema.',
+      affectedUrls: [url]
+    });
+  }
+  if (llmSignals.overallLLMFriendliness < 40) {
+    issues.push({
+      code: "GEO_LOW_LLM_SCORE",
+      severity: "warning",
+      category: "content",
+      title: `Low AI-friendliness score: ${llmSignals.overallLLMFriendliness}/100`,
+      description: "Your content structure and signals are not optimized for AI consumption. AI systems may struggle to extract and cite your content.",
+      impact: "Significantly reduced chances of appearing in AI search results",
+      howToFix: "Improve content structure (clear headings, lists, tables), add citations and statistics, include FAQ sections, and add comprehensive schema markup.",
+      affectedUrls: [url]
+    });
+  }
+  return issues;
+}
+async function analyzeGEO(html, url, robotsTxt) {
+  let aiCrawlerAccess = {
+    robotsTxtAllowsAI: true,
+    blockedCrawlers: [],
+    allowedCrawlers: Object.keys(AI_CRAWLERS),
+    hasPrerendering: false,
+    jsRenderingRequired: false,
+    serverSideRendered: true
+  };
+  if (robotsTxt) {
+    const robotsAnalysis = analyzeRobotsTxtForAI(robotsTxt);
+    aiCrawlerAccess.blockedCrawlers = robotsAnalysis.blocked;
+    aiCrawlerAccess.allowedCrawlers = robotsAnalysis.allowed;
+    aiCrawlerAccess.robotsTxtAllowsAI = robotsAnalysis.blocked.length === 0;
+  }
+  const renderingMode = detectRenderingMode(html);
+  aiCrawlerAccess.hasPrerendering = renderingMode.hasPrerendering;
+  aiCrawlerAccess.jsRenderingRequired = renderingMode.jsRenderingRequired;
+  aiCrawlerAccess.serverSideRendered = renderingMode.serverSideRendered;
+  const contentStructure = analyzeContentStructure(html);
+  const citationReadiness = analyzeCitationReadiness(html);
+  const entityExtraction = analyzeEntityExtraction(html);
+  const llmSignals = calculateLLMSignals(
+    contentStructure,
+    citationReadiness,
+    entityExtraction,
+    html
+  );
+  const issues = generateGEOIssues(
+    aiCrawlerAccess,
+    contentStructure,
+    citationReadiness,
+    entityExtraction,
+    llmSignals,
+    url
+  );
+  const recommendations = [];
+  if (aiCrawlerAccess.blockedCrawlers.length > 0) {
+    recommendations.push("\u{1F6A8} URGENT: Unblock AI crawlers in robots.txt");
+  }
+  if (aiCrawlerAccess.jsRenderingRequired) {
+    recommendations.push("\u{1F6A8} URGENT: Implement SSR/SSG for AI crawler access");
+  }
+  if (!contentStructure.hasFAQSchema && entityExtraction.questionAnswerPairs > 0) {
+    recommendations.push("Add FAQPage schema for your Q&A content");
+  }
+  if (!contentStructure.hasStructuredData) {
+    recommendations.push("Add JSON-LD structured data (Article, Organization)");
+  }
+  if (citationReadiness.trustSignals.length < 3) {
+    recommendations.push("Add more trust signals: citations, statistics, author info");
+  }
+  if (contentStructure.listsAndTables < 2) {
+    recommendations.push("Add lists and tables - AI loves structured content");
+  }
+  if (!entityExtraction.hasDefinitions) {
+    recommendations.push("Add clear definitions for key terms");
+  }
+  if (entityExtraction.questionAnswerPairs < 3) {
+    recommendations.push("Add FAQ section with common questions");
+  }
+  let score = llmSignals.overallLLMFriendliness;
+  if (aiCrawlerAccess.blockedCrawlers.length > 0) {
+    score -= aiCrawlerAccess.blockedCrawlers.length * 10;
+  }
+  if (aiCrawlerAccess.jsRenderingRequired && !aiCrawlerAccess.hasPrerendering) {
+    score -= 30;
+  }
+  if (contentStructure.hasFAQSchema) score += 10;
+  if (contentStructure.hasArticleSchema) score += 5;
+  if (aiCrawlerAccess.robotsTxtAllowsAI) score += 10;
+  score = Math.max(0, Math.min(100, score));
+  return {
+    score: Math.round(score),
+    aiCrawlerAccess,
+    contentStructure,
+    citationReadiness,
+    entityExtraction,
+    llmSignals,
+    issues,
+    recommendations
+  };
+}
+function generateAIFriendlyRobotsTxt(siteUrl) {
+  return `# AI-Optimized robots.txt
+# Generated by RankCLI - https://rankcli.dev
+
+# Allow all standard crawlers
+User-agent: *
+Allow: /
+Disallow: /api/
+Disallow: /admin/
+Disallow: /_next/static/
+Disallow: /private/
+
+# === AI Search Crawlers ===
+# OpenAI (ChatGPT)
+User-agent: GPTBot
+Allow: /
+
+User-agent: ChatGPT-User
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+
+# Anthropic (Claude)
+User-agent: Claude-Web
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+# Perplexity
+User-agent: PerplexityBot
+Allow: /
+
+# Google AI (Gemini/Bard)
+User-agent: Google-Extended
+Allow: /
+
+# Microsoft (Copilot)
+User-agent: bingbot
+Allow: /
+
+# Apple Intelligence
+User-agent: Applebot-Extended
+Allow: /
+
+# Meta AI
+User-agent: meta-externalagent
+Allow: /
+
+# You.com
+User-agent: YouBot
+Allow: /
+
+# Cohere
+User-agent: cohere-ai
+Allow: /
+
+# Sitemap
+Sitemap: ${siteUrl}/sitemap.xml
+`;
+}
+var AI_CRAWLERS_INFO = AI_CRAWLERS;
+
+export {
+  analyzeRobotsTxtForAI,
+  detectRenderingMode,
+  analyzeContentStructure,
+  analyzeCitationReadiness,
+  analyzeEntityExtraction,
+  calculateLLMSignals,
+  generateGEOIssues,
+  analyzeGEO,
+  generateAIFriendlyRobotsTxt,
+  AI_CRAWLERS_INFO
+};