@rankcli/agent-runtime 0.0.1

package/src/audit/checks/ai-content-structure.ts

@@ -0,0 +1,325 @@
+/**
+ * AI Content Structure Checks
+ *
+ * AI systems prefer well-structured content that's easy to parse and quote.
+ * These checks verify content is formatted optimally for AI consumption:
+ * - Tables for comparisons/data
+ * - Numbered lists for steps/procedures
+ * - Bullet points for features/benefits
+ * - Clear Q&A format for direct answers
+ * - Concise, quotable statements
+ */
+
+import * as cheerio from 'cheerio';
+import type { AuditIssue } from '../types.js';
+
+export interface AIContentStructureData {
+  tables: {
+    count: number;
+    hasComparisonTable: boolean;
+    hasDataTable: boolean;
+  };
+  lists: {
+    orderedLists: number;
+    unorderedLists: number;
+    hasStepByStep: boolean;
+    hasBulletedFeatures: boolean;
+  };
+  qaFormat: {
+    hasExplicitQA: boolean;
+    questionCount: number;
+    hasDirectAnswers: boolean;
+  };
+  quotability: {
+    hasDefinitions: boolean;
+    hasConciseStatements: boolean;
+    shortParagraphRatio: number;
+  };
+  structureScore: number;
+}
+
+export function analyzeAIContentStructure(
+  html: string,
+  url: string
+): { issues: AuditIssue[]; data: AIContentStructureData } {
+  const issues: AuditIssue[] = [];
+  const $ = cheerio.load(html);
+
+  // Remove nav, footer, aside, scripts, styles for content analysis
+  $('nav, footer, aside, script, style, noscript, header').remove();
+
+  // Analyze tables
+  const tables = $('table');
+  const tableCount = tables.length;
+  let hasComparisonTable = false;
+  let hasDataTable = false;
+
+  tables.each((_, table) => {
+    const $table = $(table);
+    const headers = $table.find('th').length;
+    const rows = $table.find('tr').length;
+    const cells = $table.find('td').length;
+
+    // Comparison table: multiple columns with headers
+    if (headers >= 2 && rows >= 3) {
+      hasComparisonTable = true;
+    }
+
+    // Data table: has numeric data
+    const cellTexts = $table.find('td').map((_, td) => $(td).text()).get();
+    const hasNumbers = cellTexts.some(text => /\d+/.test(text));
+    if (hasNumbers && cells >= 4) {
+      hasDataTable = true;
+    }
+  });
+
+  // Analyze lists
+  const orderedLists = $('ol').length;
+  const unorderedLists = $('ul').not('nav ul').length;
+
+  // Check for step-by-step content
+  let hasStepByStep = false;
+  $('ol').each((_, ol) => {
+    const items = $(ol).find('> li').length;
+    if (items >= 3) {
+      hasStepByStep = true;
+    }
+  });
+
+  // Check for bulleted features/benefits
+  let hasBulletedFeatures = false;
+  $('ul').not('nav ul').each((_, ul) => {
+    const items = $(ul).find('> li').length;
+    if (items >= 3) {
+      hasBulletedFeatures = true;
+    }
+  });
+
+  // Analyze Q&A format
+  const bodyText = $('body').text().toLowerCase();
+  const headings = $('h1, h2, h3, h4, h5, h6');
+  let questionCount = 0;
+  let hasDirectAnswers = false;
+
+  headings.each((_, heading) => {
+    const text = $(heading).text();
+    // Check if heading is a question
+    if (text.includes('?') || /^(what|how|why|when|where|who|which|can|does|is|are|should|will)\s/i.test(text)) {
+      questionCount++;
+
+      // Check if followed by a direct answer (short paragraph)
+      const nextP = $(heading).next('p');
+      if (nextP.length) {
+        const answerText = nextP.text();
+        if (answerText.length > 20 && answerText.length < 300) {
+          hasDirectAnswers = true;
+        }
+      }
+    }
+  });
+
+  // Check for explicit Q&A format (FAQ style)
+  const hasExplicitQA =
+    $('[itemtype*="FAQPage"]').length > 0 ||
+    $('[itemtype*="Question"]').length > 0 ||
+    $('.faq, #faq, [class*="faq"], [id*="faq"]').length > 0 ||
+    $('details summary').length >= 2 ||
+    $('dt').length >= 2; // Definition lists often used for Q&A
+
+  // Analyze quotability
+  const paragraphs = $('p');
+  let shortParagraphCount = 0;
+  let totalParagraphs = 0;
+  let hasDefinitions = false;
+  let hasConciseStatements = false;
+
+  paragraphs.each((_, p) => {
+    const text = $(p).text().trim();
+    if (text.length < 20) return; // Skip very short paragraphs
+
+    totalParagraphs++;
+
+    // Short paragraphs (under 150 chars) are more quotable
+    if (text.length < 150) {
+      shortParagraphCount++;
+    }
+
+    // Check for definition patterns ("X is a...", "X refers to...")
+    if (/^[A-Z][^.]+\s+(is|are|refers to|means|describes)\s+/i.test(text)) {
+      hasDefinitions = true;
+    }
+
+    // Concise statements with clear structure
+    if (text.length < 200 && (
+      text.includes(':') ||
+      /^(The|A|An)\s+\w+/.test(text) ||
+      /^\d+\.?\s+/.test(text)
+    )) {
+      hasConciseStatements = true;
+    }
+  });
+
+  const shortParagraphRatio = totalParagraphs > 0
+    ? shortParagraphCount / totalParagraphs
+    : 0;
+
+  // Calculate structure score (0-100)
+  let structureScore = 50; // Base score
+
+  // Tables bonus
+  if (hasComparisonTable) structureScore += 10;
+  if (hasDataTable) structureScore += 5;
+
+  // Lists bonus
+  if (hasStepByStep) structureScore += 10;
+  if (hasBulletedFeatures) structureScore += 5;
+
+  // Q&A bonus
+  if (hasExplicitQA) structureScore += 10;
+  if (questionCount >= 3) structureScore += 5;
+  if (hasDirectAnswers) structureScore += 5;
+
+  // Quotability bonus
+  if (hasDefinitions) structureScore += 5;
+  if (hasConciseStatements) structureScore += 5;
+  if (shortParagraphRatio > 0.3) structureScore += 5;
+
+  // Penalties
+  if (tableCount === 0 && orderedLists === 0 && unorderedLists === 0) {
+    structureScore -= 15; // No structured content at all
+  }
+  if (questionCount === 0) {
+    structureScore -= 5; // No question-based headings
+  }
+
+  structureScore = Math.max(0, Math.min(100, structureScore));
+
+  // Generate issues
+
+  // No tables for comparisons
+  if (!hasComparisonTable && bodyText.includes('compar') || bodyText.includes('vs') || bodyText.includes('versus')) {
+    issues.push({
+      code: 'AI_NO_COMPARISON_TABLE',
+      severity: 'notice',
+      category: 'ai-readiness',
+      title: 'Comparison content without table format',
+      description: 'Your content mentions comparisons but doesn\'t use a table format. AI systems can easily parse and quote table data.',
+      impact: 'AI may not accurately extract comparison data, reducing chances of being cited in AI-generated comparisons.',
+      howToFix: 'Convert comparison content to HTML tables with clear headers. Example: Feature | Option A | Option B',
+      affectedUrls: [url],
+    });
+  }
+
+  // No numbered lists for procedural content
+  if (!hasStepByStep && (bodyText.includes('step') || bodyText.includes('how to') || bodyText.includes('guide'))) {
+    issues.push({
+      code: 'AI_NO_NUMBERED_STEPS',
+      severity: 'notice',
+      category: 'ai-readiness',
+      title: 'Procedural content without numbered steps',
+      description: 'Your content appears to be a guide or how-to but doesn\'t use numbered lists. AI prefers numbered lists for step-by-step content.',
+      impact: 'AI may not accurately quote your steps in order, or may skip your content for better-structured alternatives.',
+      howToFix: 'Convert step-by-step instructions to <ol> (ordered list) format with clear, actionable items.',
+      affectedUrls: [url],
+    });
+  }
+
+  // No Q&A format
+  if (!hasExplicitQA && questionCount === 0) {
+    issues.push({
+      code: 'AI_NO_QA_FORMAT',
+      severity: 'notice',
+      category: 'ai-readiness',
+      title: 'No question-and-answer format detected',
+      description: 'Content lacks explicit Q&A structure. AI systems often look for clear question-answer pairs to provide direct responses.',
+      impact: 'Lower chance of being featured in AI direct answers or FAQ-style responses.',
+      howToFix: 'Add FAQ section with common questions as headings (H2/H3) followed by concise answers. Implement FAQ schema markup.',
+      affectedUrls: [url],
+      details: {
+        suggestions: [
+          'Use question words in headings (What, How, Why, etc.)',
+          'Follow questions with short, direct answer paragraphs',
+          'Add FAQ schema markup for structured data',
+          'Consider using <details>/<summary> for expandable Q&A',
+        ],
+      },
+    });
+  }
+
+  // Questions without direct answers
+  if (questionCount >= 2 && !hasDirectAnswers) {
+    issues.push({
+      code: 'AI_QUESTIONS_NO_DIRECT_ANSWERS',
+      severity: 'notice',
+      category: 'ai-readiness',
+      title: 'Questions in headings lack direct answers',
+      description: `Found ${questionCount} question-style headings but answers are too long or not immediately following. AI prefers concise answers right after questions.`,
+      impact: 'AI may struggle to extract clear answers, reducing citation likelihood.',
+      howToFix: 'Start each answer section with a 1-2 sentence direct answer, then expand with details. First sentence should standalone as a complete answer.',
+      affectedUrls: [url],
+    });
+  }
+
+  // No definitions
+  if (!hasDefinitions && (bodyText.includes('what is') || bodyText.includes('definition'))) {
+    issues.push({
+      code: 'AI_NO_DEFINITIONS',
+      severity: 'notice',
+      category: 'ai-readiness',
+      title: 'Missing clear definitions',
+      description: 'Content discusses concepts but lacks Wikipedia-style definitions. AI loves clear "X is a..." statements.',
+      impact: 'Less likely to be cited for definitional queries in AI search.',
+      howToFix: 'Add clear definitions early in content: "[Term] is a [category] that [distinguishing features]." Make first paragraph a complete definition.',
+      affectedUrls: [url],
+    });
+  }
+
+  // Low quotability
+  if (structureScore < 50) {
+    issues.push({
+      code: 'AI_LOW_STRUCTURE_SCORE',
+      severity: 'warning',
+      category: 'ai-readiness',
+      title: 'Content structure not optimized for AI parsing',
+      description: `AI structure score: ${structureScore}/100. Content lacks structured elements that AI can easily parse and quote.`,
+      impact: 'Lower likelihood of being cited in AI-generated answers due to poor content structure.',
+      howToFix: 'Improve content structure: add tables for data, numbered lists for steps, bullet points for features, Q&A sections for common questions.',
+      affectedUrls: [url],
+      details: {
+        structureScore,
+        hasTable: tableCount > 0,
+        hasOrderedList: orderedLists > 0,
+        hasUnorderedList: unorderedLists > 0,
+        hasQAFormat: hasExplicitQA || questionCount > 0,
+      },
+    });
+  }
+
+  return {
+    issues,
+    data: {
+      tables: {
+        count: tableCount,
+        hasComparisonTable,
+        hasDataTable,
+      },
+      lists: {
+        orderedLists,
+        unorderedLists,
+        hasStepByStep,
+        hasBulletedFeatures,
+      },
+      qaFormat: {
+        hasExplicitQA,
+        questionCount,
+        hasDirectAnswers,
+      },
+      quotability: {
+        hasDefinitions,
+        hasConciseStatements,
+        shortParagraphRatio,
+      },
+      structureScore,
+    },
+  };
+}

package/src/audit/checks/ai-readiness.ts

@@ -0,0 +1,339 @@
+// AI/LLM Readiness Checks
+// Checks for llms.txt, AI bot blocking, and JS rendering ratio
+
+import { httpGet } from '../../utils/http.js';
+import * as cheerio from 'cheerio';
+import type { AuditIssue } from '../types.js';
+import { ISSUE_DEFINITIONS } from '../types.js';
+
+// Known AI bot user agents
+const AI_BOTS = {
+  GPTBot: 'GPTBot',
+  'ChatGPT-User': 'ChatGPT-User',
+  'OAI-SearchBot': 'OAI-SearchBot',
+  ClaudeBot: 'ClaudeBot',
+  'Claude-Web': 'Claude-Web',
+  'anthropic-ai': 'anthropic-ai',
+  PerplexityBot: 'PerplexityBot',
+  'Google-Extended': 'Google-Extended',
+  Bytespider: 'Bytespider',
+  CCBot: 'CCBot',
+};
+
+export interface LlmsTxtResult {
+  exists: boolean;
+  content?: string;
+  valid: boolean;
+  errors: string[];
+}
+
+export interface AIBotBlockingResult {
+  robotsExists: boolean;
+  blockedBots: string[];
+  allowedBots: string[];
+  allBlocked: boolean;
+}
+
+export interface AIReadinessData {
+  llmsTxt: LlmsTxtResult;
+  botBlocking: AIBotBlockingResult;
+  jsRenderingRatio: number; // 0-100%
+}
+
+/**
+ * Check for llms.txt file
+ * See: https://llmstxt.org/
+ */
+export async function checkLlmsTxt(baseUrl: string): Promise<{ issues: AuditIssue[]; data: LlmsTxtResult }> {
+  const issues: AuditIssue[] = [];
+  const url = new URL('/llms.txt', baseUrl).href;
+
+  try {
+    const response = await httpGet<string>(url, {
+      timeout: 10000,
+      validateStatus: () => true,
+    });
+
+    if (response.status === 404 || response.status >= 400) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.LLMS_TXT_MISSING,
+        affectedUrls: [url],
+      });
+      return {
+        issues,
+        data: { exists: false, valid: false, errors: ['File not found'] },
+      };
+    }
+
+    const content = response.data as string;
+
+    // Basic validation of llms.txt format
+    // Expected format starts with # followed by site name, then markdown content
+    const lines = content.split('\n').filter((line) => line.trim());
+    const errors: string[] = [];
+    let valid = true;
+
+    // Check if it starts with a title (# Site Name)
+    if (lines.length === 0) {
+      errors.push('File is empty');
+      valid = false;
+    } else if (!lines[0].startsWith('#')) {
+      errors.push('File should start with a markdown heading (# Site Name)');
+      valid = false;
+    }
+
+    // Check for common sections
+    const hasDescription = content.toLowerCase().includes('## ') || content.length > 50;
+    if (!hasDescription) {
+      errors.push('File should contain meaningful content describing your site for AI');
+      valid = false;
+    }
+
+    if (!valid) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.LLMS_TXT_INVALID,
+        affectedUrls: [url],
+        details: { errors },
+      });
+    }
+
+    return {
+      issues,
+      data: { exists: true, content, valid, errors },
+    };
+  } catch (error) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.LLMS_TXT_MISSING,
+      affectedUrls: [url],
+      details: { error: error instanceof Error ? error.message : 'Unknown error' },
+    });
+    return {
+      issues,
+      data: { exists: false, valid: false, errors: ['Failed to fetch'] },
+    };
+  }
+}
+
+/**
+ * Check robots.txt for AI bot blocking
+ */
+export async function checkAIBotBlocking(baseUrl: string): Promise<{ issues: AuditIssue[]; data: AIBotBlockingResult }> {
+  const issues: AuditIssue[] = [];
+  const url = new URL('/robots.txt', baseUrl).href;
+
+  const blockedBots: string[] = [];
+  const allowedBots: string[] = [];
+
+  try {
+    const response = await httpGet<string>(url, {
+      timeout: 10000,
+      validateStatus: () => true,
+    });
+
+    if (response.status === 404 || response.status >= 400) {
+      // No robots.txt means all bots are allowed
+      return {
+        issues,
+        data: {
+          robotsExists: false,
+          blockedBots: [],
+          allowedBots: Object.keys(AI_BOTS),
+          allBlocked: false,
+        },
+      };
+    }
+
+    const content = response.data as string;
+    const lines = content.split('\n');
+
+    // Parse robots.txt for AI bot rules
+    let currentUserAgent = '';
+    const botRules: Record<string, { allowed: boolean; disallowAll: boolean }> = {};
+
+    for (const line of lines) {
+      const trimmed = line.trim().toLowerCase();
+
+      if (trimmed.startsWith('user-agent:')) {
+        currentUserAgent = trimmed.split(':')[1].trim();
+      } else if (trimmed.startsWith('disallow:')) {
+        const path = trimmed.split(':')[1]?.trim() || '';
+
+        // Check if this user agent matches any AI bot
+        for (const [botKey, botName] of Object.entries(AI_BOTS)) {
+          if (currentUserAgent === '*' || currentUserAgent === botKey.toLowerCase() || currentUserAgent === botName.toLowerCase()) {
+            if (path === '/' || path === '/*') {
+              if (!botRules[botKey]) {
+                botRules[botKey] = { allowed: false, disallowAll: false };
+              }
+              if (currentUserAgent !== '*' || (currentUserAgent === '*' && !botRules[botKey].allowed)) {
+                botRules[botKey].disallowAll = true;
+              }
+            }
+          }
+        }
+      } else if (trimmed.startsWith('allow:')) {
+        // Explicit allow for a bot
+        for (const [botKey] of Object.entries(AI_BOTS)) {
+          if (currentUserAgent === botKey.toLowerCase()) {
+            if (!botRules[botKey]) {
+              botRules[botKey] = { allowed: true, disallowAll: false };
+            }
+            botRules[botKey].allowed = true;
+          }
+        }
+      }
+    }
+
+    // Determine blocked/allowed bots
+    for (const [botKey] of Object.entries(AI_BOTS)) {
+      const rules = botRules[botKey];
+      if (rules?.disallowAll && !rules?.allowed) {
+        blockedBots.push(botKey);
+      } else {
+        allowedBots.push(botKey);
+      }
+    }
+
+    // Generate issues for blocked bots
+    if (blockedBots.length === Object.keys(AI_BOTS).length) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.AI_BOT_BLOCKED_ALL,
+        affectedUrls: [url],
+        details: { blockedBots },
+      });
+    } else {
+      // Individual bot blocking notices
+      if (blockedBots.includes('GPTBot') || blockedBots.includes('ChatGPT-User')) {
+        issues.push({
+          ...ISSUE_DEFINITIONS.AI_BOT_GPTBOT_BLOCKED,
+          affectedUrls: [url],
+        });
+      }
+      if (blockedBots.includes('ClaudeBot') || blockedBots.includes('Claude-Web') || blockedBots.includes('anthropic-ai')) {
+        issues.push({
+          ...ISSUE_DEFINITIONS.AI_BOT_CLAUDEBOT_BLOCKED,
+          affectedUrls: [url],
+        });
+      }
+      if (blockedBots.includes('PerplexityBot')) {
+        issues.push({
+          ...ISSUE_DEFINITIONS.AI_BOT_PERPLEXITY_BLOCKED,
+          affectedUrls: [url],
+        });
+      }
+      if (blockedBots.includes('Google-Extended')) {
+        issues.push({
+          ...ISSUE_DEFINITIONS.AI_BOT_GOOGLE_EXTENDED_BLOCKED,
+          affectedUrls: [url],
+        });
+      }
+    }
+
+    return {
+      issues,
+      data: {
+        robotsExists: true,
+        blockedBots,
+        allowedBots,
+        allBlocked: blockedBots.length === Object.keys(AI_BOTS).length,
+      },
+    };
+  } catch (error) {
+    return {
+      issues,
+      data: {
+        robotsExists: false,
+        blockedBots: [],
+        allowedBots: Object.keys(AI_BOTS),
+        allBlocked: false,
+      },
+    };
+  }
+}
+
+/**
+ * Check JavaScript rendering ratio
+ * Compares static HTML content size vs rendered content
+ */
+export function checkJSRenderingRatio(
+  html: string,
+  url: string
+): { issues: AuditIssue[]; data: { ratio: number; staticWordCount: number } } {
+  const issues: AuditIssue[] = [];
+  const $ = cheerio.load(html);
+
+  // Get static text content (what AI crawlers would see without JS)
+  // Remove script and style tags
+  $('script, style, noscript').remove();
+
+  const staticText = $('body').text().replace(/\s+/g, ' ').trim();
+  const staticWordCount = staticText.split(/\s+/).filter((word) => word.length > 0).length;
+
+  // Check for signs of heavy JS rendering
+  const hasReactRoot = $('#root, #app, #__next, [data-reactroot]').length > 0;
+  const hasVueApp = $('#app[data-v-app], [data-v-]').length > 0;
+  const hasAngularApp = $('[ng-app], [data-ng-app]').length > 0;
+  const hasEmptyBody = staticWordCount < 50;
+
+  // Estimate JS rendering ratio based on signals
+  let ratio = 0;
+  if (hasEmptyBody && (hasReactRoot || hasVueApp || hasAngularApp)) {
+    ratio = 90; // Likely SPA with most content rendered by JS
+  } else if (hasReactRoot || hasVueApp || hasAngularApp) {
+    ratio = 50; // Has JS framework but some static content
+  } else if (staticWordCount < 100) {
+    ratio = 60; // Very little static content
+  } else {
+    ratio = 10; // Mostly static content
+  }
+
+  // Check for noscript fallback
+  const hasNoScript = $('noscript').length > 0;
+  if (hasNoScript && ratio > 50) {
+    ratio -= 20; // Has fallback content
+  }
+
+  if (ratio > 50) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.HIGH_JS_RENDERING_RATIO,
+      affectedUrls: [url],
+      details: { ratio: `${ratio}%`, staticWordCount },
+    });
+  }
+
+  return {
+    issues,
+    data: { ratio, staticWordCount },
+  };
+}
+
+/**
+ * Run all AI readiness checks
+ */
+export async function runAIReadinessChecks(
+  baseUrl: string,
+  html: string
+): Promise<{ issues: AuditIssue[]; data: AIReadinessData }> {
+  const allIssues: AuditIssue[] = [];
+
+  // Check llms.txt
+  const llmsResult = await checkLlmsTxt(baseUrl);
+  allIssues.push(...llmsResult.issues);
+
+  // Check AI bot blocking
+  const botResult = await checkAIBotBlocking(baseUrl);
+  allIssues.push(...botResult.issues);
+
+  // Check JS rendering ratio
+  const jsResult = checkJSRenderingRatio(html, baseUrl);
+  allIssues.push(...jsResult.issues);
+
+  return {
+    issues: allIssues,
+    data: {
+      llmsTxt: llmsResult.data,
+      botBlocking: botResult.data,
+      jsRenderingRatio: jsResult.data.ratio,
+    },
+  };
+}