@rankcli/agent-runtime 0.0.14 → 0.0.16

package/src/audit/checks/ai-content-structure.ts

@@ -1,325 +0,0 @@
-/**
- * AI Content Structure Checks
- *
- * AI systems prefer well-structured content that's easy to parse and quote.
- * These checks verify content is formatted optimally for AI consumption:
- * - Tables for comparisons/data
- * - Numbered lists for steps/procedures
- * - Bullet points for features/benefits
- * - Clear Q&A format for direct answers
- * - Concise, quotable statements
- */
-
-import * as cheerio from 'cheerio';
-import type { AuditIssue } from '../types.js';
-
-export interface AIContentStructureData {
-  tables: {
-    count: number;
-    hasComparisonTable: boolean;
-    hasDataTable: boolean;
-  };
-  lists: {
-    orderedLists: number;
-    unorderedLists: number;
-    hasStepByStep: boolean;
-    hasBulletedFeatures: boolean;
-  };
-  qaFormat: {
-    hasExplicitQA: boolean;
-    questionCount: number;
-    hasDirectAnswers: boolean;
-  };
-  quotability: {
-    hasDefinitions: boolean;
-    hasConciseStatements: boolean;
-    shortParagraphRatio: number;
-  };
-  structureScore: number;
-}
-
-export function analyzeAIContentStructure(
-  html: string,
-  url: string
-): { issues: AuditIssue[]; data: AIContentStructureData } {
-  const issues: AuditIssue[] = [];
-  const $ = cheerio.load(html);
-
-  // Remove nav, footer, aside, scripts, styles for content analysis
-  $('nav, footer, aside, script, style, noscript, header').remove();
-
-  // Analyze tables
-  const tables = $('table');
-  const tableCount = tables.length;
-  let hasComparisonTable = false;
-  let hasDataTable = false;
-
-  tables.each((_, table) => {
-    const $table = $(table);
-    const headers = $table.find('th').length;
-    const rows = $table.find('tr').length;
-    const cells = $table.find('td').length;
-
-    // Comparison table: multiple columns with headers
-    if (headers >= 2 && rows >= 3) {
-      hasComparisonTable = true;
-    }
-
-    // Data table: has numeric data
-    const cellTexts = $table.find('td').map((_, td) => $(td).text()).get();
-    const hasNumbers = cellTexts.some(text => /\d+/.test(text));
-    if (hasNumbers && cells >= 4) {
-      hasDataTable = true;
-    }
-  });
-
-  // Analyze lists
-  const orderedLists = $('ol').length;
-  const unorderedLists = $('ul').not('nav ul').length;
-
-  // Check for step-by-step content
-  let hasStepByStep = false;
-  $('ol').each((_, ol) => {
-    const items = $(ol).find('> li').length;
-    if (items >= 3) {
-      hasStepByStep = true;
-    }
-  });
-
-  // Check for bulleted features/benefits
-  let hasBulletedFeatures = false;
-  $('ul').not('nav ul').each((_, ul) => {
-    const items = $(ul).find('> li').length;
-    if (items >= 3) {
-      hasBulletedFeatures = true;
-    }
-  });
-
-  // Analyze Q&A format
-  const bodyText = $('body').text().toLowerCase();
-  const headings = $('h1, h2, h3, h4, h5, h6');
-  let questionCount = 0;
-  let hasDirectAnswers = false;
-
-  headings.each((_, heading) => {
-    const text = $(heading).text();
-    // Check if heading is a question
-    if (text.includes('?') || /^(what|how|why|when|where|who|which|can|does|is|are|should|will)\s/i.test(text)) {
-      questionCount++;
-
-      // Check if followed by a direct answer (short paragraph)
-      const nextP = $(heading).next('p');
-      if (nextP.length) {
-        const answerText = nextP.text();
-        if (answerText.length > 20 && answerText.length < 300) {
-          hasDirectAnswers = true;
-        }
-      }
-    }
-  });
-
-  // Check for explicit Q&A format (FAQ style)
-  const hasExplicitQA =
-    $('[itemtype*="FAQPage"]').length > 0 ||
-    $('[itemtype*="Question"]').length > 0 ||
-    $('.faq, #faq, [class*="faq"], [id*="faq"]').length > 0 ||
-    $('details summary').length >= 2 ||
-    $('dt').length >= 2; // Definition lists often used for Q&A
-
-  // Analyze quotability
-  const paragraphs = $('p');
-  let shortParagraphCount = 0;
-  let totalParagraphs = 0;
-  let hasDefinitions = false;
-  let hasConciseStatements = false;
-
-  paragraphs.each((_, p) => {
-    const text = $(p).text().trim();
-    if (text.length < 20) return; // Skip very short paragraphs
-
-    totalParagraphs++;
-
-    // Short paragraphs (under 150 chars) are more quotable
-    if (text.length < 150) {
-      shortParagraphCount++;
-    }
-
-    // Check for definition patterns ("X is a...", "X refers to...")
-    if (/^[A-Z][^.]+\s+(is|are|refers to|means|describes)\s+/i.test(text)) {
-      hasDefinitions = true;
-    }
-
-    // Concise statements with clear structure
-    if (text.length < 200 && (
-      text.includes(':') ||
-      /^(The|A|An)\s+\w+/.test(text) ||
-      /^\d+\.?\s+/.test(text)
-    )) {
-      hasConciseStatements = true;
-    }
-  });
-
-  const shortParagraphRatio = totalParagraphs > 0
-    ? shortParagraphCount / totalParagraphs
-    : 0;
-
-  // Calculate structure score (0-100)
-  let structureScore = 50; // Base score
-
-  // Tables bonus
-  if (hasComparisonTable) structureScore += 10;
-  if (hasDataTable) structureScore += 5;
-
-  // Lists bonus
-  if (hasStepByStep) structureScore += 10;
-  if (hasBulletedFeatures) structureScore += 5;
-
-  // Q&A bonus
-  if (hasExplicitQA) structureScore += 10;
-  if (questionCount >= 3) structureScore += 5;
-  if (hasDirectAnswers) structureScore += 5;
-
-  // Quotability bonus
-  if (hasDefinitions) structureScore += 5;
-  if (hasConciseStatements) structureScore += 5;
-  if (shortParagraphRatio > 0.3) structureScore += 5;
-
-  // Penalties
-  if (tableCount === 0 && orderedLists === 0 && unorderedLists === 0) {
-    structureScore -= 15; // No structured content at all
-  }
-  if (questionCount === 0) {
-    structureScore -= 5; // No question-based headings
-  }
-
-  structureScore = Math.max(0, Math.min(100, structureScore));
-
-  // Generate issues
-
-  // No tables for comparisons
-  if (!hasComparisonTable && bodyText.includes('compar') || bodyText.includes('vs') || bodyText.includes('versus')) {
-    issues.push({
-      code: 'AI_NO_COMPARISON_TABLE',
-      severity: 'notice',
-      category: 'ai-readiness',
-      title: 'Comparison content without table format',
-      description: 'Your content mentions comparisons but doesn\'t use a table format. AI systems can easily parse and quote table data.',
-      impact: 'AI may not accurately extract comparison data, reducing chances of being cited in AI-generated comparisons.',
-      howToFix: 'Convert comparison content to HTML tables with clear headers. Example: Feature | Option A | Option B',
-      affectedUrls: [url],
-    });
-  }
-
-  // No numbered lists for procedural content
-  if (!hasStepByStep && (bodyText.includes('step') || bodyText.includes('how to') || bodyText.includes('guide'))) {
-    issues.push({
-      code: 'AI_NO_NUMBERED_STEPS',
-      severity: 'notice',
-      category: 'ai-readiness',
-      title: 'Procedural content without numbered steps',
-      description: 'Your content appears to be a guide or how-to but doesn\'t use numbered lists. AI prefers numbered lists for step-by-step content.',
-      impact: 'AI may not accurately quote your steps in order, or may skip your content for better-structured alternatives.',
-      howToFix: 'Convert step-by-step instructions to <ol> (ordered list) format with clear, actionable items.',
-      affectedUrls: [url],
-    });
-  }
-
-  // No Q&A format
-  if (!hasExplicitQA && questionCount === 0) {
-    issues.push({
-      code: 'AI_NO_QA_FORMAT',
-      severity: 'notice',
-      category: 'ai-readiness',
-      title: 'No question-and-answer format detected',
-      description: 'Content lacks explicit Q&A structure. AI systems often look for clear question-answer pairs to provide direct responses.',
-      impact: 'Lower chance of being featured in AI direct answers or FAQ-style responses.',
-      howToFix: 'Add FAQ section with common questions as headings (H2/H3) followed by concise answers. Implement FAQ schema markup.',
-      affectedUrls: [url],
-      details: {
-        suggestions: [
-          'Use question words in headings (What, How, Why, etc.)',
-          'Follow questions with short, direct answer paragraphs',
-          'Add FAQ schema markup for structured data',
-          'Consider using <details>/<summary> for expandable Q&A',
-        ],
-      },
-    });
-  }
-
-  // Questions without direct answers
-  if (questionCount >= 2 && !hasDirectAnswers) {
-    issues.push({
-      code: 'AI_QUESTIONS_NO_DIRECT_ANSWERS',
-      severity: 'notice',
-      category: 'ai-readiness',
-      title: 'Questions in headings lack direct answers',
-      description: `Found ${questionCount} question-style headings but answers are too long or not immediately following. AI prefers concise answers right after questions.`,
-      impact: 'AI may struggle to extract clear answers, reducing citation likelihood.',
-      howToFix: 'Start each answer section with a 1-2 sentence direct answer, then expand with details. First sentence should standalone as a complete answer.',
-      affectedUrls: [url],
-    });
-  }
-
-  // No definitions
-  if (!hasDefinitions && (bodyText.includes('what is') || bodyText.includes('definition'))) {
-    issues.push({
-      code: 'AI_NO_DEFINITIONS',
-      severity: 'notice',
-      category: 'ai-readiness',
-      title: 'Missing clear definitions',
-      description: 'Content discusses concepts but lacks Wikipedia-style definitions. AI loves clear "X is a..." statements.',
-      impact: 'Less likely to be cited for definitional queries in AI search.',
-      howToFix: 'Add clear definitions early in content: "[Term] is a [category] that [distinguishing features]." Make first paragraph a complete definition.',
-      affectedUrls: [url],
-    });
-  }
-
-  // Low quotability
-  if (structureScore < 50) {
-    issues.push({
-      code: 'AI_LOW_STRUCTURE_SCORE',
-      severity: 'warning',
-      category: 'ai-readiness',
-      title: 'Content structure not optimized for AI parsing',
-      description: `AI structure score: ${structureScore}/100. Content lacks structured elements that AI can easily parse and quote.`,
-      impact: 'Lower likelihood of being cited in AI-generated answers due to poor content structure.',
-      howToFix: 'Improve content structure: add tables for data, numbered lists for steps, bullet points for features, Q&A sections for common questions.',
-      affectedUrls: [url],
-      details: {
-        structureScore,
-        hasTable: tableCount > 0,
-        hasOrderedList: orderedLists > 0,
-        hasUnorderedList: unorderedLists > 0,
-        hasQAFormat: hasExplicitQA || questionCount > 0,
-      },
-    });
-  }
-
-  return {
-    issues,
-    data: {
-      tables: {
-        count: tableCount,
-        hasComparisonTable,
-        hasDataTable,
-      },
-      lists: {
-        orderedLists,
-        unorderedLists,
-        hasStepByStep,
-        hasBulletedFeatures,
-      },
-      qaFormat: {
-        hasExplicitQA,
-        questionCount,
-        hasDirectAnswers,
-      },
-      quotability: {
-        hasDefinitions,
-        hasConciseStatements,
-        shortParagraphRatio,
-      },
-      structureScore,
-    },
-  };
-}

package/src/audit/checks/ai-readiness.ts

@@ -1,339 +0,0 @@
-// AI/LLM Readiness Checks
-// Checks for llms.txt, AI bot blocking, and JS rendering ratio
-
-import { httpGet } from '../../utils/http.js';
-import * as cheerio from 'cheerio';
-import type { AuditIssue } from '../types.js';
-import { ISSUE_DEFINITIONS } from '../types.js';
-
-// Known AI bot user agents
-const AI_BOTS = {
-  GPTBot: 'GPTBot',
-  'ChatGPT-User': 'ChatGPT-User',
-  'OAI-SearchBot': 'OAI-SearchBot',
-  ClaudeBot: 'ClaudeBot',
-  'Claude-Web': 'Claude-Web',
-  'anthropic-ai': 'anthropic-ai',
-  PerplexityBot: 'PerplexityBot',
-  'Google-Extended': 'Google-Extended',
-  Bytespider: 'Bytespider',
-  CCBot: 'CCBot',
-};
-
-export interface LlmsTxtResult {
-  exists: boolean;
-  content?: string;
-  valid: boolean;
-  errors: string[];
-}
-
-export interface AIBotBlockingResult {
-  robotsExists: boolean;
-  blockedBots: string[];
-  allowedBots: string[];
-  allBlocked: boolean;
-}
-
-export interface AIReadinessData {
-  llmsTxt: LlmsTxtResult;
-  botBlocking: AIBotBlockingResult;
-  jsRenderingRatio: number; // 0-100%
-}
-
-/**
- * Check for llms.txt file
- * See: https://llmstxt.org/
- */
-export async function checkLlmsTxt(baseUrl: string): Promise<{ issues: AuditIssue[]; data: LlmsTxtResult }> {
-  const issues: AuditIssue[] = [];
-  const url = new URL('/llms.txt', baseUrl).href;
-
-  try {
-    const response = await httpGet<string>(url, {
-      timeout: 10000,
-      validateStatus: () => true,
-    });
-
-    if (response.status === 404 || response.status >= 400) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.LLMS_TXT_MISSING,
-        affectedUrls: [url],
-      });
-      return {
-        issues,
-        data: { exists: false, valid: false, errors: ['File not found'] },
-      };
-    }
-
-    const content = response.data as string;
-
-    // Basic validation of llms.txt format
-    // Expected format starts with # followed by site name, then markdown content
-    const lines = content.split('\n').filter((line) => line.trim());
-    const errors: string[] = [];
-    let valid = true;
-
-    // Check if it starts with a title (# Site Name)
-    if (lines.length === 0) {
-      errors.push('File is empty');
-      valid = false;
-    } else if (!lines[0].startsWith('#')) {
-      errors.push('File should start with a markdown heading (# Site Name)');
-      valid = false;
-    }
-
-    // Check for common sections
-    const hasDescription = content.toLowerCase().includes('## ') || content.length > 50;
-    if (!hasDescription) {
-      errors.push('File should contain meaningful content describing your site for AI');
-      valid = false;
-    }
-
-    if (!valid) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.LLMS_TXT_INVALID,
-        affectedUrls: [url],
-        details: { errors },
-      });
-    }
-
-    return {
-      issues,
-      data: { exists: true, content, valid, errors },
-    };
-  } catch (error) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.LLMS_TXT_MISSING,
-      affectedUrls: [url],
-      details: { error: error instanceof Error ? error.message : 'Unknown error' },
-    });
-    return {
-      issues,
-      data: { exists: false, valid: false, errors: ['Failed to fetch'] },
-    };
-  }
-}
-
-/**
- * Check robots.txt for AI bot blocking
- */
-export async function checkAIBotBlocking(baseUrl: string): Promise<{ issues: AuditIssue[]; data: AIBotBlockingResult }> {
-  const issues: AuditIssue[] = [];
-  const url = new URL('/robots.txt', baseUrl).href;
-
-  const blockedBots: string[] = [];
-  const allowedBots: string[] = [];
-
-  try {
-    const response = await httpGet<string>(url, {
-      timeout: 10000,
-      validateStatus: () => true,
-    });
-
-    if (response.status === 404 || response.status >= 400) {
-      // No robots.txt means all bots are allowed
-      return {
-        issues,
-        data: {
-          robotsExists: false,
-          blockedBots: [],
-          allowedBots: Object.keys(AI_BOTS),
-          allBlocked: false,
-        },
-      };
-    }
-
-    const content = response.data as string;
-    const lines = content.split('\n');
-
-    // Parse robots.txt for AI bot rules
-    let currentUserAgent = '';
-    const botRules: Record<string, { allowed: boolean; disallowAll: boolean }> = {};
-
-    for (const line of lines) {
-      const trimmed = line.trim().toLowerCase();
-
-      if (trimmed.startsWith('user-agent:')) {
-        currentUserAgent = trimmed.split(':')[1].trim();
-      } else if (trimmed.startsWith('disallow:')) {
-        const path = trimmed.split(':')[1]?.trim() || '';
-
-        // Check if this user agent matches any AI bot
-        for (const [botKey, botName] of Object.entries(AI_BOTS)) {
-          if (currentUserAgent === '*' || currentUserAgent === botKey.toLowerCase() || currentUserAgent === botName.toLowerCase()) {
-            if (path === '/' || path === '/*') {
-              if (!botRules[botKey]) {
-                botRules[botKey] = { allowed: false, disallowAll: false };
-              }
-              if (currentUserAgent !== '*' || (currentUserAgent === '*' && !botRules[botKey].allowed)) {
-                botRules[botKey].disallowAll = true;
-              }
-            }
-          }
-        }
-      } else if (trimmed.startsWith('allow:')) {
-        // Explicit allow for a bot
-        for (const [botKey] of Object.entries(AI_BOTS)) {
-          if (currentUserAgent === botKey.toLowerCase()) {
-            if (!botRules[botKey]) {
-              botRules[botKey] = { allowed: true, disallowAll: false };
-            }
-            botRules[botKey].allowed = true;
-          }
-        }
-      }
-    }
-
-    // Determine blocked/allowed bots
-    for (const [botKey] of Object.entries(AI_BOTS)) {
-      const rules = botRules[botKey];
-      if (rules?.disallowAll && !rules?.allowed) {
-        blockedBots.push(botKey);
-      } else {
-        allowedBots.push(botKey);
-      }
-    }
-
-    // Generate issues for blocked bots
-    if (blockedBots.length === Object.keys(AI_BOTS).length) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.AI_BOT_BLOCKED_ALL,
-        affectedUrls: [url],
-        details: { blockedBots },
-      });
-    } else {
-      // Individual bot blocking notices
-      if (blockedBots.includes('GPTBot') || blockedBots.includes('ChatGPT-User')) {
-        issues.push({
-          ...ISSUE_DEFINITIONS.AI_BOT_GPTBOT_BLOCKED,
-          affectedUrls: [url],
-        });
-      }
-      if (blockedBots.includes('ClaudeBot') || blockedBots.includes('Claude-Web') || blockedBots.includes('anthropic-ai')) {
-        issues.push({
-          ...ISSUE_DEFINITIONS.AI_BOT_CLAUDEBOT_BLOCKED,
-          affectedUrls: [url],
-        });
-      }
-      if (blockedBots.includes('PerplexityBot')) {
-        issues.push({
-          ...ISSUE_DEFINITIONS.AI_BOT_PERPLEXITY_BLOCKED,
-          affectedUrls: [url],
-        });
-      }
-      if (blockedBots.includes('Google-Extended')) {
-        issues.push({
-          ...ISSUE_DEFINITIONS.AI_BOT_GOOGLE_EXTENDED_BLOCKED,
-          affectedUrls: [url],
-        });
-      }
-    }
-
-    return {
-      issues,
-      data: {
-        robotsExists: true,
-        blockedBots,
-        allowedBots,
-        allBlocked: blockedBots.length === Object.keys(AI_BOTS).length,
-      },
-    };
-  } catch (error) {
-    return {
-      issues,
-      data: {
-        robotsExists: false,
-        blockedBots: [],
-        allowedBots: Object.keys(AI_BOTS),
-        allBlocked: false,
-      },
-    };
-  }
-}
-
-/**
- * Check JavaScript rendering ratio
- * Compares static HTML content size vs rendered content
- */
-export function checkJSRenderingRatio(
-  html: string,
-  url: string
-): { issues: AuditIssue[]; data: { ratio: number; staticWordCount: number } } {
-  const issues: AuditIssue[] = [];
-  const $ = cheerio.load(html);
-
-  // Get static text content (what AI crawlers would see without JS)
-  // Remove script and style tags
-  $('script, style, noscript').remove();
-
-  const staticText = $('body').text().replace(/\s+/g, ' ').trim();
-  const staticWordCount = staticText.split(/\s+/).filter((word) => word.length > 0).length;
-
-  // Check for signs of heavy JS rendering
-  const hasReactRoot = $('#root, #app, #__next, [data-reactroot]').length > 0;
-  const hasVueApp = $('#app[data-v-app], [data-v-]').length > 0;
-  const hasAngularApp = $('[ng-app], [data-ng-app]').length > 0;
-  const hasEmptyBody = staticWordCount < 50;
-
-  // Estimate JS rendering ratio based on signals
-  let ratio = 0;
-  if (hasEmptyBody && (hasReactRoot || hasVueApp || hasAngularApp)) {
-    ratio = 90; // Likely SPA with most content rendered by JS
-  } else if (hasReactRoot || hasVueApp || hasAngularApp) {
-    ratio = 50; // Has JS framework but some static content
-  } else if (staticWordCount < 100) {
-    ratio = 60; // Very little static content
-  } else {
-    ratio = 10; // Mostly static content
-  }
-
-  // Check for noscript fallback
-  const hasNoScript = $('noscript').length > 0;
-  if (hasNoScript && ratio > 50) {
-    ratio -= 20; // Has fallback content
-  }
-
-  if (ratio > 50) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.HIGH_JS_RENDERING_RATIO,
-      affectedUrls: [url],
-      details: { ratio: `${ratio}%`, staticWordCount },
-    });
-  }
-
-  return {
-    issues,
-    data: { ratio, staticWordCount },
-  };
-}
-
-/**
- * Run all AI readiness checks
- */
-export async function runAIReadinessChecks(
-  baseUrl: string,
-  html: string
-): Promise<{ issues: AuditIssue[]; data: AIReadinessData }> {
-  const allIssues: AuditIssue[] = [];
-
-  // Check llms.txt
-  const llmsResult = await checkLlmsTxt(baseUrl);
-  allIssues.push(...llmsResult.issues);
-
-  // Check AI bot blocking
-  const botResult = await checkAIBotBlocking(baseUrl);
-  allIssues.push(...botResult.issues);
-
-  // Check JS rendering ratio
-  const jsResult = checkJSRenderingRatio(html, baseUrl);
-  allIssues.push(...jsResult.issues);
-
-  return {
-    issues: allIssues,
-    data: {
-      llmsTxt: llmsResult.data,
-      botBlocking: botResult.data,
-      jsRenderingRatio: jsResult.data.ratio,
-    },
-  };
-}