@rankcli/agent-runtime 0.0.15 → 0.0.16

package/src/audit/checks/content-science.ts

@@ -1,589 +0,0 @@
-// Content Science Module - Novel SEO Checks Based on Academic Research
-// Implements: Zipf's Law, Shannon Entropy, BM25 Scoring, Cosine Similarity
-// These techniques differentiate RankCLI from competitors like Ahrefs/SEMrush
-
-import * as cheerio from 'cheerio';
-import type { AuditIssue } from '../types.js';
-
-// ==================== ZIPF'S LAW ANALYSIS ====================
-// Natural language follows Zipf's law: frequency ∝ 1/rank^α (α ≈ 1)
-// Deviation from this indicates unnatural text (keyword stuffing)
-
-export interface ZipfAnalysis {
-  alpha: number; // Exponent - natural text ≈ 0.9-1.1
-  isNatural: boolean;
-  rSquared: number; // Goodness of fit
-  stuffedKeywords: Array<{ word: string; deviation: number }>;
-  missingMidTail: string[];
-}
-
-/**
- * Analyze keyword distribution against Zipf's law
- * Detects keyword stuffing through statistical distribution analysis
- */
-export function analyzeZipfDistribution(html: string): ZipfAnalysis {
-  const $ = cheerio.load(html);
-  $('script, style, nav, footer, header').remove();
-  const text = $('body').text().toLowerCase();
-
-  // Extract words (min 3 chars to filter stopwords)
-  const words = text.match(/\b[a-z]{3,}\b/g) || [];
-  if (words.length < 100) {
-    return {
-      alpha: 1,
-      isNatural: true,
-      rSquared: 0,
-      stuffedKeywords: [],
-      missingMidTail: [],
-    };
-  }
-
-  // Count word frequencies
-  const freq = new Map<string, number>();
-  for (const word of words) {
-    freq.set(word, (freq.get(word) || 0) + 1);
-  }
-
-  // Sort by frequency (descending)
-  const sorted = [...freq.entries()].sort((a, b) => b[1] - a[1]);
-
-  // Take top 100 words for analysis
-  const topWords = sorted.slice(0, Math.min(100, sorted.length));
-
-  // Fit Zipf's law using log-log linear regression
-  // log(freq) = log(C) - α * log(rank)
-  const logRanks = topWords.map((_, i) => Math.log(i + 1));
-  const logFreqs = topWords.map(([_, f]) => Math.log(f));
-
-  const { slope, intercept, rSquared } = linearRegression(logRanks, logFreqs);
-  const alpha = -slope; // α is negative of slope
-
-  // Find deviations (potential keyword stuffing)
-  const stuffedKeywords: Array<{ word: string; deviation: number }> = [];
-  const expectedC = Math.exp(intercept);
-
-  for (let i = 0; i < topWords.length; i++) {
-    const [word, actualFreq] = topWords[i];
-    const expectedFreq = expectedC * Math.pow(i + 1, -alpha);
-    const deviation = (actualFreq - expectedFreq) / expectedFreq;
-
-    // Flag words with >50% higher frequency than expected (excluding very common words)
-    if (deviation > 0.5 && word.length > 4 && i > 5) {
-      stuffedKeywords.push({ word, deviation: Math.round(deviation * 100) });
-    }
-  }
-
-  // Find missing mid-tail terms (ranks 20-50 with lower than expected frequency)
-  const missingMidTail: string[] = [];
-  for (let i = 20; i < Math.min(50, topWords.length); i++) {
-    const [word, actualFreq] = topWords[i];
-    const expectedFreq = expectedC * Math.pow(i + 1, -alpha);
-    const deviation = (actualFreq - expectedFreq) / expectedFreq;
-
-    if (deviation < -0.3) {
-      missingMidTail.push(word);
-    }
-  }
-
-  return {
-    alpha,
-    isNatural: alpha >= 0.8 && alpha <= 1.2 && rSquared > 0.85,
-    rSquared,
-    stuffedKeywords: stuffedKeywords.slice(0, 10),
-    missingMidTail: missingMidTail.slice(0, 5),
-  };
-}
-
-// ==================== SHANNON ENTROPY ====================
-// Information entropy measures vocabulary diversity
-// Low entropy = repetitive content, High entropy = diverse vocabulary
-
-export interface EntropyAnalysis {
-  wordEntropy: number;
-  normalizedEntropy: number; // 0-1 scale
-  vocabularyRichness: number; // Type-token ratio
-  repetitionScore: number; // 0-1, higher = more repetitive
-  bigramEntropy: number;
-  qualityIndicator: 'excellent' | 'good' | 'average' | 'poor';
-}
-
-/**
- * Calculate Shannon entropy for content quality assessment
- */
-export function analyzeEntropy(html: string): EntropyAnalysis {
-  const $ = cheerio.load(html);
-  $('script, style, nav, footer, header').remove();
-  const text = $('body').text().toLowerCase();
-
-  const words = text.match(/\b[a-z]{2,}\b/g) || [];
-  if (words.length < 50) {
-    return {
-      wordEntropy: 0,
-      normalizedEntropy: 0,
-      vocabularyRichness: 0,
-      repetitionScore: 1,
-      bigramEntropy: 0,
-      qualityIndicator: 'poor',
-    };
-  }
-
-  // Word frequency
-  const wordFreq = new Map<string, number>();
-  for (const word of words) {
-    wordFreq.set(word, (wordFreq.get(word) || 0) + 1);
-  }
-
-  // Bigram frequency
-  const bigramFreq = new Map<string, number>();
-  for (let i = 0; i < words.length - 1; i++) {
-    const bigram = `${words[i]} ${words[i + 1]}`;
-    bigramFreq.set(bigram, (bigramFreq.get(bigram) || 0) + 1);
-  }
-
-  // Shannon entropy: H = -Σ p(x) * log2(p(x))
-  const wordEntropy = calculateShannonEntropy(wordFreq, words.length);
-  const bigramEntropy = calculateShannonEntropy(bigramFreq, words.length - 1);
-
-  // Maximum possible entropy (uniform distribution)
-  const maxEntropy = Math.log2(wordFreq.size);
-  const normalizedEntropy = maxEntropy > 0 ? wordEntropy / maxEntropy : 0;
-
-  // Type-token ratio (unique words / total words)
-  const vocabularyRichness = wordFreq.size / words.length;
-
-  // Repetition score (inverse of normalized entropy)
-  const repetitionScore = 1 - normalizedEntropy;
-
-  // Quality indicator
-  let qualityIndicator: EntropyAnalysis['qualityIndicator'];
-  if (normalizedEntropy > 0.85 && vocabularyRichness > 0.4) {
-    qualityIndicator = 'excellent';
-  } else if (normalizedEntropy > 0.75 && vocabularyRichness > 0.3) {
-    qualityIndicator = 'good';
-  } else if (normalizedEntropy > 0.6 && vocabularyRichness > 0.2) {
-    qualityIndicator = 'average';
-  } else {
-    qualityIndicator = 'poor';
-  }
-
-  return {
-    wordEntropy,
-    normalizedEntropy,
-    vocabularyRichness,
-    repetitionScore,
-    bigramEntropy,
-    qualityIndicator,
-  };
-}
-
-function calculateShannonEntropy(freq: Map<string, number>, total: number): number {
-  let entropy = 0;
-  for (const count of freq.values()) {
-    const p = count / total;
-    if (p > 0) {
-      entropy -= p * Math.log2(p);
-    }
-  }
-  return entropy;
-}
-
-// ==================== BM25 SCORING ====================
-// Okapi BM25 is the ranking function used by Elasticsearch/Lucene
-// More accurate than keyword density for relevance scoring
-
-export interface BM25Analysis {
-  keyword: string;
-  score: number;
-  termFrequency: number;
-  saturationPoint: boolean; // TF above optimal
-  lengthPenalty: number;
-  recommendation: string;
-}
-
-/**
- * Calculate BM25 relevance score for target keywords
- * Parameters tuned for web content (k1=1.5, b=0.75)
- */
-export function calculateBM25(
-  html: string,
-  keywords: string[],
-  avgDocLength: number = 1500, // Average web page word count
-  k1: number = 1.5,
-  b: number = 0.75
-): BM25Analysis[] {
-  const $ = cheerio.load(html);
-  $('script, style').remove();
-  const text = $('body').text().toLowerCase();
-  const words = text.match(/\b[a-z]+\b/g) || [];
-  const docLength = words.length;
-
-  const results: BM25Analysis[] = [];
-
-  for (const keyword of keywords) {
-    const keywordLower = keyword.toLowerCase();
-    const keywordWords = keywordLower.split(/\s+/);
-
-    // Count exact phrase or individual word matches
-    let tf: number;
-    if (keywordWords.length > 1) {
-      // Phrase matching
-      const regex = new RegExp(keywordLower.replace(/\s+/g, '\\s+'), 'gi');
-      tf = (text.match(regex) || []).length;
-    } else {
-      // Single word
-      tf = words.filter((w) => w === keywordLower).length;
-    }
-
-    // BM25 formula (IDF = 1 for single document analysis)
-    const numerator = tf * (k1 + 1);
-    const denominator = tf + k1 * (1 - b + b * (docLength / avgDocLength));
-    const score = numerator / denominator;
-
-    // Check if additional occurrences would help (saturation check)
-    const nextTF = tf + 1;
-    const nextScore = (nextTF * (k1 + 1)) / (nextTF + k1 * (1 - b + b * (docLength / avgDocLength)));
-    const marginalGain = (nextScore - score) / (score || 1);
-    const saturationPoint = marginalGain < 0.1; // Less than 10% improvement
-
-    // Length penalty
-    const lengthPenalty = b * (docLength / avgDocLength - 1);
-
-    // Generate recommendation
-    let recommendation: string;
-    if (tf === 0) {
-      recommendation = `Add "${keyword}" to your content`;
-    } else if (saturationPoint) {
-      recommendation = `Keyword "${keyword}" is saturated (${tf} occurrences). Focus on semantic variations.`;
-    } else if (lengthPenalty > 0.3) {
-      recommendation = `Content is ${Math.round(lengthPenalty * 100)}% longer than average. Consider tightening.`;
-    } else if (tf < 3) {
-      recommendation = `Consider adding 1-2 more natural mentions of "${keyword}"`;
-    } else {
-      recommendation = `Good keyword presence for "${keyword}"`;
-    }
-
-    results.push({
-      keyword,
-      score: Math.round(score * 100) / 100,
-      termFrequency: tf,
-      saturationPoint,
-      lengthPenalty: Math.round(lengthPenalty * 100) / 100,
-      recommendation,
-    });
-  }
-
-  return results;
-}
-
-// ==================== COSINE SIMILARITY ====================
-// Better duplicate/cannibalization detection than exact matching
-
-export interface SimilarityAnalysis {
-  similarity: number; // 0-1
-  sharedTerms: string[];
-  uniqueToA: string[];
-  uniqueToB: string[];
-  recommendation: 'duplicate' | 'cannibalization' | 'related' | 'distinct';
-}
-
-/**
- * Build TF-IDF vector for document
- */
-export function buildTFIDFVector(text: string, vocabulary: string[]): number[] {
-  const words = text.toLowerCase().match(/\b[a-z]+\b/g) || [];
-  const wordCount = new Map<string, number>();
-
-  for (const word of words) {
-    wordCount.set(word, (wordCount.get(word) || 0) + 1);
-  }
-
-  const docLength = words.length || 1;
-  const vector: number[] = [];
-
-  for (const term of vocabulary) {
-    const tf = (wordCount.get(term) || 0) / docLength;
-    // IDF approximation: log(2) for present terms, 0 for absent
-    const idf = wordCount.has(term) ? Math.log(2) : 0;
-    vector.push(tf * idf);
-  }
-
-  return vector;
-}
-
-/**
- * Calculate cosine similarity between two content pieces
- */
-export function calculateCosineSimilarity(textA: string, textB: string): SimilarityAnalysis {
-  const wordsA = new Set((textA.toLowerCase().match(/\b[a-z]{3,}\b/g) || []));
-  const wordsB = new Set((textB.toLowerCase().match(/\b[a-z]{3,}\b/g) || []));
-
-  // Build combined vocabulary
-  const vocabulary = [...new Set([...wordsA, ...wordsB])];
-
-  // Build TF-IDF vectors
-  const vectorA = buildTFIDFVector(textA, vocabulary);
-  const vectorB = buildTFIDFVector(textB, vocabulary);
-
-  // Cosine similarity
-  let dotProduct = 0;
-  let normA = 0;
-  let normB = 0;
-
-  for (let i = 0; i < vocabulary.length; i++) {
-    dotProduct += vectorA[i] * vectorB[i];
-    normA += vectorA[i] * vectorA[i];
-    normB += vectorB[i] * vectorB[i];
-  }
-
-  const similarity = normA > 0 && normB > 0 ? dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)) : 0;
-
-  // Find shared and unique terms
-  const sharedTerms = [...wordsA].filter((w) => wordsB.has(w)).slice(0, 20);
-  const uniqueToA = [...wordsA].filter((w) => !wordsB.has(w)).slice(0, 10);
-  const uniqueToB = [...wordsB].filter((w) => !wordsA.has(w)).slice(0, 10);
-
-  // Recommendation based on similarity
-  let recommendation: SimilarityAnalysis['recommendation'];
-  if (similarity > 0.95) {
-    recommendation = 'duplicate';
-  } else if (similarity > 0.75) {
-    recommendation = 'cannibalization';
-  } else if (similarity > 0.5) {
-    recommendation = 'related';
-  } else {
-    recommendation = 'distinct';
-  }
-
-  return {
-    similarity: Math.round(similarity * 100) / 100,
-    sharedTerms,
-    uniqueToA,
-    uniqueToB,
-    recommendation,
-  };
-}
-
-// ==================== INFORMATION GAIN ====================
-// Identifies keywords that differentiate top-ranking content
-
-export interface InformationGainResult {
-  keyword: string;
-  informationGain: number;
-  presentInTopContent: boolean;
-  priority: 'critical' | 'high' | 'medium' | 'low';
-}
-
-/**
- * Calculate information gain for keywords (simplified for single-document analysis)
- * Compares content against expected term distribution
- */
-export function calculateInformationGain(
-  html: string,
-  topicKeywords: string[]
-): InformationGainResult[] {
-  const $ = cheerio.load(html);
-  $('script, style').remove();
-  const text = $('body').text().toLowerCase();
-
-  const results: InformationGainResult[] = [];
-
-  for (const keyword of topicKeywords) {
-    const keywordLower = keyword.toLowerCase();
-    const isPresent = text.includes(keywordLower);
-
-    // Simple information gain approximation
-    // Higher IG for important topic keywords that are missing
-    const wordLength = keywordLower.split(/\s+/).length;
-    const baseIG = wordLength > 1 ? 0.5 : 0.3; // Phrases are more specific
-
-    const informationGain = isPresent ? baseIG * 0.8 : baseIG;
-
-    let priority: InformationGainResult['priority'];
-    if (!isPresent && informationGain > 0.4) {
-      priority = 'critical';
-    } else if (!isPresent && informationGain > 0.3) {
-      priority = 'high';
-    } else if (!isPresent) {
-      priority = 'medium';
-    } else {
-      priority = 'low';
-    }
-
-    results.push({
-      keyword,
-      informationGain: Math.round(informationGain * 100) / 100,
-      presentInTopContent: isPresent,
-      priority,
-    });
-  }
-
-  return results.sort((a, b) => b.informationGain - a.informationGain);
-}
-
-// ==================== HELPER FUNCTIONS ====================
-
-function linearRegression(x: number[], y: number[]): { slope: number; intercept: number; rSquared: number } {
-  const n = x.length;
-  let sumX = 0,
-    sumY = 0,
-    sumXY = 0,
-    sumXX = 0,
-    sumYY = 0;
-
-  for (let i = 0; i < n; i++) {
-    sumX += x[i];
-    sumY += y[i];
-    sumXY += x[i] * y[i];
-    sumXX += x[i] * x[i];
-    sumYY += y[i] * y[i];
-  }
-
-  const slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX);
-  const intercept = (sumY - slope * sumX) / n;
-
-  // R-squared
-  const yMean = sumY / n;
-  let ssRes = 0,
-    ssTot = 0;
-  for (let i = 0; i < n; i++) {
-    const predicted = slope * x[i] + intercept;
-    ssRes += (y[i] - predicted) ** 2;
-    ssTot += (y[i] - yMean) ** 2;
-  }
-  const rSquared = ssTot > 0 ? 1 - ssRes / ssTot : 0;
-
-  return { slope, intercept, rSquared };
-}
-
-// ==================== MAIN ANALYSIS FUNCTION ====================
-
-export interface ContentScienceData {
-  zipf: ZipfAnalysis;
-  entropy: EntropyAnalysis;
-  bm25: BM25Analysis[];
-  overallQuality: 'excellent' | 'good' | 'needs-work' | 'poor';
-}
-
-/**
- * Run comprehensive content science analysis
- */
-export function analyzeContentScience(
-  html: string,
-  url: string,
-  targetKeywords: string[] = []
-): { issues: AuditIssue[]; data: ContentScienceData } {
-  const issues: AuditIssue[] = [];
-
-  // Zipf's Law Analysis
-  const zipf = analyzeZipfDistribution(html);
-
-  if (!zipf.isNatural && zipf.stuffedKeywords.length > 0) {
-    issues.push({
-      code: 'ZIPF_KEYWORD_STUFFING',
-      severity: 'warning',
-      category: 'content',
-      title: 'Unnatural keyword distribution detected',
-      description: `Content deviates from natural language patterns (Zipf α=${zipf.alpha.toFixed(2)}, expected ~1.0). Possible keyword stuffing detected.`,
-      impact: 'Search engines can detect unnatural text patterns, potentially triggering spam filters.',
-      howToFix: `Reduce repetition of: ${zipf.stuffedKeywords.map((k) => `"${k.word}" (+${k.deviation}%)`).join(', ')}`,
-      affectedUrls: [url],
-      details: {
-        alpha: zipf.alpha,
-        rSquared: zipf.rSquared,
-        stuffedKeywords: zipf.stuffedKeywords,
-      },
-    });
-  }
-
-  // Entropy Analysis
-  const entropy = analyzeEntropy(html);
-
-  if (entropy.qualityIndicator === 'poor') {
-    issues.push({
-      code: 'LOW_CONTENT_ENTROPY',
-      severity: 'warning',
-      category: 'content',
-      title: 'Low vocabulary diversity (thin content signal)',
-      description: `Content entropy is low (${entropy.normalizedEntropy.toFixed(2)}). This indicates repetitive or thin content.`,
-      impact: 'Low diversity content may be seen as low-quality by search engines.',
-      howToFix: 'Expand vocabulary, add more unique insights, and reduce repetitive phrases.',
-      affectedUrls: [url],
-      details: {
-        normalizedEntropy: entropy.normalizedEntropy,
-        vocabularyRichness: entropy.vocabularyRichness,
-        repetitionScore: entropy.repetitionScore,
-      },
-    });
-  }
-
-  if (entropy.repetitionScore > 0.7) {
-    issues.push({
-      code: 'HIGH_CONTENT_REPETITION',
-      severity: 'notice',
-      category: 'content',
-      title: 'High content repetition detected',
-      description: `Content shows ${Math.round(entropy.repetitionScore * 100)}% repetition pattern.`,
-      impact: 'Highly repetitive content provides less value and may hurt engagement.',
-      howToFix: 'Vary your vocabulary and sentence structures. Avoid repeating the same phrases.',
-      affectedUrls: [url],
-    });
-  }
-
-  // BM25 Analysis (if keywords provided)
-  const bm25 = targetKeywords.length > 0 ? calculateBM25(html, targetKeywords) : [];
-
-  for (const result of bm25) {
-    if (result.termFrequency === 0) {
-      issues.push({
-        code: 'BM25_KEYWORD_MISSING',
-        severity: 'warning',
-        category: 'on-page',
-        title: `Target keyword "${result.keyword}" not found`,
-        description: 'This keyword is not present in your content at all.',
-        impact: 'Page unlikely to rank for this keyword without any mention of it.',
-        howToFix: `Add natural mentions of "${result.keyword}" in your content.`,
-        affectedUrls: [url],
-        details: { keyword: result.keyword, bm25Score: 0 },
-      });
-    } else if (result.saturationPoint && result.termFrequency > 10) {
-      issues.push({
-        code: 'BM25_KEYWORD_SATURATED',
-        severity: 'notice',
-        category: 'on-page',
-        title: `Keyword "${result.keyword}" over-optimized`,
-        description: `Found ${result.termFrequency} times. Additional mentions provide diminishing returns.`,
-        impact: 'Over-optimization can trigger spam signals.',
-        howToFix: 'Use semantic variations and related terms instead of exact repetition.',
-        affectedUrls: [url],
-        details: { keyword: result.keyword, termFrequency: result.termFrequency },
-      });
-    }
-  }
-
-  // Overall quality assessment
-  let overallQuality: ContentScienceData['overallQuality'];
-  const qualityScore =
-    (zipf.isNatural ? 30 : 0) +
-    (entropy.qualityIndicator === 'excellent' ? 40 : entropy.qualityIndicator === 'good' ? 30 : entropy.qualityIndicator === 'average' ? 20 : 0) +
-    (bm25.filter((b) => b.termFrequency > 0 && !b.saturationPoint).length / Math.max(bm25.length, 1)) * 30;
-
-  if (qualityScore >= 80) {
-    overallQuality = 'excellent';
-  } else if (qualityScore >= 60) {
-    overallQuality = 'good';
-  } else if (qualityScore >= 40) {
-    overallQuality = 'needs-work';
-  } else {
-    overallQuality = 'poor';
-  }
-
-  return {
-    issues,
-    data: {
-      zipf,
-      entropy,
-      bm25,
-      overallQuality,
-    },
-  };
-}