@rankcli/agent-runtime 0.0.1

package/src/audit/checks/internal-link-graph.ts

@@ -0,0 +1,436 @@
+// Internal Link Graph Analysis
+// Advanced internal linking analysis: hub/authority detection, link depth, PageRank sculpting
+// Based on advanced SEO research
+
+import * as cheerio from 'cheerio';
+import type { AuditIssue } from '../types.js';
+
+export interface InternalLink {
+  source: string;
+  target: string;
+  anchor: string;
+  context: 'navigation' | 'content' | 'footer' | 'sidebar';
+  isEditorial: boolean;
+}
+
+export interface PageLinkStats {
+  url: string;
+  inboundLinks: number;
+  outboundLinks: number;
+  hubScore: number; // Pages that link to many others
+  authorityScore: number; // Pages that receive many links
+  depth: number; // Clicks from homepage
+}
+
+export interface InternalLinkGraphData {
+  totalInternalLinks: number;
+  orphanPages: string[];
+  deepPages: string[]; // More than 3 clicks deep
+  hubPages: string[];
+  authorityPages: string[];
+  linkDistribution: { url: string; links: number }[];
+  firstLinkAnchors: Map<string, string>;
+  contextualLinkRatio: number;
+  navigationVsContentRatio: number;
+}
+
+/**
+ * Extract internal links from HTML with context analysis
+ */
+export function extractInternalLinks(html: string, pageUrl: string): InternalLink[] {
+  const $ = cheerio.load(html);
+  const links: InternalLink[] = [];
+  const baseUrl = new URL(pageUrl);
+
+  $('a[href]').each((_, el) => {
+    const $el = $(el);
+    const href = $el.attr('href') || '';
+    const anchor = $el.text().trim();
+
+    // Skip empty, javascript:, and fragment links
+    if (!href || href.startsWith('javascript:') || href.startsWith('#')) {
+      return;
+    }
+
+    // Resolve relative URLs
+    let targetUrl: URL;
+    try {
+      targetUrl = new URL(href, pageUrl);
+    } catch {
+      return; // Invalid URL
+    }
+
+    // Check if internal link
+    if (targetUrl.hostname !== baseUrl.hostname) {
+      return; // External link
+    }
+
+    // Determine link context
+    let context: 'navigation' | 'content' | 'footer' | 'sidebar' = 'content';
+
+    const parents = $el.parents();
+    for (let i = 0; i < parents.length; i++) {
+      const parent = parents.eq(i);
+      const tagName = parent.prop('tagName')?.toLowerCase() || '';
+      const className = parent.attr('class')?.toLowerCase() || '';
+      const id = parent.attr('id')?.toLowerCase() || '';
+
+      if (
+        tagName === 'nav' ||
+        className.includes('nav') ||
+        className.includes('menu') ||
+        id.includes('nav') ||
+        id.includes('menu')
+      ) {
+        context = 'navigation';
+        break;
+      }
+      if (
+        tagName === 'footer' ||
+        className.includes('footer') ||
+        id.includes('footer')
+      ) {
+        context = 'footer';
+        break;
+      }
+      if (
+        tagName === 'aside' ||
+        className.includes('sidebar') ||
+        className.includes('widget') ||
+        id.includes('sidebar')
+      ) {
+        context = 'sidebar';
+        break;
+      }
+    }
+
+    // Determine if editorial (appears to be within content with surrounding text)
+    let isEditorial = context === 'content';
+    if (isEditorial) {
+      const parent = $el.parent();
+      const parentText = parent.text().trim();
+      // Editorial links usually have surrounding text beyond just the anchor
+      isEditorial = parentText.length > anchor.length + 10;
+    }
+
+    links.push({
+      source: pageUrl,
+      target: targetUrl.href.split('#')[0], // Remove fragment
+      anchor,
+      context,
+      isEditorial,
+    });
+  });
+
+  return links;
+}
+
+/**
+ * Analyze link depth from homepage
+ */
+export function calculateLinkDepth(
+  links: InternalLink[],
+  homepageUrl: string
+): Map<string, number> {
+  const depths = new Map<string, number>();
+  depths.set(homepageUrl, 0);
+
+  // BFS to calculate depths
+  const queue = [homepageUrl];
+  const visited = new Set([homepageUrl]);
+
+  while (queue.length > 0) {
+    const current = queue.shift()!;
+    const currentDepth = depths.get(current) || 0;
+
+    // Find all links from current page
+    const outbound = links.filter((l) => l.source === current);
+
+    for (const link of outbound) {
+      if (!visited.has(link.target)) {
+        visited.add(link.target);
+        depths.set(link.target, currentDepth + 1);
+        queue.push(link.target);
+      }
+    }
+  }
+
+  return depths;
+}
+
+/**
+ * Calculate simplified hub/authority scores (similar to HITS algorithm)
+ */
+export function calculateHubAuthority(
+  links: InternalLink[],
+  iterations: number = 10
+): { hubs: Map<string, number>; authorities: Map<string, number> } {
+  // Get all unique URLs
+  const urls = new Set<string>();
+  for (const link of links) {
+    urls.add(link.source);
+    urls.add(link.target);
+  }
+
+  // Initialize scores
+  const hubs = new Map<string, number>();
+  const authorities = new Map<string, number>();
+
+  for (const url of urls) {
+    hubs.set(url, 1);
+    authorities.set(url, 1);
+  }
+
+  // Build adjacency lists
+  const outbound = new Map<string, string[]>();
+  const inbound = new Map<string, string[]>();
+
+  for (const link of links) {
+    const out = outbound.get(link.source) || [];
+    out.push(link.target);
+    outbound.set(link.source, out);
+
+    const inb = inbound.get(link.target) || [];
+    inb.push(link.source);
+    inbound.set(link.target, inb);
+  }
+
+  // Iterative calculation
+  for (let i = 0; i < iterations; i++) {
+    // Update authority scores (sum of hub scores of pages linking to it)
+    const newAuthorities = new Map<string, number>();
+    for (const url of urls) {
+      const sources = inbound.get(url) || [];
+      let score = 0;
+      for (const source of sources) {
+        score += hubs.get(source) || 0;
+      }
+      newAuthorities.set(url, score);
+    }
+
+    // Update hub scores (sum of authority scores of pages it links to)
+    const newHubs = new Map<string, number>();
+    for (const url of urls) {
+      const targets = outbound.get(url) || [];
+      let score = 0;
+      for (const target of targets) {
+        score += newAuthorities.get(target) || 0;
+      }
+      newHubs.set(url, score);
+    }
+
+    // Normalize
+    const maxAuth = Math.max(...newAuthorities.values(), 1);
+    const maxHub = Math.max(...newHubs.values(), 1);
+
+    for (const url of urls) {
+      authorities.set(url, (newAuthorities.get(url) || 0) / maxAuth);
+      hubs.set(url, (newHubs.get(url) || 0) / maxHub);
+    }
+  }
+
+  return { hubs, authorities };
+}
+
+/**
+ * Detect orphan pages (no internal links pointing to them)
+ */
+export function detectOrphanPages(
+  links: InternalLink[],
+  knownUrls: string[]
+): string[] {
+  const linkedUrls = new Set(links.map((l) => l.target));
+  return knownUrls.filter((url) => !linkedUrls.has(url));
+}
+
+/**
+ * Analyze first link to each page (first link priority)
+ */
+export function analyzeFirstLinkPriority(
+  links: InternalLink[]
+): Map<string, { anchor: string; context: string }> {
+  const firstLinks = new Map<string, { anchor: string; context: string }>();
+
+  for (const link of links) {
+    if (!firstLinks.has(link.target)) {
+      firstLinks.set(link.target, {
+        anchor: link.anchor,
+        context: link.context,
+      });
+    }
+  }
+
+  return firstLinks;
+}
+
+/**
+ * Check for link hoarding (pages with few outbound internal links)
+ */
+export function detectLinkHoarding(
+  links: InternalLink[],
+  threshold: number = 3
+): string[] {
+  const outboundCount = new Map<string, number>();
+
+  for (const link of links) {
+    outboundCount.set(link.source, (outboundCount.get(link.source) || 0) + 1);
+  }
+
+  // Find pages with very few outbound links
+  return [...outboundCount.entries()]
+    .filter(([_, count]) => count < threshold)
+    .map(([url]) => url);
+}
+
+/**
+ * Analyze PageRank sinks (pages that receive links but don't link out)
+ */
+export function detectPageRankSinks(links: InternalLink[]): string[] {
+  const outboundPages = new Set(links.map((l) => l.source));
+  const inboundPages = new Set(links.map((l) => l.target));
+
+  // Sinks: pages that receive links but don't link out
+  return [...inboundPages].filter((url) => !outboundPages.has(url));
+}
+
+/**
+ * Main function: Analyze internal link structure
+ */
+export function analyzeInternalLinkGraph(
+  html: string,
+  url: string
+): { issues: AuditIssue[]; data: Partial<InternalLinkGraphData> } {
+  const issues: AuditIssue[] = [];
+  const links = extractInternalLinks(html, url);
+
+  // Count link types
+  const navigationLinks = links.filter((l) => l.context === 'navigation').length;
+  const contentLinks = links.filter((l) => l.context === 'content').length;
+  const footerLinks = links.filter((l) => l.context === 'footer').length;
+  const editorialLinks = links.filter((l) => l.isEditorial).length;
+
+  const totalLinks = links.length;
+  const contextualLinkRatio = totalLinks > 0 ? contentLinks / totalLinks : 0;
+  const navigationVsContentRatio = contentLinks > 0 ? navigationLinks / contentLinks : 0;
+
+  // Check for issues on single page
+  if (contentLinks === 0 && totalLinks > 0) {
+    issues.push({
+      code: 'NO_CONTEXTUAL_INTERNAL_LINKS',
+      severity: 'warning',
+      category: 'links',
+      title: 'No contextual internal links',
+      description: 'All internal links are in navigation/footer. No editorial links in content.',
+      impact: 'Editorial links in content pass more SEO value than navigational links.',
+      howToFix: 'Add relevant internal links within your main content to related pages.',
+      affectedUrls: [url],
+      details: { navigationLinks, contentLinks, footerLinks },
+    });
+  }
+
+  if (editorialLinks < 2 && contentLinks > 0) {
+    issues.push({
+      code: 'LOW_EDITORIAL_LINKS',
+      severity: 'notice',
+      category: 'links',
+      title: 'Few editorial internal links',
+      description: `Only ${editorialLinks} editorial (in-content) internal links found.`,
+      impact: 'Editorial links with surrounding context provide stronger topical signals.',
+      howToFix: 'Add 2-5 relevant internal links within your body content.',
+      affectedUrls: [url],
+    });
+  }
+
+  // Analyze first link anchors
+  const firstLinks = analyzeFirstLinkPriority(links);
+
+  // Check for generic anchor texts in first links
+  const genericAnchors = ['click here', 'read more', 'learn more', 'here', 'this'];
+  for (const [target, linkInfo] of firstLinks) {
+    if (genericAnchors.some((g) => linkInfo.anchor.toLowerCase().includes(g))) {
+      issues.push({
+        code: 'FIRST_LINK_GENERIC_ANCHOR',
+        severity: 'notice',
+        category: 'links',
+        title: 'First link uses generic anchor text',
+        description: `First link to ${target} uses generic anchor "${linkInfo.anchor}".`,
+        impact: 'Google may prioritize first link anchor for topic signals.',
+        howToFix: 'Use descriptive, keyword-relevant anchor text for internal links.',
+        affectedUrls: [url],
+        details: { target, anchor: linkInfo.anchor },
+      });
+    }
+  }
+
+  // Check for link distribution (too many links)
+  if (totalLinks > 100) {
+    issues.push({
+      code: 'TOO_MANY_INTERNAL_LINKS',
+      severity: 'warning',
+      category: 'links',
+      title: 'Excessive internal links',
+      description: `Page has ${totalLinks} internal links, which may dilute PageRank distribution.`,
+      impact: 'Too many links reduce the value passed to each linked page.',
+      howToFix: 'Reduce internal links to the most important and relevant pages.',
+      affectedUrls: [url],
+      details: { totalLinks, navigationLinks, contentLinks, footerLinks },
+    });
+  }
+
+  // Check for no internal links at all
+  if (totalLinks === 0) {
+    issues.push({
+      code: 'NO_INTERNAL_LINKS',
+      severity: 'error',
+      category: 'links',
+      title: 'No internal links found',
+      description: 'Page has no internal links to other pages on the site.',
+      impact: 'Creates a dead end for users and search engine crawlers.',
+      howToFix: 'Add relevant internal links to related content.',
+      affectedUrls: [url],
+    });
+  }
+
+  return {
+    issues,
+    data: {
+      totalInternalLinks: totalLinks,
+      contextualLinkRatio,
+      navigationVsContentRatio,
+      firstLinkAnchors: new Map(
+        [...firstLinks.entries()].map(([k, v]) => [k, v.anchor])
+      ),
+      linkDistribution: [{ url, links: totalLinks }],
+    },
+  };
+}
+
+/**
+ * Multi-page analysis for topic clusters and pillar pages
+ */
+export function analyzeTopicClusters(
+  links: InternalLink[]
+): { pillarPages: string[]; clusters: Map<string, string[]> } {
+  // Calculate hub/authority scores
+  const { hubs, authorities } = calculateHubAuthority(links);
+
+  // Pillar pages have high hub scores (link to many cluster pages)
+  const pillarThreshold = 0.7;
+  const pillarPages = [...hubs.entries()]
+    .filter(([_, score]) => score >= pillarThreshold)
+    .map(([url]) => url);
+
+  // Group pages by their primary pillar (page they link to most with highest authority)
+  const clusters = new Map<string, string[]>();
+
+  for (const pillar of pillarPages) {
+    const clusterPages = links
+      .filter((l) => l.target === pillar || l.source === pillar)
+      .map((l) => (l.source === pillar ? l.target : l.source))
+      .filter((url) => !pillarPages.includes(url));
+
+    clusters.set(pillar, [...new Set(clusterPages)]);
+  }
+
+  return { pillarPages, clusters };
+}

package/src/audit/checks/keyword-analysis.ts

@@ -0,0 +1,239 @@
+// Keyword Density Analysis
+// Analyzes keyword usage and distribution across page elements
+
+import * as cheerio from 'cheerio';
+import type { AuditIssue } from '../types.js';
+import { ISSUE_DEFINITIONS } from '../types.js';
+
+// Common English stop words to filter out
+const STOP_WORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with',
+  'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been', 'be', 'have', 'has', 'had',
+  'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must',
+  'can', 'this', 'that', 'these', 'those', 'it', 'its', 'they', 'them', 'their',
+  'we', 'us', 'our', 'you', 'your', 'i', 'me', 'my', 'he', 'she', 'him', 'her', 'his',
+  'not', 'no', 'nor', 'so', 'if', 'then', 'else', 'when', 'where', 'why', 'how', 'what',
+  'who', 'which', 'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some',
+  'such', 'only', 'own', 'same', 'than', 'too', 'very', 'just', 'also', 'now', 'here',
+  'there', 'about', 'after', 'before', 'above', 'below', 'between', 'into', 'through',
+  'during', 'under', 'again', 'further', 'once', 'any', 'being', 'because', 'while',
+]);
+
+export interface KeywordInfo {
+  word: string;
+  count: number;
+  density: number; // Percentage
+  inTitle: boolean;
+  inH1: boolean;
+  inH2: boolean;
+  inMetaDesc: boolean;
+}
+
+export interface KeywordPhraseInfo {
+  phrase: string;
+  count: number;
+  wordCount: number;
+}
+
+export interface KeywordAnalysisData {
+  totalWords: number;
+  uniqueWords: number;
+  topKeywords: KeywordInfo[];
+  topPhrases: {
+    twoWord: KeywordPhraseInfo[];
+    threeWord: KeywordPhraseInfo[];
+    fourWord: KeywordPhraseInfo[];
+  };
+  keywordDistribution: {
+    title: string[];
+    h1: string[];
+    h2: string[];
+    metaDesc: string[];
+  };
+  potentialStuffing: string[];
+}
+
+/**
+ * Extract text content from an element
+ */
+function extractText($: cheerio.CheerioAPI, selector: string): string {
+  return $(selector)
+    .text()
+    .toLowerCase()
+    .replace(/[^\w\s]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+/**
+ * Tokenize text into words
+ */
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s]/g, ' ')
+    .split(/\s+/)
+    .filter((word) => word.length > 2 && !STOP_WORDS.has(word) && !/^\d+$/.test(word));
+}
+
+/**
+ * Count word frequencies
+ */
+function countWords(words: string[]): Map<string, number> {
+  const counts = new Map<string, number>();
+  for (const word of words) {
+    counts.set(word, (counts.get(word) || 0) + 1);
+  }
+  return counts;
+}
+
+/**
+ * Extract n-grams (phrases)
+ */
+function extractNGrams(words: string[], n: number): Map<string, number> {
+  const ngrams = new Map<string, number>();
+
+  for (let i = 0; i <= words.length - n; i++) {
+    const ngram = words.slice(i, i + n).join(' ');
+    // Skip if any word is a stop word
+    const ngramWords = ngram.split(' ');
+    if (ngramWords.some((w) => STOP_WORDS.has(w))) continue;
+
+    ngrams.set(ngram, (ngrams.get(ngram) || 0) + 1);
+  }
+
+  return ngrams;
+}
+
+/**
+ * Analyze keyword density and distribution
+ */
+export function analyzeKeywords(html: string, url: string): { issues: AuditIssue[]; data: KeywordAnalysisData } {
+  const issues: AuditIssue[] = [];
+  const $ = cheerio.load(html);
+
+  // Extract text from different elements
+  const title = extractText($, 'title');
+  const h1 = extractText($, 'h1');
+  const h2 = extractText($, 'h2');
+  const metaDesc = $('meta[name="description"]').attr('content')?.toLowerCase() || '';
+
+  // Remove script and style content
+  $('script, style, noscript').remove();
+  const bodyText = extractText($, 'body');
+
+  // Tokenize
+  const bodyWords = tokenize(bodyText);
+  const titleWords = tokenize(title);
+  const h1Words = tokenize(h1);
+  const h2Words = tokenize(h2);
+  const metaDescWords = tokenize(metaDesc);
+
+  // Count words
+  const wordCounts = countWords(bodyWords);
+  const totalWords = bodyWords.length;
+
+  // Calculate keyword density
+  const topKeywordsMap: KeywordInfo[] = [];
+
+  for (const [word, count] of wordCounts) {
+    const density = (count / totalWords) * 100;
+    topKeywordsMap.push({
+      word,
+      count,
+      density: Math.round(density * 100) / 100,
+      inTitle: titleWords.includes(word),
+      inH1: h1Words.includes(word),
+      inH2: h2Words.includes(word),
+      inMetaDesc: metaDescWords.includes(word),
+    });
+  }
+
+  // Sort by count and take top 20
+  topKeywordsMap.sort((a, b) => b.count - a.count);
+  const topKeywords = topKeywordsMap.slice(0, 20);
+
+  // Extract n-grams
+  const twoWordPhrases = extractNGrams(bodyWords, 2);
+  const threeWordPhrases = extractNGrams(bodyWords, 3);
+  const fourWordPhrases = extractNGrams(bodyWords, 4);
+
+  // Convert to sorted arrays
+  const sortPhrases = (phrases: Map<string, number>, n: number): KeywordPhraseInfo[] => {
+    return Array.from(phrases)
+      .filter(([_, count]) => count >= 2)
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 10)
+      .map(([phrase, count]) => ({ phrase, count, wordCount: n }));
+  };
+
+  // Check for keyword stuffing (density > 3%)
+  const potentialStuffing = topKeywords.filter((k) => k.density > 3).map((k) => k.word);
+
+  // Generate issues
+  if (potentialStuffing.length > 0) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.KEYWORD_STUFFING,
+      affectedUrls: [url],
+      details: {
+        keywords: potentialStuffing,
+        densities: potentialStuffing.map((k) => {
+          const info = topKeywords.find((tk) => tk.word === k);
+          return { word: k, density: info?.density };
+        }),
+      },
+    });
+  }
+
+  // Check if top keywords are in title
+  const topBodyKeywords = topKeywords.slice(0, 5).map((k) => k.word);
+  const keywordsInTitle = topBodyKeywords.filter((k) => titleWords.includes(k));
+
+  if (keywordsInTitle.length === 0 && topBodyKeywords.length > 0) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.NO_KEYWORDS_IN_TITLE,
+      affectedUrls: [url],
+      details: {
+        topKeywords: topBodyKeywords,
+        title,
+        recommendation: `Consider including "${topBodyKeywords[0]}" in your title`,
+      },
+    });
+  }
+
+  // Check if top keywords are in H1
+  const keywordsInH1 = topBodyKeywords.filter((k) => h1Words.includes(k));
+
+  if (keywordsInH1.length === 0 && topBodyKeywords.length > 0 && h1) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.NO_KEYWORDS_IN_H1,
+      affectedUrls: [url],
+      details: {
+        topKeywords: topBodyKeywords,
+        h1,
+        recommendation: `Consider including "${topBodyKeywords[0]}" in your H1`,
+      },
+    });
+  }
+
+  return {
+    issues,
+    data: {
+      totalWords,
+      uniqueWords: wordCounts.size,
+      topKeywords,
+      topPhrases: {
+        twoWord: sortPhrases(twoWordPhrases, 2),
+        threeWord: sortPhrases(threeWordPhrases, 3),
+        fourWord: sortPhrases(fourWordPhrases, 4),
+      },
+      keywordDistribution: {
+        title: titleWords,
+        h1: h1Words,
+        h2: h2Words,
+        metaDesc: metaDescWords,
+      },
+      potentialStuffing,
+    },
+  };
+}