@rankcli/agent-runtime 0.0.14 → 0.0.16

package/src/audit/checks/canonical-advanced.ts

@@ -1,150 +0,0 @@
-import * as cheerio from 'cheerio';
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-import { ISSUE_DEFINITIONS } from '../types.js';
-
-export interface CanonicalData {
-  canonical?: string;
-  isSelfReferencing: boolean;
-  isCrossDomain: boolean;
-  canonicalChain?: string[];
-  finalCanonical?: string;
-}
-
-export async function analyzeCanonicalAdvanced(
-  html: string,
-  url: string,
-  options: { checkChain?: boolean; sitemapUrls?: string[] } = {}
-): Promise<{ issues: AuditIssue[]; data: CanonicalData }> {
-  const issues: AuditIssue[] = [];
-  const $ = cheerio.load(html);
-
-  const canonicalLink = $('link[rel="canonical"]').attr('href');
-  const baseUrl = new URL(url);
-
-  const data: CanonicalData = {
-    canonical: canonicalLink,
-    isSelfReferencing: false,
-    isCrossDomain: false,
-  };
-
-  if (!canonicalLink) {
-    return { issues, data };
-  }
-
-  // Resolve canonical URL
-  let canonicalUrl: URL;
-  try {
-    canonicalUrl = new URL(canonicalLink, url);
-  } catch {
-    return { issues, data };
-  }
-
-  // Check if self-referencing
-  const normalizedUrl = normalizeUrl(url);
-  const normalizedCanonical = normalizeUrl(canonicalUrl.href);
-  data.isSelfReferencing = normalizedUrl === normalizedCanonical;
-
-  // Check for cross-domain canonical
-  if (canonicalUrl.hostname !== baseUrl.hostname) {
-    data.isCrossDomain = true;
-    issues.push({
-      ...ISSUE_DEFINITIONS.CANONICAL_CROSS_DOMAIN,
-      affectedUrls: [url],
-      details: {
-        canonical: canonicalUrl.href,
-        sourceDomain: baseUrl.hostname,
-        targetDomain: canonicalUrl.hostname,
-      },
-    });
-  }
-
-  // Check for canonical chain if enabled
-  if (options.checkChain && !data.isSelfReferencing) {
-    const chain = await followCanonicalChain(canonicalUrl.href, [url]);
-    if (chain.length > 2) {
-      data.canonicalChain = chain;
-      data.finalCanonical = chain[chain.length - 1];
-      issues.push({
-        ...ISSUE_DEFINITIONS.CANONICAL_CHAIN,
-        affectedUrls: [url],
-        details: {
-          chain,
-          depth: chain.length,
-        },
-      });
-    }
-  }
-
-  // Check if canonical matches sitemap
-  if (options.sitemapUrls && options.sitemapUrls.length > 0) {
-    const sitemapNormalized = options.sitemapUrls.map(normalizeUrl);
-    const urlInSitemap = sitemapNormalized.includes(normalizedUrl);
-    const canonicalInSitemap = sitemapNormalized.includes(normalizedCanonical);
-
-    if (urlInSitemap && !canonicalInSitemap && !data.isSelfReferencing) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.CANONICAL_MISMATCH_SITEMAP,
-        affectedUrls: [url],
-        details: {
-          pageUrl: url,
-          canonical: canonicalUrl.href,
-          message: 'Page is in sitemap but its canonical URL is not',
-        },
-      });
-    }
-  }
-
-  return { issues, data };
-}
-
-async function followCanonicalChain(
-  url: string,
-  chain: string[] = [],
-  maxDepth: number = 5
-): Promise<string[]> {
-  if (chain.length >= maxDepth) return chain;
-
-  try {
-    const response = await httpGet<string>(url, {
-      timeout: 5000,
-      maxRedirects: 5,
-      validateStatus: (status) => status < 400,
-    });
-
-    const $ = cheerio.load(response.data);
-    const nextCanonical = $('link[rel="canonical"]').attr('href');
-
-    if (!nextCanonical) {
-      return [...chain, url];
-    }
-
-    const resolvedCanonical = new URL(nextCanonical, url).href;
-    const normalizedCurrent = normalizeUrl(url);
-    const normalizedNext = normalizeUrl(resolvedCanonical);
-
-    // Self-referencing, end of chain
-    if (normalizedCurrent === normalizedNext) {
-      return [...chain, url];
-    }
-
-    // Circular reference detection
-    if (chain.some(u => normalizeUrl(u) === normalizedNext)) {
-      return [...chain, url, resolvedCanonical + ' (circular)'];
-    }
-
-    return followCanonicalChain(resolvedCanonical, [...chain, url], maxDepth);
-  } catch {
-    return [...chain, url];
-  }
-}
-
-function normalizeUrl(urlStr: string): string {
-  try {
-    const parsed = new URL(urlStr);
-    // Remove trailing slash, lowercase hostname
-    return `${parsed.protocol}//${parsed.hostname.toLowerCase()}${parsed.pathname.replace(/\/$/, '')}`;
-  } catch {
-    return urlStr.toLowerCase().replace(/\/$/, '');
-  }
-}

package/src/audit/checks/canonical-domain.ts

@@ -1,196 +0,0 @@
-/**
- * Canonical Domain Check
- *
- * Verifies that the site properly handles www vs non-www versions
- * to avoid duplicate content issues.
- *
- * Best practice: One version should redirect to the other (301 redirect).
- * Both versions serving content = duplicate content problem.
- */
-
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-
-export interface CanonicalDomainData {
-  primaryDomain: string;
-  wwwVersion: {
-    url: string;
-    status: number | null;
-    redirectsTo?: string;
-    accessible: boolean;
-  };
-  nonWwwVersion: {
-    url: string;
-    status: number | null;
-    redirectsTo?: string;
-    accessible: boolean;
-  };
-  hasProperRedirect: boolean;
-  preferredVersion: 'www' | 'non-www' | 'unknown';
-  httpsRedirect: {
-    httpToHttps: boolean;
-    httpsStatus: number | null;
-  };
-}
-
-export async function analyzeCanonicalDomain(
-  url: string
-): Promise<{ issues: AuditIssue[]; data: CanonicalDomainData }> {
-  const issues: AuditIssue[] = [];
-  const parsedUrl = new URL(url);
-  const hostname = parsedUrl.hostname;
-
-  // Determine www and non-www versions
-  const isWww = hostname.startsWith('www.');
-  const wwwHostname = isWww ? hostname : `www.${hostname}`;
-  const nonWwwHostname = isWww ? hostname.substring(4) : hostname;
-
-  // Skip if hostname has multiple subdomains (e.g., api.staging.example.com)
-  const parts = nonWwwHostname.split('.');
-  if (parts.length > 2 && !nonWwwHostname.match(/\.(co|com|org|net|gov)\.[a-z]{2}$/i)) {
-    // This is a subdomain, not a main domain
-    return {
-      issues: [],
-      data: {
-        primaryDomain: hostname,
-        wwwVersion: { url: '', status: null, accessible: false },
-        nonWwwVersion: { url: '', status: null, accessible: false },
-        hasProperRedirect: true, // N/A for subdomains
-        preferredVersion: 'unknown',
-        httpsRedirect: { httpToHttps: false, httpsStatus: null },
-      },
-    };
-  }
-
-  const wwwUrl = `https://${wwwHostname}${parsedUrl.pathname}`;
-  const nonWwwUrl = `https://${nonWwwHostname}${parsedUrl.pathname}`;
-
-  // Test both versions
-  const [wwwResult, nonWwwResult] = await Promise.all([
-    testUrl(wwwUrl),
-    testUrl(nonWwwUrl),
-  ]);
-
-  // Determine if proper redirect exists
-  let hasProperRedirect = false;
-  let preferredVersion: 'www' | 'non-www' | 'unknown' = 'unknown';
-
-  // Check if www redirects to non-www
-  if (wwwResult.status && wwwResult.status >= 300 && wwwResult.status < 400) {
-    if (wwwResult.redirectsTo?.includes(nonWwwHostname)) {
-      hasProperRedirect = true;
-      preferredVersion = 'non-www';
-    }
-  }
-
-  // Check if non-www redirects to www
-  if (nonWwwResult.status && nonWwwResult.status >= 300 && nonWwwResult.status < 400) {
-    if (nonWwwResult.redirectsTo?.includes(wwwHostname)) {
-      hasProperRedirect = true;
-      preferredVersion = 'www';
-    }
-  }
-
-  // Both accessible without redirect = problem
-  const bothAccessible =
-    wwwResult.accessible &&
-    nonWwwResult.accessible &&
-    wwwResult.status === 200 &&
-    nonWwwResult.status === 200;
-
-  if (bothAccessible && !hasProperRedirect) {
-    issues.push({
-      code: 'CANONICAL_WWW_DUPLICATE',
-      severity: 'warning',
-      category: 'crawlability',
-      title: 'Both www and non-www versions accessible',
-      description: `Both ${wwwUrl} and ${nonWwwUrl} return 200 OK. This creates duplicate content.`,
-      impact: 'Search engines may index both versions, diluting page authority and causing ranking issues.',
-      howToFix: 'Set up a 301 redirect from one version to the other. Most sites prefer non-www, but either works. Configure at server/CDN level.',
-      affectedUrls: [wwwUrl, nonWwwUrl],
-      details: {
-        wwwStatus: wwwResult.status,
-        nonWwwStatus: nonWwwResult.status,
-        recommendation: 'Choose one version as canonical and 301 redirect the other.',
-      },
-    });
-  }
-
-  // Check HTTP to HTTPS redirect
-  const httpUrl = `http://${hostname}${parsedUrl.pathname}`;
-  const httpResult = await testUrl(httpUrl);
-
-  let httpToHttps = false;
-  if (httpResult.status && httpResult.status >= 300 && httpResult.status < 400) {
-    if (httpResult.redirectsTo?.startsWith('https://')) {
-      httpToHttps = true;
-    }
-  }
-
-  if (!httpToHttps && httpResult.accessible && httpResult.status === 200) {
-    issues.push({
-      code: 'CANONICAL_NO_HTTPS_REDIRECT',
-      severity: 'warning',
-      category: 'security',
-      title: 'HTTP does not redirect to HTTPS',
-      description: 'The HTTP version of the site is accessible without redirecting to HTTPS.',
-      impact: 'Duplicate content issues and security concerns. Users may access insecure version.',
-      howToFix: 'Configure a 301 redirect from HTTP to HTTPS at the server or CDN level.',
-      affectedUrls: [httpUrl],
-    });
-  }
-
-  return {
-    issues,
-    data: {
-      primaryDomain: hostname,
-      wwwVersion: {
-        url: wwwUrl,
-        status: wwwResult.status,
-        redirectsTo: wwwResult.redirectsTo,
-        accessible: wwwResult.accessible,
-      },
-      nonWwwVersion: {
-        url: nonWwwUrl,
-        status: nonWwwResult.status,
-        redirectsTo: nonWwwResult.redirectsTo,
-        accessible: nonWwwResult.accessible,
-      },
-      hasProperRedirect,
-      preferredVersion,
-      httpsRedirect: {
-        httpToHttps,
-        httpsStatus: httpResult.status,
-      },
-    },
-  };
-}
-
-async function testUrl(url: string): Promise<{
-  status: number | null;
-  redirectsTo?: string;
-  accessible: boolean;
-}> {
-  try {
-    const response = await httpGet<string>(url, {
-      timeout: 10000,
-      maxRedirects: 0, // Don't follow redirects
-      validateStatus: () => true,
-    });
-
-    const status = response.status;
-    const location = response.headers.location || response.headers.Location;
-
-    return {
-      status,
-      redirectsTo: location,
-      accessible: status !== null && status < 500,
-    };
-  } catch (error) {
-    // DNS resolution failure, connection refused, etc.
-    return {
-      status: null,
-      accessible: false,
-    };
-  }
-}

package/src/audit/checks/citation-quality.ts

@@ -1,358 +0,0 @@
-/**
- * Citation Quality Checks
- *
- * AI prioritizes content that cites reputable sources. Well-cited content
- * demonstrates expertise and trustworthiness (E-E-A-T signals).
- * These checks verify proper citation practices.
- */
-
-import * as cheerio from 'cheerio';
-import type { AuditIssue } from '../types.js';
-
-// Reputable source domains by category
-const REPUTABLE_SOURCES = {
-  academic: [
-    'scholar.google.com', 'pubmed.ncbi.nlm.nih.gov', 'ncbi.nlm.nih.gov',
-    'arxiv.org', 'jstor.org', 'researchgate.net', 'sciencedirect.com',
-    'nature.com', 'science.org', 'springer.com', 'wiley.com',
-  ],
-  government: [
-    'gov', 'gov.uk', 'europa.eu', 'un.org', 'who.int', 'cdc.gov',
-    'nih.gov', 'fda.gov', 'epa.gov', 'sec.gov', 'ftc.gov',
-  ],
-  educational: [
-    '.edu', 'mit.edu', 'stanford.edu', 'harvard.edu', 'oxford.ac.uk',
-    'cambridge.org', 'coursera.org', 'khanacademy.org',
-  ],
-  authoritative: [
-    'wikipedia.org', 'britannica.com', 'statista.com', 'pewresearch.org',
-    'gallup.com', 'mckinsey.com', 'hbr.org', 'forbes.com', 'reuters.com',
-    'apnews.com', 'bbc.com', 'nytimes.com', 'wsj.com', 'economist.com',
-  ],
-  technical: [
-    'github.com', 'stackoverflow.com', 'developer.mozilla.org', 'w3.org',
-    'ietf.org', 'rfc-editor.org', 'docs.google.com', 'developers.google.com',
-  ],
-};
-
-export interface CitationQualityData {
-  totalLinks: number;
-  externalLinks: number;
-  citationLinks: number;
-  reputableSources: {
-    academic: string[];
-    government: string[];
-    educational: string[];
-    authoritative: string[];
-    technical: string[];
-  };
-  citationPatterns: {
-    hasInlineCitations: boolean;
-    hasReferencesSection: boolean;
-    hasStatistics: boolean;
-    statisticsWithSources: number;
-    statisticsWithoutSources: number;
-  };
-  citationScore: number;
-}
-
-export function analyzeCitationQuality(
-  html: string,
-  url: string
-): { issues: AuditIssue[]; data: CitationQualityData } {
-  const issues: AuditIssue[] = [];
-  const $ = cheerio.load(html);
-  const parsedUrl = new URL(url);
-  const currentDomain = parsedUrl.hostname;
-
-  // Remove nav, footer, aside for content analysis
-  const $content = $('body').clone();
-  $content.find('nav, footer, aside, script, style, noscript').remove();
-
-  // Collect all links
-  const links = $content.find('a[href]');
-  let totalLinks = 0;
-  let externalLinks = 0;
-  let citationLinks = 0;
-
-  const reputableSources: CitationQualityData['reputableSources'] = {
-    academic: [],
-    government: [],
-    educational: [],
-    authoritative: [],
-    technical: [],
-  };
-
-  links.each((_, link) => {
-    const href = $(link).attr('href');
-    if (!href) return;
-
-    totalLinks++;
-
-    try {
-      const linkUrl = new URL(href, url);
-
-      // Skip internal links
-      if (linkUrl.hostname === currentDomain || linkUrl.hostname.endsWith(`.${currentDomain}`)) {
-        return;
-      }
-
-      // Skip common non-citation links
-      if (linkUrl.hostname.includes('facebook.com') ||
-          linkUrl.hostname.includes('twitter.com') ||
-          linkUrl.hostname.includes('instagram.com') ||
-          linkUrl.hostname.includes('linkedin.com/share') ||
-          linkUrl.hostname.includes('pinterest.com')) {
-        return;
-      }
-
-      externalLinks++;
-
-      // Check if it's a reputable source
-      const hostname = linkUrl.hostname.toLowerCase();
-
-      for (const source of REPUTABLE_SOURCES.academic) {
-        if (hostname.includes(source)) {
-          reputableSources.academic.push(hostname);
-          citationLinks++;
-          return;
-        }
-      }
-
-      for (const source of REPUTABLE_SOURCES.government) {
-        if (hostname.endsWith(source) || hostname.includes(`.${source}`)) {
-          reputableSources.government.push(hostname);
-          citationLinks++;
-          return;
-        }
-      }
-
-      for (const source of REPUTABLE_SOURCES.educational) {
-        if (hostname.endsWith(source) || hostname.includes(source)) {
-          reputableSources.educational.push(hostname);
-          citationLinks++;
-          return;
-        }
-      }
-
-      for (const source of REPUTABLE_SOURCES.authoritative) {
-        if (hostname.includes(source)) {
-          reputableSources.authoritative.push(hostname);
-          citationLinks++;
-          return;
-        }
-      }
-
-      for (const source of REPUTABLE_SOURCES.technical) {
-        if (hostname.includes(source)) {
-          reputableSources.technical.push(hostname);
-          citationLinks++;
-          return;
-        }
-      }
-
-    } catch {
-      // Invalid URL
-    }
-  });
-
-  // Check for citation patterns
-  const bodyText = $content.text();
-  const bodyHtml = $content.html() || '';
-
-  // Inline citations like [1], (Source: X), (2023), etc.
-  const hasInlineCitations =
-    /\[\d+\]/.test(bodyText) ||
-    /\(Source:/i.test(bodyText) ||
-    /\(Citation:/i.test(bodyText) ||
-    /according to/i.test(bodyText) ||
-    /study (by|from|shows|found)/i.test(bodyText);
-
-  // References section
-  const hasReferencesSection =
-    /(references|sources|citations|bibliography|works cited)/i.test(
-      $('h2, h3, h4, h5, h6').text()
-    ) ||
-    $('#references, .references, #sources, .sources, #citations').length > 0;
-
-  // Statistics detection
-  const statisticPatterns = [
-    /(\d+(?:\.\d+)?)\s*%/g, // Percentages
-    /(\d+(?:,\d{3})*(?:\.\d+)?)\s*(million|billion|trillion)/gi, // Large numbers
-    /\$(\d+(?:,\d{3})*(?:\.\d+)?)/g, // Dollar amounts
-    /(\d+)\s*out of\s*(\d+)/gi, // Ratios
-    /(\d+)x\s/gi, // Multipliers
-  ];
-
-  let statisticsWithSources = 0;
-  let statisticsWithoutSources = 0;
-
-  // Find paragraphs with statistics
-  $content.find('p').each((_, p) => {
-    const pText = $(p).text();
-    const pHtml = $(p).html() || '';
-
-    let hasStatistic = false;
-    for (const pattern of statisticPatterns) {
-      if (pattern.test(pText)) {
-        hasStatistic = true;
-        pattern.lastIndex = 0; // Reset regex
-        break;
-      }
-    }
-
-    if (hasStatistic) {
-      // Check if paragraph has a citation link
-      const hasLink = $(p).find('a[href]').length > 0;
-      const hasCitationText = /\(.*\d{4}\)|according to|source:|study/i.test(pText);
-
-      if (hasLink || hasCitationText) {
-        statisticsWithSources++;
-      } else {
-        statisticsWithoutSources++;
-      }
-    }
-  });
-
-  const hasStatistics = statisticsWithSources + statisticsWithoutSources > 0;
-
-  // Calculate citation score (0-100)
-  let citationScore = 30; // Base score
-
-  // Reputable source bonuses
-  if (reputableSources.academic.length > 0) citationScore += 15;
-  if (reputableSources.government.length > 0) citationScore += 15;
-  if (reputableSources.educational.length > 0) citationScore += 10;
-  if (reputableSources.authoritative.length > 0) citationScore += 10;
-  if (reputableSources.technical.length > 0) citationScore += 5;
-
-  // Citation pattern bonuses
-  if (hasInlineCitations) citationScore += 10;
-  if (hasReferencesSection) citationScore += 10;
-
-  // Statistics citation bonus/penalty
-  if (hasStatistics) {
-    const citedRatio = statisticsWithSources / (statisticsWithSources + statisticsWithoutSources);
-    if (citedRatio >= 0.8) citationScore += 10;
-    else if (citedRatio >= 0.5) citationScore += 5;
-    else if (citedRatio < 0.3) citationScore -= 10;
-  }
-
-  // Penalty for no external links at all
-  if (externalLinks === 0) citationScore -= 20;
-
-  citationScore = Math.max(0, Math.min(100, citationScore));
-
-  // Generate issues
-
-  // No reputable sources cited
-  const totalReputable = Object.values(reputableSources).flat().length;
-  if (totalReputable === 0 && externalLinks > 0) {
-    issues.push({
-      code: 'AI_NO_REPUTABLE_CITATIONS',
-      severity: 'warning',
-      category: 'ai-readiness',
-      title: 'No reputable sources cited',
-      description: 'Content has external links but none to recognized authoritative sources. AI prioritizes content that cites reputable sources.',
-      impact: 'Lower E-E-A-T signals and reduced likelihood of AI citing your content.',
-      howToFix: 'Add citations to reputable sources: academic papers, government sites (.gov), educational institutions (.edu), or established publications.',
-      affectedUrls: [url],
-      details: {
-        externalLinks,
-        reputableLinks: 0,
-        suggestedSources: [
-          'Academic: scholar.google.com, pubmed.ncbi.nlm.nih.gov',
-          'Government: .gov, who.int, cdc.gov',
-          'Technical: developer.mozilla.org, w3.org',
-        ],
-      },
-    });
-  }
-
-  // No external links at all
-  if (externalLinks === 0 && totalLinks > 0) {
-    issues.push({
-      code: 'AI_NO_EXTERNAL_CITATIONS',
-      severity: 'warning',
-      category: 'ai-readiness',
-      title: 'No external sources linked',
-      description: 'Content has no external links. AI values content that references and links to authoritative sources.',
-      impact: 'Appears self-contained without external validation, reducing trust signals for AI.',
-      howToFix: 'Add relevant external links to support your claims. Link to studies, official documentation, or authoritative sources.',
-      affectedUrls: [url],
-    });
-  }
-
-  // Statistics without sources
-  if (statisticsWithoutSources > 0) {
-    issues.push({
-      code: 'AI_UNCITED_STATISTICS',
-      severity: 'warning',
-      category: 'ai-readiness',
-      title: 'Statistics without source citations',
-      description: `Found ${statisticsWithoutSources} statistic(s) without linked sources. AI is trained to prefer claims backed by citations.`,
-      impact: 'Uncited statistics reduce content credibility and E-E-A-T signals.',
-      howToFix: 'Add source links or citations for all statistics. Include the source name and year when possible (e.g., "according to [Source, 2024]").',
-      affectedUrls: [url],
-      details: {
-        statisticsWithSources,
-        statisticsWithoutSources,
-        citedRatio: statisticsWithSources / (statisticsWithSources + statisticsWithoutSources),
-      },
-    });
-  }
-
-  // No references section for content-heavy pages
-  const wordCount = bodyText.split(/\s+/).length;
-  if (wordCount > 1000 && !hasReferencesSection && citationLinks > 0) {
-    issues.push({
-      code: 'AI_NO_REFERENCES_SECTION',
-      severity: 'notice',
-      category: 'ai-readiness',
-      title: 'No dedicated references section',
-      description: 'Long-form content with citations but no dedicated references section. A references section signals academic rigor.',
-      impact: 'Missing opportunity to demonstrate comprehensive research and boost E-E-A-T.',
-      howToFix: 'Add a "References" or "Sources" section at the end listing all cited sources with full attribution.',
-      affectedUrls: [url],
-    });
-  }
-
-  // Low citation score
-  if (citationScore < 40) {
-    issues.push({
-      code: 'AI_LOW_CITATION_QUALITY',
-      severity: 'warning',
-      category: 'ai-readiness',
-      title: 'Content citation quality needs improvement',
-      description: `Citation quality score: ${citationScore}/100. Well-cited content ranks higher in AI search results.`,
-      impact: 'Low citation quality signals reduce E-E-A-T and AI citation likelihood.',
-      howToFix: 'Improve citations: 1) Link to reputable sources, 2) Cite statistics with sources, 3) Add a references section, 4) Use inline citations for claims.',
-      affectedUrls: [url],
-      details: {
-        citationScore,
-        externalLinks,
-        reputableSources: totalReputable,
-        statisticsWithSources,
-        statisticsWithoutSources,
-      },
-    });
-  }
-
-  return {
-    issues,
-    data: {
-      totalLinks,
-      externalLinks,
-      citationLinks,
-      reputableSources,
-      citationPatterns: {
-        hasInlineCitations,
-        hasReferencesSection,
-        hasStatistics,
-        statisticsWithSources,
-        statisticsWithoutSources,
-      },
-      citationScore,
-    },
-  };
-}