@rankcli/agent-runtime 0.0.1

package/src/audit/checks/canonical-advanced.ts

@@ -0,0 +1,150 @@
+import * as cheerio from 'cheerio';
+import { httpGet } from '../../utils/http.js';
+import type { AuditIssue } from '../types.js';
+import { ISSUE_DEFINITIONS } from '../types.js';
+
+export interface CanonicalData {
+  canonical?: string;
+  isSelfReferencing: boolean;
+  isCrossDomain: boolean;
+  canonicalChain?: string[];
+  finalCanonical?: string;
+}
+
+export async function analyzeCanonicalAdvanced(
+  html: string,
+  url: string,
+  options: { checkChain?: boolean; sitemapUrls?: string[] } = {}
+): Promise<{ issues: AuditIssue[]; data: CanonicalData }> {
+  const issues: AuditIssue[] = [];
+  const $ = cheerio.load(html);
+
+  const canonicalLink = $('link[rel="canonical"]').attr('href');
+  const baseUrl = new URL(url);
+
+  const data: CanonicalData = {
+    canonical: canonicalLink,
+    isSelfReferencing: false,
+    isCrossDomain: false,
+  };
+
+  if (!canonicalLink) {
+    return { issues, data };
+  }
+
+  // Resolve canonical URL
+  let canonicalUrl: URL;
+  try {
+    canonicalUrl = new URL(canonicalLink, url);
+  } catch {
+    return { issues, data };
+  }
+
+  // Check if self-referencing
+  const normalizedUrl = normalizeUrl(url);
+  const normalizedCanonical = normalizeUrl(canonicalUrl.href);
+  data.isSelfReferencing = normalizedUrl === normalizedCanonical;
+
+  // Check for cross-domain canonical
+  if (canonicalUrl.hostname !== baseUrl.hostname) {
+    data.isCrossDomain = true;
+    issues.push({
+      ...ISSUE_DEFINITIONS.CANONICAL_CROSS_DOMAIN,
+      affectedUrls: [url],
+      details: {
+        canonical: canonicalUrl.href,
+        sourceDomain: baseUrl.hostname,
+        targetDomain: canonicalUrl.hostname,
+      },
+    });
+  }
+
+  // Check for canonical chain if enabled
+  if (options.checkChain && !data.isSelfReferencing) {
+    const chain = await followCanonicalChain(canonicalUrl.href, [url]);
+    if (chain.length > 2) {
+      data.canonicalChain = chain;
+      data.finalCanonical = chain[chain.length - 1];
+      issues.push({
+        ...ISSUE_DEFINITIONS.CANONICAL_CHAIN,
+        affectedUrls: [url],
+        details: {
+          chain,
+          depth: chain.length,
+        },
+      });
+    }
+  }
+
+  // Check if canonical matches sitemap
+  if (options.sitemapUrls && options.sitemapUrls.length > 0) {
+    const sitemapNormalized = options.sitemapUrls.map(normalizeUrl);
+    const urlInSitemap = sitemapNormalized.includes(normalizedUrl);
+    const canonicalInSitemap = sitemapNormalized.includes(normalizedCanonical);
+
+    if (urlInSitemap && !canonicalInSitemap && !data.isSelfReferencing) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.CANONICAL_MISMATCH_SITEMAP,
+        affectedUrls: [url],
+        details: {
+          pageUrl: url,
+          canonical: canonicalUrl.href,
+          message: 'Page is in sitemap but its canonical URL is not',
+        },
+      });
+    }
+  }
+
+  return { issues, data };
+}
+
+async function followCanonicalChain(
+  url: string,
+  chain: string[] = [],
+  maxDepth: number = 5
+): Promise<string[]> {
+  if (chain.length >= maxDepth) return chain;
+
+  try {
+    const response = await httpGet<string>(url, {
+      timeout: 5000,
+      maxRedirects: 5,
+      validateStatus: (status) => status < 400,
+    });
+
+    const $ = cheerio.load(response.data);
+    const nextCanonical = $('link[rel="canonical"]').attr('href');
+
+    if (!nextCanonical) {
+      return [...chain, url];
+    }
+
+    const resolvedCanonical = new URL(nextCanonical, url).href;
+    const normalizedCurrent = normalizeUrl(url);
+    const normalizedNext = normalizeUrl(resolvedCanonical);
+
+    // Self-referencing, end of chain
+    if (normalizedCurrent === normalizedNext) {
+      return [...chain, url];
+    }
+
+    // Circular reference detection
+    if (chain.some(u => normalizeUrl(u) === normalizedNext)) {
+      return [...chain, url, resolvedCanonical + ' (circular)'];
+    }
+
+    return followCanonicalChain(resolvedCanonical, [...chain, url], maxDepth);
+  } catch {
+    return [...chain, url];
+  }
+}
+
+function normalizeUrl(urlStr: string): string {
+  try {
+    const parsed = new URL(urlStr);
+    // Remove trailing slash, lowercase hostname
+    return `${parsed.protocol}//${parsed.hostname.toLowerCase()}${parsed.pathname.replace(/\/$/, '')}`;
+  } catch {
+    return urlStr.toLowerCase().replace(/\/$/, '');
+  }
+}

package/src/audit/checks/canonical-domain.ts

@@ -0,0 +1,196 @@
+/**
+ * Canonical Domain Check
+ *
+ * Verifies that the site properly handles www vs non-www versions
+ * to avoid duplicate content issues.
+ *
+ * Best practice: One version should redirect to the other (301 redirect).
+ * Both versions serving content = duplicate content problem.
+ */
+
+import { httpGet } from '../../utils/http.js';
+import type { AuditIssue } from '../types.js';
+
+export interface CanonicalDomainData {
+  primaryDomain: string;
+  wwwVersion: {
+    url: string;
+    status: number | null;
+    redirectsTo?: string;
+    accessible: boolean;
+  };
+  nonWwwVersion: {
+    url: string;
+    status: number | null;
+    redirectsTo?: string;
+    accessible: boolean;
+  };
+  hasProperRedirect: boolean;
+  preferredVersion: 'www' | 'non-www' | 'unknown';
+  httpsRedirect: {
+    httpToHttps: boolean;
+    httpsStatus: number | null;
+  };
+}
+
+export async function analyzeCanonicalDomain(
+  url: string
+): Promise<{ issues: AuditIssue[]; data: CanonicalDomainData }> {
+  const issues: AuditIssue[] = [];
+  const parsedUrl = new URL(url);
+  const hostname = parsedUrl.hostname;
+
+  // Determine www and non-www versions
+  const isWww = hostname.startsWith('www.');
+  const wwwHostname = isWww ? hostname : `www.${hostname}`;
+  const nonWwwHostname = isWww ? hostname.substring(4) : hostname;
+
+  // Skip if hostname has multiple subdomains (e.g., api.staging.example.com)
+  const parts = nonWwwHostname.split('.');
+  if (parts.length > 2 && !nonWwwHostname.match(/\.(co|com|org|net|gov)\.[a-z]{2}$/i)) {
+    // This is a subdomain, not a main domain
+    return {
+      issues: [],
+      data: {
+        primaryDomain: hostname,
+        wwwVersion: { url: '', status: null, accessible: false },
+        nonWwwVersion: { url: '', status: null, accessible: false },
+        hasProperRedirect: true, // N/A for subdomains
+        preferredVersion: 'unknown',
+        httpsRedirect: { httpToHttps: false, httpsStatus: null },
+      },
+    };
+  }
+
+  const wwwUrl = `https://${wwwHostname}${parsedUrl.pathname}`;
+  const nonWwwUrl = `https://${nonWwwHostname}${parsedUrl.pathname}`;
+
+  // Test both versions
+  const [wwwResult, nonWwwResult] = await Promise.all([
+    testUrl(wwwUrl),
+    testUrl(nonWwwUrl),
+  ]);
+
+  // Determine if proper redirect exists
+  let hasProperRedirect = false;
+  let preferredVersion: 'www' | 'non-www' | 'unknown' = 'unknown';
+
+  // Check if www redirects to non-www
+  if (wwwResult.status && wwwResult.status >= 300 && wwwResult.status < 400) {
+    if (wwwResult.redirectsTo?.includes(nonWwwHostname)) {
+      hasProperRedirect = true;
+      preferredVersion = 'non-www';
+    }
+  }
+
+  // Check if non-www redirects to www
+  if (nonWwwResult.status && nonWwwResult.status >= 300 && nonWwwResult.status < 400) {
+    if (nonWwwResult.redirectsTo?.includes(wwwHostname)) {
+      hasProperRedirect = true;
+      preferredVersion = 'www';
+    }
+  }
+
+  // Both accessible without redirect = problem
+  const bothAccessible =
+    wwwResult.accessible &&
+    nonWwwResult.accessible &&
+    wwwResult.status === 200 &&
+    nonWwwResult.status === 200;
+
+  if (bothAccessible && !hasProperRedirect) {
+    issues.push({
+      code: 'CANONICAL_WWW_DUPLICATE',
+      severity: 'warning',
+      category: 'crawlability',
+      title: 'Both www and non-www versions accessible',
+      description: `Both ${wwwUrl} and ${nonWwwUrl} return 200 OK. This creates duplicate content.`,
+      impact: 'Search engines may index both versions, diluting page authority and causing ranking issues.',
+      howToFix: 'Set up a 301 redirect from one version to the other. Most sites prefer non-www, but either works. Configure at server/CDN level.',
+      affectedUrls: [wwwUrl, nonWwwUrl],
+      details: {
+        wwwStatus: wwwResult.status,
+        nonWwwStatus: nonWwwResult.status,
+        recommendation: 'Choose one version as canonical and 301 redirect the other.',
+      },
+    });
+  }
+
+  // Check HTTP to HTTPS redirect
+  const httpUrl = `http://${hostname}${parsedUrl.pathname}`;
+  const httpResult = await testUrl(httpUrl);
+
+  let httpToHttps = false;
+  if (httpResult.status && httpResult.status >= 300 && httpResult.status < 400) {
+    if (httpResult.redirectsTo?.startsWith('https://')) {
+      httpToHttps = true;
+    }
+  }
+
+  if (!httpToHttps && httpResult.accessible && httpResult.status === 200) {
+    issues.push({
+      code: 'CANONICAL_NO_HTTPS_REDIRECT',
+      severity: 'warning',
+      category: 'security',
+      title: 'HTTP does not redirect to HTTPS',
+      description: 'The HTTP version of the site is accessible without redirecting to HTTPS.',
+      impact: 'Duplicate content issues and security concerns. Users may access insecure version.',
+      howToFix: 'Configure a 301 redirect from HTTP to HTTPS at the server or CDN level.',
+      affectedUrls: [httpUrl],
+    });
+  }
+
+  return {
+    issues,
+    data: {
+      primaryDomain: hostname,
+      wwwVersion: {
+        url: wwwUrl,
+        status: wwwResult.status,
+        redirectsTo: wwwResult.redirectsTo,
+        accessible: wwwResult.accessible,
+      },
+      nonWwwVersion: {
+        url: nonWwwUrl,
+        status: nonWwwResult.status,
+        redirectsTo: nonWwwResult.redirectsTo,
+        accessible: nonWwwResult.accessible,
+      },
+      hasProperRedirect,
+      preferredVersion,
+      httpsRedirect: {
+        httpToHttps,
+        httpsStatus: httpResult.status,
+      },
+    },
+  };
+}
+
+async function testUrl(url: string): Promise<{
+  status: number | null;
+  redirectsTo?: string;
+  accessible: boolean;
+}> {
+  try {
+    const response = await httpGet<string>(url, {
+      timeout: 10000,
+      maxRedirects: 0, // Don't follow redirects
+      validateStatus: () => true,
+    });
+
+    const status = response.status;
+    const location = response.headers.location || response.headers.Location;
+
+    return {
+      status,
+      redirectsTo: location,
+      accessible: status !== null && status < 500,
+    };
+  } catch (error) {
+    // DNS resolution failure, connection refused, etc.
+    return {
+      status: null,
+      accessible: false,
+    };
+  }
+}

package/src/audit/checks/citation-quality.ts

@@ -0,0 +1,358 @@
+/**
+ * Citation Quality Checks
+ *
+ * AI prioritizes content that cites reputable sources. Well-cited content
+ * demonstrates expertise and trustworthiness (E-E-A-T signals).
+ * These checks verify proper citation practices.
+ */
+
+import * as cheerio from 'cheerio';
+import type { AuditIssue } from '../types.js';
+
+// Reputable source domains by category
+const REPUTABLE_SOURCES = {
+  academic: [
+    'scholar.google.com', 'pubmed.ncbi.nlm.nih.gov', 'ncbi.nlm.nih.gov',
+    'arxiv.org', 'jstor.org', 'researchgate.net', 'sciencedirect.com',
+    'nature.com', 'science.org', 'springer.com', 'wiley.com',
+  ],
+  government: [
+    'gov', 'gov.uk', 'europa.eu', 'un.org', 'who.int', 'cdc.gov',
+    'nih.gov', 'fda.gov', 'epa.gov', 'sec.gov', 'ftc.gov',
+  ],
+  educational: [
+    '.edu', 'mit.edu', 'stanford.edu', 'harvard.edu', 'oxford.ac.uk',
+    'cambridge.org', 'coursera.org', 'khanacademy.org',
+  ],
+  authoritative: [
+    'wikipedia.org', 'britannica.com', 'statista.com', 'pewresearch.org',
+    'gallup.com', 'mckinsey.com', 'hbr.org', 'forbes.com', 'reuters.com',
+    'apnews.com', 'bbc.com', 'nytimes.com', 'wsj.com', 'economist.com',
+  ],
+  technical: [
+    'github.com', 'stackoverflow.com', 'developer.mozilla.org', 'w3.org',
+    'ietf.org', 'rfc-editor.org', 'docs.google.com', 'developers.google.com',
+  ],
+};
+
+export interface CitationQualityData {
+  totalLinks: number;
+  externalLinks: number;
+  citationLinks: number;
+  reputableSources: {
+    academic: string[];
+    government: string[];
+    educational: string[];
+    authoritative: string[];
+    technical: string[];
+  };
+  citationPatterns: {
+    hasInlineCitations: boolean;
+    hasReferencesSection: boolean;
+    hasStatistics: boolean;
+    statisticsWithSources: number;
+    statisticsWithoutSources: number;
+  };
+  citationScore: number;
+}
+
+export function analyzeCitationQuality(
+  html: string,
+  url: string
+): { issues: AuditIssue[]; data: CitationQualityData } {
+  const issues: AuditIssue[] = [];
+  const $ = cheerio.load(html);
+  const parsedUrl = new URL(url);
+  const currentDomain = parsedUrl.hostname;
+
+  // Remove nav, footer, aside for content analysis
+  const $content = $('body').clone();
+  $content.find('nav, footer, aside, script, style, noscript').remove();
+
+  // Collect all links
+  const links = $content.find('a[href]');
+  let totalLinks = 0;
+  let externalLinks = 0;
+  let citationLinks = 0;
+
+  const reputableSources: CitationQualityData['reputableSources'] = {
+    academic: [],
+    government: [],
+    educational: [],
+    authoritative: [],
+    technical: [],
+  };
+
+  links.each((_, link) => {
+    const href = $(link).attr('href');
+    if (!href) return;
+
+    totalLinks++;
+
+    try {
+      const linkUrl = new URL(href, url);
+
+      // Skip internal links
+      if (linkUrl.hostname === currentDomain || linkUrl.hostname.endsWith(`.${currentDomain}`)) {
+        return;
+      }
+
+      // Skip common non-citation links
+      if (linkUrl.hostname.includes('facebook.com') ||
+          linkUrl.hostname.includes('twitter.com') ||
+          linkUrl.hostname.includes('instagram.com') ||
+          linkUrl.hostname.includes('linkedin.com/share') ||
+          linkUrl.hostname.includes('pinterest.com')) {
+        return;
+      }
+
+      externalLinks++;
+
+      // Check if it's a reputable source
+      const hostname = linkUrl.hostname.toLowerCase();
+
+      for (const source of REPUTABLE_SOURCES.academic) {
+        if (hostname.includes(source)) {
+          reputableSources.academic.push(hostname);
+          citationLinks++;
+          return;
+        }
+      }
+
+      for (const source of REPUTABLE_SOURCES.government) {
+        if (hostname.endsWith(source) || hostname.includes(`.${source}`)) {
+          reputableSources.government.push(hostname);
+          citationLinks++;
+          return;
+        }
+      }
+
+      for (const source of REPUTABLE_SOURCES.educational) {
+        if (hostname.endsWith(source) || hostname.includes(source)) {
+          reputableSources.educational.push(hostname);
+          citationLinks++;
+          return;
+        }
+      }
+
+      for (const source of REPUTABLE_SOURCES.authoritative) {
+        if (hostname.includes(source)) {
+          reputableSources.authoritative.push(hostname);
+          citationLinks++;
+          return;
+        }
+      }
+
+      for (const source of REPUTABLE_SOURCES.technical) {
+        if (hostname.includes(source)) {
+          reputableSources.technical.push(hostname);
+          citationLinks++;
+          return;
+        }
+      }
+
+    } catch {
+      // Invalid URL
+    }
+  });
+
+  // Check for citation patterns
+  const bodyText = $content.text();
+  const bodyHtml = $content.html() || '';
+
+  // Inline citations like [1], (Source: X), (2023), etc.
+  const hasInlineCitations =
+    /\[\d+\]/.test(bodyText) ||
+    /\(Source:/i.test(bodyText) ||
+    /\(Citation:/i.test(bodyText) ||
+    /according to/i.test(bodyText) ||
+    /study (by|from|shows|found)/i.test(bodyText);
+
+  // References section
+  const hasReferencesSection =
+    /(references|sources|citations|bibliography|works cited)/i.test(
+      $('h2, h3, h4, h5, h6').text()
+    ) ||
+    $('#references, .references, #sources, .sources, #citations').length > 0;
+
+  // Statistics detection
+  const statisticPatterns = [
+    /(\d+(?:\.\d+)?)\s*%/g, // Percentages
+    /(\d+(?:,\d{3})*(?:\.\d+)?)\s*(million|billion|trillion)/gi, // Large numbers
+    /\$(\d+(?:,\d{3})*(?:\.\d+)?)/g, // Dollar amounts
+    /(\d+)\s*out of\s*(\d+)/gi, // Ratios
+    /(\d+)x\s/gi, // Multipliers
+  ];
+
+  let statisticsWithSources = 0;
+  let statisticsWithoutSources = 0;
+
+  // Find paragraphs with statistics
+  $content.find('p').each((_, p) => {
+    const pText = $(p).text();
+    const pHtml = $(p).html() || '';
+
+    let hasStatistic = false;
+    for (const pattern of statisticPatterns) {
+      if (pattern.test(pText)) {
+        hasStatistic = true;
+        pattern.lastIndex = 0; // Reset regex
+        break;
+      }
+    }
+
+    if (hasStatistic) {
+      // Check if paragraph has a citation link
+      const hasLink = $(p).find('a[href]').length > 0;
+      const hasCitationText = /\(.*\d{4}\)|according to|source:|study/i.test(pText);
+
+      if (hasLink || hasCitationText) {
+        statisticsWithSources++;
+      } else {
+        statisticsWithoutSources++;
+      }
+    }
+  });
+
+  const hasStatistics = statisticsWithSources + statisticsWithoutSources > 0;
+
+  // Calculate citation score (0-100)
+  let citationScore = 30; // Base score
+
+  // Reputable source bonuses
+  if (reputableSources.academic.length > 0) citationScore += 15;
+  if (reputableSources.government.length > 0) citationScore += 15;
+  if (reputableSources.educational.length > 0) citationScore += 10;
+  if (reputableSources.authoritative.length > 0) citationScore += 10;
+  if (reputableSources.technical.length > 0) citationScore += 5;
+
+  // Citation pattern bonuses
+  if (hasInlineCitations) citationScore += 10;
+  if (hasReferencesSection) citationScore += 10;
+
+  // Statistics citation bonus/penalty
+  if (hasStatistics) {
+    const citedRatio = statisticsWithSources / (statisticsWithSources + statisticsWithoutSources);
+    if (citedRatio >= 0.8) citationScore += 10;
+    else if (citedRatio >= 0.5) citationScore += 5;
+    else if (citedRatio < 0.3) citationScore -= 10;
+  }
+
+  // Penalty for no external links at all
+  if (externalLinks === 0) citationScore -= 20;
+
+  citationScore = Math.max(0, Math.min(100, citationScore));
+
+  // Generate issues
+
+  // No reputable sources cited
+  const totalReputable = Object.values(reputableSources).flat().length;
+  if (totalReputable === 0 && externalLinks > 0) {
+    issues.push({
+      code: 'AI_NO_REPUTABLE_CITATIONS',
+      severity: 'warning',
+      category: 'ai-readiness',
+      title: 'No reputable sources cited',
+      description: 'Content has external links but none to recognized authoritative sources. AI prioritizes content that cites reputable sources.',
+      impact: 'Lower E-E-A-T signals and reduced likelihood of AI citing your content.',
+      howToFix: 'Add citations to reputable sources: academic papers, government sites (.gov), educational institutions (.edu), or established publications.',
+      affectedUrls: [url],
+      details: {
+        externalLinks,
+        reputableLinks: 0,
+        suggestedSources: [
+          'Academic: scholar.google.com, pubmed.ncbi.nlm.nih.gov',
+          'Government: .gov, who.int, cdc.gov',
+          'Technical: developer.mozilla.org, w3.org',
+        ],
+      },
+    });
+  }
+
+  // No external links at all
+  if (externalLinks === 0 && totalLinks > 0) {
+    issues.push({
+      code: 'AI_NO_EXTERNAL_CITATIONS',
+      severity: 'warning',
+      category: 'ai-readiness',
+      title: 'No external sources linked',
+      description: 'Content has no external links. AI values content that references and links to authoritative sources.',
+      impact: 'Appears self-contained without external validation, reducing trust signals for AI.',
+      howToFix: 'Add relevant external links to support your claims. Link to studies, official documentation, or authoritative sources.',
+      affectedUrls: [url],
+    });
+  }
+
+  // Statistics without sources
+  if (statisticsWithoutSources > 0) {
+    issues.push({
+      code: 'AI_UNCITED_STATISTICS',
+      severity: 'warning',
+      category: 'ai-readiness',
+      title: 'Statistics without source citations',
+      description: `Found ${statisticsWithoutSources} statistic(s) without linked sources. AI is trained to prefer claims backed by citations.`,
+      impact: 'Uncited statistics reduce content credibility and E-E-A-T signals.',
+      howToFix: 'Add source links or citations for all statistics. Include the source name and year when possible (e.g., "according to [Source, 2024]").',
+      affectedUrls: [url],
+      details: {
+        statisticsWithSources,
+        statisticsWithoutSources,
+        citedRatio: statisticsWithSources / (statisticsWithSources + statisticsWithoutSources),
+      },
+    });
+  }
+
+  // No references section for content-heavy pages
+  const wordCount = bodyText.split(/\s+/).length;
+  if (wordCount > 1000 && !hasReferencesSection && citationLinks > 0) {
+    issues.push({
+      code: 'AI_NO_REFERENCES_SECTION',
+      severity: 'notice',
+      category: 'ai-readiness',
+      title: 'No dedicated references section',
+      description: 'Long-form content with citations but no dedicated references section. A references section signals academic rigor.',
+      impact: 'Missing opportunity to demonstrate comprehensive research and boost E-E-A-T.',
+      howToFix: 'Add a "References" or "Sources" section at the end listing all cited sources with full attribution.',
+      affectedUrls: [url],
+    });
+  }
+
+  // Low citation score
+  if (citationScore < 40) {
+    issues.push({
+      code: 'AI_LOW_CITATION_QUALITY',
+      severity: 'warning',
+      category: 'ai-readiness',
+      title: 'Content citation quality needs improvement',
+      description: `Citation quality score: ${citationScore}/100. Well-cited content ranks higher in AI search results.`,
+      impact: 'Low citation quality signals reduce E-E-A-T and AI citation likelihood.',
+      howToFix: 'Improve citations: 1) Link to reputable sources, 2) Cite statistics with sources, 3) Add a references section, 4) Use inline citations for claims.',
+      affectedUrls: [url],
+      details: {
+        citationScore,
+        externalLinks,
+        reputableSources: totalReputable,
+        statisticsWithSources,
+        statisticsWithoutSources,
+      },
+    });
+  }
+
+  return {
+    issues,
+    data: {
+      totalLinks,
+      externalLinks,
+      citationLinks,
+      reputableSources,
+      citationPatterns: {
+        hasInlineCitations,
+        hasReferencesSection,
+        hasStatistics,
+        statisticsWithSources,
+        statisticsWithoutSources,
+      },
+      citationScore,
+    },
+  };
+}