@rankcli/agent-runtime 0.0.14 → 0.0.16

package/src/audit/checks/links.ts

@@ -1,273 +0,0 @@
-import * as cheerio from 'cheerio';
-import { httpHead } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-import { ISSUE_DEFINITIONS } from '../types.js';
-
-export interface LinkData {
-  internal: { href: string; text: string; nofollow: boolean }[];
-  external: { href: string; text: string; nofollow: boolean }[];
-  totalLinks: number;
-  brokenInternal: string[];
-  brokenExternal: string[];
-  jsOnlyNavigation: { element: string; text: string; reason: string }[];
-  ratio: {
-    internal: number;
-    external: number;
-    internalToExternalRatio: number | null; // null if no external links
-  };
-}
-
-export async function analyzeLinks(
-  html: string,
-  baseUrl: string,
-  checkBroken: boolean = false
-): Promise<{ issues: AuditIssue[]; data: LinkData }> {
-  const issues: AuditIssue[] = [];
-  const $ = cheerio.load(html);
-  const baseHostname = new URL(baseUrl).hostname;
-
-  const internal: LinkData['internal'] = [];
-  const external: LinkData['external'] = [];
-  const brokenInternal: string[] = [];
-  const brokenExternal: string[] = [];
-  const jsOnlyNavigation: LinkData['jsOnlyNavigation'] = [];
-
-  // Extract all links
-  $('a[href]').each((_, el) => {
-    const href = $(el).attr('href') || '';
-    const text = $(el).text().trim();
-    const rel = $(el).attr('rel') || '';
-    const nofollow = rel.includes('nofollow');
-
-    // Skip javascript:, mailto:, tel:, and anchor links
-    if (href.startsWith('javascript:') || href.startsWith('mailto:') ||
-        href.startsWith('tel:') || href.startsWith('#')) {
-      return;
-    }
-
-    try {
-      const fullUrl = new URL(href, baseUrl).href;
-      const linkHostname = new URL(fullUrl).hostname;
-      const isInternal = linkHostname === baseHostname;
-
-      if (isInternal) {
-        internal.push({ href: fullUrl, text, nofollow });
-        if (nofollow) {
-          issues.push({
-            ...ISSUE_DEFINITIONS.NOFOLLOW_INTERNAL,
-            affectedUrls: [baseUrl],
-            details: { link: fullUrl, anchorText: text },
-          });
-        }
-      } else {
-        external.push({ href: fullUrl, text, nofollow });
-      }
-    } catch {
-      // Invalid URL, skip
-    }
-  });
-
-  const totalLinks = internal.length + external.length;
-
-  // Check for JavaScript-only navigation
-  // Look for elements with onClick that appear to be navigation
-  const navPatterns = /navigate|router|history\.push|location\.href|window\.location|goto|redirect/i;
-  const navTextPatterns = /^(home|about|contact|pricing|blog|docs|features|products?|services?|sign\s*up|log\s*in|register|dashboard|account|settings|help|faq|support|menu|nav)$/i;
-
-  // Check buttons and other elements with onClick
-  $('button[onclick], div[onclick], span[onclick], li[onclick]').each((_, el) => {
-    const $el = $(el);
-    const onclick = $el.attr('onclick') || '';
-    const text = $el.text().trim().substring(0, 50);
-
-    // Check if onClick contains navigation-like code
-    if (navPatterns.test(onclick) || navTextPatterns.test(text)) {
-      jsOnlyNavigation.push({
-        element: el.tagName.toLowerCase(),
-        text: text || '(no text)',
-        reason: 'Element uses onClick for navigation instead of <a href>',
-      });
-    }
-  });
-
-  // Check <a> tags with non-functional href but onClick (common anti-pattern)
-  $('a[onclick]').each((_, el) => {
-    const $el = $(el);
-    const href = $el.attr('href') || '';
-    const onclick = $el.attr('onclick') || '';
-    const text = $el.text().trim().substring(0, 50);
-
-    // href="#" or "javascript:" with onClick is a red flag
-    if ((href === '#' || href === '' || href.startsWith('javascript:')) && onclick) {
-      jsOnlyNavigation.push({
-        element: 'a',
-        text: text || '(no text)',
-        reason: `Link has href="${href}" with onClick - crawlers cannot follow this`,
-      });
-    }
-  });
-
-  // Check for React/Vue patterns in inline scripts (data-* attributes often used)
-  $('[data-href], [data-to], [data-link], [data-route]').each((_, el) => {
-    const $el = $(el);
-    const tagName = el.tagName.toLowerCase();
-    const text = $el.text().trim().substring(0, 50);
-
-    // If it's not an <a> tag with a real href, it's JS-only navigation
-    if (tagName !== 'a' || !$el.attr('href') || $el.attr('href') === '#') {
-      jsOnlyNavigation.push({
-        element: tagName,
-        text: text || '(no text)',
-        reason: 'Uses data attribute for routing instead of real href',
-      });
-    }
-  });
-
-  // Report JS-only navigation issues (deduplicated, limit to 10)
-  if (jsOnlyNavigation.length > 0) {
-    const uniqueNav = jsOnlyNavigation.slice(0, 10);
-    issues.push({
-      ...ISSUE_DEFINITIONS.JS_ONLY_NAVIGATION,
-      affectedUrls: [baseUrl],
-      details: {
-        count: jsOnlyNavigation.length,
-        examples: uniqueNav,
-      },
-    });
-  }
-
-  // Calculate link ratio
-  const internalCount = internal.length;
-  const externalCount = external.length;
-  const internalToExternalRatio = externalCount > 0 ? internalCount / externalCount : null;
-
-  // Check for too many links
-  if (totalLinks > 100) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.TOO_MANY_LINKS,
-      affectedUrls: [baseUrl],
-      details: { totalLinks, internal: internalCount, external: externalCount },
-    });
-  }
-
-  // Check for no internal links (bad for crawlability and user navigation)
-  if (internalCount === 0 && totalLinks > 0) {
-    issues.push({
-      code: 'LINKS_NO_INTERNAL',
-      severity: 'warning',
-      category: 'links',
-      title: 'No internal links found',
-      description: 'This page has no internal links to other pages on your site.',
-      impact: 'Internal links are crucial for SEO. They help search engines discover pages and distribute link equity throughout your site.',
-      howToFix: 'Add relevant internal links to related content, navigation, or related articles section.',
-      affectedUrls: [baseUrl],
-      details: { totalLinks, external: externalCount },
-    });
-  }
-
-  // Check for no external links (pages should reference external resources)
-  if (externalCount === 0 && totalLinks > 5) {
-    issues.push({
-      code: 'LINKS_NO_EXTERNAL',
-      severity: 'notice',
-      category: 'links',
-      title: 'No external links found',
-      description: 'This page has no links to external websites.',
-      impact: 'Linking to high-quality external sources can improve credibility and help search engines understand your content context.',
-      howToFix: 'Add links to authoritative external sources that support your content. This builds trust and provides value to readers.',
-      affectedUrls: [baseUrl],
-      details: { totalLinks, internal: internalCount },
-    });
-  }
-
-  // Check for poor internal-to-external ratio (too many external vs internal)
-  if (externalCount > internalCount && externalCount > 5 && internalCount < 3) {
-    issues.push({
-      code: 'LINKS_RATIO_POOR',
-      severity: 'warning',
-      category: 'links',
-      title: 'Poor internal-to-external link ratio',
-      description: `Page has ${externalCount} external links but only ${internalCount} internal links.`,
-      impact: 'A healthy page should have more internal links than external links. Too many external links can dilute page authority and may look spammy.',
-      howToFix: 'Add more relevant internal links to balance the ratio. Aim for at least 2-3 internal links for every external link.',
-      affectedUrls: [baseUrl],
-      details: {
-        internal: internalCount,
-        external: externalCount,
-        ratio: internalToExternalRatio ? `1:${(1/internalToExternalRatio).toFixed(1)}` : 'N/A',
-        recommendation: 'Aim for a ratio of at least 2:1 (internal:external)',
-      },
-    });
-  }
-
-  // Check for broken links (limited to avoid too many requests)
-  if (checkBroken) {
-    // Check internal links (up to 10)
-    const internalToCheck = internal.slice(0, 10);
-    for (const link of internalToCheck) {
-      try {
-        const response = await httpHead(link.href, {
-          timeout: 5000,
-          maxRedirects: 5,
-          validateStatus: () => true,
-        });
-        if (response.status >= 400) {
-          brokenInternal.push(link.href);
-        }
-      } catch {
-        brokenInternal.push(link.href);
-      }
-    }
-
-    // Check external links (up to 5)
-    const externalToCheck = external.slice(0, 5);
-    for (const link of externalToCheck) {
-      try {
-        const response = await httpHead(link.href, {
-          timeout: 5000,
-          maxRedirects: 5,
-          validateStatus: () => true,
-        });
-        if (response.status >= 400) {
-          brokenExternal.push(link.href);
-        }
-      } catch {
-        // Don't count timeouts as broken for external links
-      }
-    }
-
-    // Add issues for broken links
-    for (const brokenLink of brokenInternal) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.BROKEN_INTERNAL_LINK,
-        affectedUrls: [baseUrl],
-        details: { brokenLink },
-      });
-    }
-
-    for (const brokenLink of brokenExternal) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.BROKEN_EXTERNAL_LINK,
-        affectedUrls: [baseUrl],
-        details: { brokenLink },
-      });
-    }
-  }
-
-  return {
-    issues,
-    data: {
-      internal,
-      external,
-      totalLinks,
-      brokenInternal,
-      brokenExternal,
-      jsOnlyNavigation,
-      ratio: {
-        internal: internalCount,
-        external: externalCount,
-        internalToExternalRatio,
-      },
-    }
-  };
-}

package/src/audit/checks/llms-txt.ts

@@ -1,224 +0,0 @@
-/**
- * llms.txt Check
- *
- * llms.txt is a proposed standard (https://llmstxt.org/) for providing
- * context to Large Language Models about a website.
- *
- * The file is a Markdown document at /llms.txt that contains:
- * - Brief description of the site/organization
- * - Links to important pages
- * - Optional detailed content at /llms-full.txt
- *
- * Format:
- * # Site Name
- * > Brief description
- *
- * ## Section
- * - [Link Text](url): Description
- */
-
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-
-export interface LlmsTxtData {
-  exists: boolean;
-  fullVersionExists: boolean;
-  content?: string;
-  structure: {
-    hasTitle: boolean;
-    hasDescription: boolean;
-    hasSections: boolean;
-    hasLinks: boolean;
-    linkCount: number;
-  };
-  contentLength: number;
-}
-
-export async function analyzeLlmsTxt(
-  url: string
-): Promise<{ issues: AuditIssue[]; data: LlmsTxtData }> {
-  const issues: AuditIssue[] = [];
-  const parsedUrl = new URL(url);
-  const baseUrl = parsedUrl.origin;
-
-  // Check for llms.txt
-  const llmsTxtUrl = new URL('/llms.txt', baseUrl).href;
-  const llmsFullUrl = new URL('/llms-full.txt', baseUrl).href;
-
-  let exists = false;
-  let fullVersionExists = false;
-  let content: string | undefined;
-  let structure = {
-    hasTitle: false,
-    hasDescription: false,
-    hasSections: false,
-    hasLinks: false,
-    linkCount: 0,
-  };
-  let contentLength = 0;
-
-  try {
-    // Check main llms.txt
-    const response = await httpGet<string>(llmsTxtUrl, {
-      timeout: 10000,
-      validateStatus: () => true,
-    });
-
-    if (response.status === 200) {
-      exists = true;
-      content = response.data as string;
-      contentLength = content.length;
-      structure = analyzeStructure(content);
-    }
-
-    // Check for full version
-    const fullResponse = await httpGet<string>(llmsFullUrl, {
-      timeout: 10000,
-      validateStatus: () => true,
-    });
-
-    if (fullResponse.status === 200) {
-      fullVersionExists = true;
-    }
-  } catch {
-    // Network error, treat as not existing
-  }
-
-  // Generate issues
-  if (!exists) {
-    issues.push({
-      code: 'AI_NO_LLMS_TXT',
-      severity: 'notice',
-      category: 'ai-readiness',
-      title: 'No llms.txt file found',
-      description: 'llms.txt is a proposed standard for providing context to LLMs about your website. Having one helps AI systems better understand and represent your site.',
-      impact: 'AI systems may have less context about your site, potentially leading to less accurate representations.',
-      howToFix: 'Create an /llms.txt file with a Markdown description of your site. Include: site name, brief description, and links to key pages. See https://llmstxt.org/ for the specification.',
-      affectedUrls: [llmsTxtUrl],
-      details: {
-        specification: 'https://llmstxt.org/',
-        exampleFormat: `# Site Name
-
-> Brief description of your site
-
-## About
-- [About Us](/about): Learn more about our company
-
-## Products
-- [Product A](/products/a): Description of product A
-- [Product B](/products/b): Description of product B
-
-## Documentation
-- [Docs](/docs): Full documentation`,
-      },
-    });
-  } else {
-    // Check structure quality
-    if (!structure.hasTitle) {
-      issues.push({
-        code: 'AI_LLMS_TXT_NO_TITLE',
-        severity: 'notice',
-        category: 'ai-readiness',
-        title: 'llms.txt missing title heading',
-        description: 'Your llms.txt file should start with a # heading containing your site/organization name.',
-        impact: 'LLMs may not properly identify your site.',
-        howToFix: 'Add a title heading at the top: # Your Site Name',
-        affectedUrls: [llmsTxtUrl],
-      });
-    }
-
-    if (!structure.hasDescription) {
-      issues.push({
-        code: 'AI_LLMS_TXT_NO_DESCRIPTION',
-        severity: 'notice',
-        category: 'ai-readiness',
-        title: 'llms.txt missing description',
-        description: 'Your llms.txt should include a brief description (blockquote starting with >) explaining what your site/organization does.',
-        impact: 'LLMs may not understand your site\'s purpose.',
-        howToFix: 'Add a description blockquote after the title: > Brief description of your site',
-        affectedUrls: [llmsTxtUrl],
-      });
-    }
-
-    if (!structure.hasLinks || structure.linkCount < 3) {
-      issues.push({
-        code: 'AI_LLMS_TXT_FEW_LINKS',
-        severity: 'notice',
-        category: 'ai-readiness',
-        title: 'llms.txt has few navigation links',
-        description: `Your llms.txt only has ${structure.linkCount} link(s). Include links to your most important pages to help LLMs navigate your content.`,
-        impact: 'LLMs may miss important content on your site.',
-        howToFix: 'Add more links in Markdown format: - [Page Title](/path): Brief description',
-        affectedUrls: [llmsTxtUrl],
-      });
-    }
-
-    if (contentLength < 200) {
-      issues.push({
-        code: 'AI_LLMS_TXT_TOO_SHORT',
-        severity: 'notice',
-        category: 'ai-readiness',
-        title: 'llms.txt content is very short',
-        description: `Your llms.txt is only ${contentLength} characters. Consider adding more context about your site.`,
-        impact: 'LLMs may not have enough context to properly represent your site.',
-        howToFix: 'Expand your llms.txt with more sections, descriptions, and links to key content.',
-        affectedUrls: [llmsTxtUrl],
-      });
-    }
-  }
-
-  return {
-    issues,
-    data: {
-      exists,
-      fullVersionExists,
-      content: content?.substring(0, 1000), // Truncate for data
-      structure,
-      contentLength,
-    },
-  };
-}
-
-function analyzeStructure(content: string): LlmsTxtData['structure'] {
-  const lines = content.split('\n');
-
-  let hasTitle = false;
-  let hasDescription = false;
-  let hasSections = false;
-  let hasLinks = false;
-  let linkCount = 0;
-
-  for (const line of lines) {
-    const trimmed = line.trim();
-
-    // Check for title (# heading at start)
-    if (trimmed.startsWith('# ') && !trimmed.startsWith('## ')) {
-      hasTitle = true;
-    }
-
-    // Check for description (blockquote)
-    if (trimmed.startsWith('>')) {
-      hasDescription = true;
-    }
-
-    // Check for sections (## headings)
-    if (trimmed.startsWith('## ')) {
-      hasSections = true;
-    }
-
-    // Check for Markdown links
-    const linkMatches = trimmed.match(/\[([^\]]+)\]\(([^)]+)\)/g);
-    if (linkMatches) {
-      hasLinks = true;
-      linkCount += linkMatches.length;
-    }
-  }
-
-  return {
-    hasTitle,
-    hasDescription,
-    hasSections,
-    hasLinks,
-    linkCount,
-  };
-}