@rankcli/agent-runtime 0.0.14 → 0.0.16

package/src/audit/checks/crawlability.ts

@@ -1,220 +0,0 @@
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue, PageData } from '../types.js';
-import { ISSUE_DEFINITIONS } from '../types.js';
-
-export interface RobotsTxtResult {
-  exists: boolean;
-  content?: string;
-  blocksAll: boolean;
-  hasSitemap: boolean;
-  sitemapUrls: string[];
-}
-
-export interface SitemapResult {
-  exists: boolean;
-  url?: string;
-  urlCount: number;
-  errors: string[];
-}
-
-export async function checkRobotsTxt(baseUrl: string): Promise<{ issues: AuditIssue[]; data: RobotsTxtResult }> {
-  const issues: AuditIssue[] = [];
-  const url = new URL('/robots.txt', baseUrl).href;
-
-  try {
-    const response = await httpGet<string>(url, {
-      timeout: 10000,
-      validateStatus: () => true,
-    });
-
-    if (response.status === 404) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.ROBOTS_TXT_MISSING,
-        affectedUrls: [url],
-      });
-      return {
-        issues,
-        data: { exists: false, blocksAll: false, hasSitemap: false, sitemapUrls: [] }
-      };
-    }
-
-    const content = response.data as string;
-    const lines = content.toLowerCase().split('\n');
-
-    // Check if robots.txt blocks all crawlers
-    let blocksAll = false;
-    let currentUserAgent = '';
-    for (const line of lines) {
-      const trimmed = line.trim();
-      if (trimmed.startsWith('user-agent:')) {
-        currentUserAgent = trimmed.split(':')[1].trim();
-      }
-      if (trimmed === 'disallow: /' && currentUserAgent === '*') {
-        blocksAll = true;
-      }
-    }
-
-    if (blocksAll) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.ROBOTS_TXT_BLOCKS_ALL,
-        affectedUrls: [url],
-      });
-    }
-
-    // Check for sitemap references
-    const sitemapUrls: string[] = [];
-    for (const line of content.split('\n')) {
-      const trimmed = line.trim().toLowerCase();
-      if (trimmed.startsWith('sitemap:')) {
-        const sitemapUrl = line.trim().substring(8).trim();
-        sitemapUrls.push(sitemapUrl);
-      }
-    }
-
-    return {
-      issues,
-      data: {
-        exists: true,
-        content,
-        blocksAll,
-        hasSitemap: sitemapUrls.length > 0,
-        sitemapUrls,
-      }
-    };
-  } catch (error) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.ROBOTS_TXT_MISSING,
-      affectedUrls: [url],
-      details: { error: error instanceof Error ? error.message : 'Unknown error' },
-    });
-    return {
-      issues,
-      data: { exists: false, blocksAll: false, hasSitemap: false, sitemapUrls: [] }
-    };
-  }
-}
-
-export async function checkSitemap(baseUrl: string, robotsData: RobotsTxtResult): Promise<{ issues: AuditIssue[]; data: SitemapResult }> {
-  const issues: AuditIssue[] = [];
-
-  // Try sitemap URLs from robots.txt first, then standard locations
-  const sitemapUrls = robotsData.sitemapUrls.length > 0
-    ? robotsData.sitemapUrls
-    : [
-        new URL('/sitemap.xml', baseUrl).href,
-        new URL('/sitemap_index.xml', baseUrl).href,
-      ];
-
-  for (const sitemapUrl of sitemapUrls) {
-    try {
-      const response = await httpGet<string>(sitemapUrl, {
-        timeout: 10000,
-        validateStatus: () => true,
-      });
-
-      if (response.status === 200 && (response.data as string).includes('<urlset') || (response.data as string).includes('<sitemapindex')) {
-        // Count URLs in sitemap
-        const content = response.data as string;
-        const urlMatches = content.match(/<loc>/g);
-        const urlCount = urlMatches ? urlMatches.length : 0;
-
-        // Check if sitemap is referenced in robots.txt
-        if (!robotsData.hasSitemap) {
-          issues.push({
-            ...ISSUE_DEFINITIONS.SITEMAP_NOT_IN_ROBOTS,
-            affectedUrls: [sitemapUrl],
-          });
-        }
-
-        return {
-          issues,
-          data: { exists: true, url: sitemapUrl, urlCount, errors: [] }
-        };
-      }
-    } catch {
-      // Try next URL
-    }
-  }
-
-  // No sitemap found
-  issues.push({
-    ...ISSUE_DEFINITIONS.SITEMAP_MISSING,
-    affectedUrls: [new URL('/sitemap.xml', baseUrl).href],
-  });
-
-  return {
-    issues,
-    data: { exists: false, urlCount: 0, errors: ['No sitemap found'] }
-  };
-}
-
-export async function checkRedirects(url: string): Promise<{ issues: AuditIssue[]; chain: string[]; finalUrl: string }> {
-  const issues: AuditIssue[] = [];
-  const chain: string[] = [url];
-  let currentUrl = url;
-  let maxRedirects = 10;
-
-  while (maxRedirects > 0) {
-    try {
-      const response = await httpGet<string>(currentUrl, {
-        timeout: 10000,
-        maxRedirects: 0,
-        validateStatus: () => true,
-      });
-
-      if (response.status >= 300 && response.status < 400) {
-        const location = response.headers.location;
-        if (!location) break;
-
-        const nextUrl = new URL(location, currentUrl).href;
-
-        // Check for redirect loop
-        if (chain.includes(nextUrl)) {
-          issues.push({
-            ...ISSUE_DEFINITIONS.REDIRECT_LOOP,
-            affectedUrls: chain,
-            details: { chain, loopTo: nextUrl },
-          });
-          break;
-        }
-
-        chain.push(nextUrl);
-        currentUrl = nextUrl;
-        maxRedirects--;
-      } else {
-        break;
-      }
-    } catch {
-      break;
-    }
-  }
-
-  // Check for redirect chain (more than 1 redirect)
-  if (chain.length > 2) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.REDIRECT_CHAIN,
-      affectedUrls: chain,
-      details: { chain, hops: chain.length - 1 },
-    });
-  }
-
-  return { issues, chain, finalUrl: currentUrl };
-}
-
-export async function runCrawlabilityChecks(baseUrl: string): Promise<AuditIssue[]> {
-  const allIssues: AuditIssue[] = [];
-
-  // Check robots.txt
-  const robotsResult = await checkRobotsTxt(baseUrl);
-  allIssues.push(...robotsResult.issues);
-
-  // Check sitemap
-  const sitemapResult = await checkSitemap(baseUrl, robotsResult.data);
-  allIssues.push(...sitemapResult.issues);
-
-  // Check for redirects on the main URL
-  const redirectResult = await checkRedirects(baseUrl);
-  allIssues.push(...redirectResult.issues);
-
-  return allIssues;
-}

package/src/audit/checks/directory-listing.ts

@@ -1,172 +0,0 @@
-/**
- * Directory Listing Security Check
- *
- * Checks if the web server is configured to allow directory listing.
- * When enabled, accessing a directory without an index file shows
- * all files in that directory - a security risk.
- *
- * Detection methods:
- * - Check common directories (/images/, /assets/, /uploads/, etc.)
- * - Look for Apache/Nginx directory listing HTML patterns
- * - Check response headers
- */
-
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-
-export interface DirectoryListingData {
-  directoryListingEnabled: boolean;
-  vulnerableDirectories: string[];
-  checkedDirectories: string[];
-  serverType?: string;
-}
-
-// Common directories to check for directory listing
-const COMMON_DIRECTORIES = [
-  '/images/',
-  '/img/',
-  '/assets/',
-  '/static/',
-  '/uploads/',
-  '/files/',
-  '/media/',
-  '/css/',
-  '/js/',
-  '/scripts/',
-  '/fonts/',
-  '/backup/',
-  '/temp/',
-  '/tmp/',
-];
-
-// Patterns that indicate directory listing is enabled
-const DIRECTORY_LISTING_PATTERNS = [
-  // Apache
-  /<title>Index of/i,
-  /Parent Directory/i,
-  /<h1>Index of/i,
-  /\[DIR\]/i,
-  /\[PARENTDIR\]/i,
-  // Nginx
-  /<title>Index of.*<\/title>/i,
-  /autoindex/i,
-  // IIS
-  /\[To Parent Directory\]/i,
-  /<title>.*- \/<\/title>/i,
-  // Generic
-  /Directory listing for/i,
-  /<pre>.*<a href="[^"]*">[^<]*<\/a>.*<\/pre>/is,
-];
-
-/**
- * Check if HTML content indicates directory listing
- */
-function isDirectoryListing(html: string): boolean {
-  return DIRECTORY_LISTING_PATTERNS.some((pattern) => pattern.test(html));
-}
-
-/**
- * Detect server type from response
- */
-function detectServerType(headers: Record<string, string>): string | undefined {
-  const server = headers.server || headers.Server;
-  if (server) {
-    if (/apache/i.test(server)) return 'Apache';
-    if (/nginx/i.test(server)) return 'Nginx';
-    if (/iis/i.test(server)) return 'IIS';
-    if (/cloudflare/i.test(server)) return 'Cloudflare';
-    return server;
-  }
-  return undefined;
-}
-
-/**
- * Analyze directory listing security
- */
-export async function analyzeDirectoryListing(url: string): Promise<{ issues: AuditIssue[]; data: DirectoryListingData }> {
-  const issues: AuditIssue[] = [];
-  const baseUrl = new URL(url);
-  const baseOrigin = baseUrl.origin;
-
-  const vulnerableDirectories: string[] = [];
-  const checkedDirectories: string[] = [];
-  let serverType: string | undefined;
-
-  // Check each common directory (parallel with short timeout)
-  const directoriesToCheck = COMMON_DIRECTORIES.slice(0, 6); // Limit to 6 directories
-
-  // Run all directory checks in parallel with 3s timeout each
-  const checkPromises = directoriesToCheck.map(async (dir) => {
-    const dirUrl = `${baseOrigin}${dir}`;
-    checkedDirectories.push(dir);
-
-    try {
-      const response = await httpGet<string>(dirUrl, {
-        timeout: 3000, // Reduced timeout
-        validateStatus: () => true,
-      });
-
-      // Get server type from response
-      if (response.headers) {
-        const detectedServer = detectServerType(response.headers as Record<string, string>);
-        if (detectedServer && !serverType) {
-          serverType = detectedServer;
-        }
-      }
-
-      // Check if response indicates directory listing
-      if (response.status === 200 && typeof response.data === 'string') {
-        if (isDirectoryListing(response.data)) {
-          vulnerableDirectories.push(dir);
-        }
-      }
-    } catch {
-      // Skip failed requests (directory doesn't exist or blocked)
-    }
-  });
-
-  // Wait for all checks with overall timeout
-  await Promise.race([
-    Promise.all(checkPromises),
-    new Promise<void>((resolve) => setTimeout(resolve, 8000)), // 8s overall timeout
-  ]);
-
-  const directoryListingEnabled = vulnerableDirectories.length > 0;
-
-  // Generate issues
-  if (directoryListingEnabled) {
-    issues.push({
-      code: 'DIRECTORY_LISTING_ENABLED',
-      severity: 'warning',
-      category: 'security',
-      title: 'Directory listing enabled',
-      description: `${vulnerableDirectories.length} director${vulnerableDirectories.length === 1 ? 'y' : 'ies'} expose file listings. This reveals your site structure and potentially sensitive files.`,
-      impact:
-        'Attackers can discover backup files, configuration files, or hidden content. This is a security vulnerability and information disclosure risk.',
-      howToFix:
-        serverType === 'Apache'
-          ? 'Add "Options -Indexes" to your .htaccess file or Apache configuration.'
-          : serverType === 'Nginx'
-            ? 'Remove or set "autoindex off" in your Nginx configuration.'
-            : 'Disable directory listing in your web server configuration. Add index files to directories.',
-      affectedUrls: vulnerableDirectories.map((dir) => `${baseOrigin}${dir}`),
-      details: {
-        vulnerableDirectories,
-        serverType,
-        checkedCount: checkedDirectories.length,
-        recommendation:
-          'Ensure all directories have an index.html or index.php, or disable directory listing at the server level.',
-      },
-    });
-  }
-
-  return {
-    issues,
-    data: {
-      directoryListingEnabled,
-      vulnerableDirectories,
-      checkedDirectories,
-      serverType,
-    },
-  };
-}

package/src/audit/checks/dom-analysis.ts

@@ -1,191 +0,0 @@
-// DOM Analysis Checks
-// Checks for DOM size, depth, and render-blocking resources
-
-import * as cheerio from 'cheerio';
-import type { AuditIssue } from '../types.js';
-import { ISSUE_DEFINITIONS } from '../types.js';
-
-const DOM_SIZE_WARNING = 1500;
-const DOM_SIZE_ERROR = 3000;
-const DOM_DEPTH_WARNING = 32;
-
-export interface DOMAnalysisData {
-  nodeCount: number;
-  maxDepth: number;
-  renderBlockingResources: {
-    css: string[];
-    js: string[];
-  };
-  totalRenderBlockingSize: number;
-  criticalPathLength: number;
-}
-
-/**
- * Count total DOM nodes
- */
-function countNodes($: cheerio.CheerioAPI): number {
-  let count = 0;
-
-  function traverse(element: unknown): void {
-    count++;
-    const el = element as { children?: unknown[] };
-    if (el.children) {
-      for (const child of el.children) {
-        traverse(child);
-      }
-    }
-  }
-
-  const root = $.root();
-  root.each((_, el) => traverse(el));
-
-  return count;
-}
-
-/**
- * Calculate maximum DOM depth
- */
-function calculateMaxDepth($: cheerio.CheerioAPI): number {
-  let maxDepth = 0;
-
-  function traverse(element: unknown, depth: number): void {
-    maxDepth = Math.max(maxDepth, depth);
-    const el = element as { children?: unknown[] };
-    if (el.children) {
-      for (const child of el.children) {
-        traverse(child, depth + 1);
-      }
-    }
-  }
-
-  const root = $.root();
-  root.each((_, el) => traverse(el, 0));
-
-  return maxDepth;
-}
-
-/**
- * Find render-blocking resources
- */
-function findRenderBlockingResources($: cheerio.CheerioAPI): { css: string[]; js: string[] } {
-  const css: string[] = [];
-  const js: string[] = [];
-
-  // Find render-blocking CSS (in head, without media="print" or disabled)
-  $('head link[rel="stylesheet"]').each((_, el) => {
-    const href = $(el).attr('href');
-    const media = $(el).attr('media');
-    const disabled = $(el).attr('disabled');
-
-    if (href && media !== 'print' && !disabled) {
-      css.push(href);
-    }
-  });
-
-  // Find inline style tags in head (blocking)
-  $('head style').each((_, el) => {
-    const content = $(el).html();
-    if (content && content.trim().length > 100) {
-      css.push('[inline-style]');
-    }
-  });
-
-  // Find render-blocking JS (in head, without async/defer)
-  $('head script[src]').each((_, el) => {
-    const src = $(el).attr('src');
-    const async = $(el).attr('async');
-    const defer = $(el).attr('defer');
-    const type = $(el).attr('type');
-
-    // Skip module scripts (they're deferred by default) and non-JS
-    if (type === 'module' || (type && !type.includes('javascript'))) {
-      return;
-    }
-
-    if (src && !async && !defer) {
-      js.push(src);
-    }
-  });
-
-  return { css, js };
-}
-
-/**
- * Analyze DOM structure and render-blocking resources
- */
-export function analyzeDOMStructure(html: string, url: string): { issues: AuditIssue[]; data: DOMAnalysisData } {
-  const issues: AuditIssue[] = [];
-  const $ = cheerio.load(html);
-
-  // Count nodes
-  const nodeCount = countNodes($);
-
-  // Calculate depth
-  const maxDepth = calculateMaxDepth($);
-
-  // Find render-blocking resources
-  const renderBlockingResources = findRenderBlockingResources($);
-  const criticalPathLength = renderBlockingResources.css.length + renderBlockingResources.js.length;
-
-  // Generate issues
-  if (nodeCount > DOM_SIZE_ERROR) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.DOM_SIZE_EXCESSIVE,
-      severity: 'error',
-      affectedUrls: [url],
-      details: {
-        nodeCount,
-        threshold: DOM_SIZE_WARNING,
-        recommendation: 'Reduce DOM size to under 1500 nodes',
-      },
-    });
-  } else if (nodeCount > DOM_SIZE_WARNING) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.DOM_SIZE_EXCESSIVE,
-      affectedUrls: [url],
-      details: {
-        nodeCount,
-        threshold: DOM_SIZE_WARNING,
-        recommendation: 'Consider simplifying page structure',
-      },
-    });
-  }
-
-  if (maxDepth > DOM_DEPTH_WARNING) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.DOM_DEPTH_EXCESSIVE,
-      affectedUrls: [url],
-      details: {
-        maxDepth,
-        threshold: DOM_DEPTH_WARNING,
-      },
-    });
-  }
-
-  if (criticalPathLength > 0) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.RENDER_BLOCKING_RESOURCES,
-      affectedUrls: [url],
-      details: {
-        cssFiles: renderBlockingResources.css,
-        jsFiles: renderBlockingResources.js,
-        totalCount: criticalPathLength,
-        recommendation:
-          criticalPathLength > 3
-            ? 'Inline critical CSS and defer non-critical resources'
-            : 'Consider using async/defer for scripts',
-      },
-    });
-  }
-
-  return {
-    issues,
-    data: {
-      nodeCount,
-      maxDepth,
-      renderBlockingResources,
-      totalRenderBlockingSize: 0, // Would need to fetch resources to calculate
-      criticalPathLength,
-    },
-  };
-}