@rankcli/agent-runtime 0.0.1

package/src/audit/checks/crawlability.ts

@@ -0,0 +1,220 @@
+import { httpGet } from '../../utils/http.js';
+import type { AuditIssue, PageData } from '../types.js';
+import { ISSUE_DEFINITIONS } from '../types.js';
+
+export interface RobotsTxtResult {
+  exists: boolean;
+  content?: string;
+  blocksAll: boolean;
+  hasSitemap: boolean;
+  sitemapUrls: string[];
+}
+
+export interface SitemapResult {
+  exists: boolean;
+  url?: string;
+  urlCount: number;
+  errors: string[];
+}
+
+export async function checkRobotsTxt(baseUrl: string): Promise<{ issues: AuditIssue[]; data: RobotsTxtResult }> {
+  const issues: AuditIssue[] = [];
+  const url = new URL('/robots.txt', baseUrl).href;
+
+  try {
+    const response = await httpGet<string>(url, {
+      timeout: 10000,
+      validateStatus: () => true,
+    });
+
+    if (response.status === 404) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.ROBOTS_TXT_MISSING,
+        affectedUrls: [url],
+      });
+      return {
+        issues,
+        data: { exists: false, blocksAll: false, hasSitemap: false, sitemapUrls: [] }
+      };
+    }
+
+    const content = response.data as string;
+    const lines = content.toLowerCase().split('\n');
+
+    // Check if robots.txt blocks all crawlers
+    let blocksAll = false;
+    let currentUserAgent = '';
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith('user-agent:')) {
+        currentUserAgent = trimmed.split(':')[1].trim();
+      }
+      if (trimmed === 'disallow: /' && currentUserAgent === '*') {
+        blocksAll = true;
+      }
+    }
+
+    if (blocksAll) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.ROBOTS_TXT_BLOCKS_ALL,
+        affectedUrls: [url],
+      });
+    }
+
+    // Check for sitemap references
+    const sitemapUrls: string[] = [];
+    for (const line of content.split('\n')) {
+      const trimmed = line.trim().toLowerCase();
+      if (trimmed.startsWith('sitemap:')) {
+        const sitemapUrl = line.trim().substring(8).trim();
+        sitemapUrls.push(sitemapUrl);
+      }
+    }
+
+    return {
+      issues,
+      data: {
+        exists: true,
+        content,
+        blocksAll,
+        hasSitemap: sitemapUrls.length > 0,
+        sitemapUrls,
+      }
+    };
+  } catch (error) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.ROBOTS_TXT_MISSING,
+      affectedUrls: [url],
+      details: { error: error instanceof Error ? error.message : 'Unknown error' },
+    });
+    return {
+      issues,
+      data: { exists: false, blocksAll: false, hasSitemap: false, sitemapUrls: [] }
+    };
+  }
+}
+
+export async function checkSitemap(baseUrl: string, robotsData: RobotsTxtResult): Promise<{ issues: AuditIssue[]; data: SitemapResult }> {
+  const issues: AuditIssue[] = [];
+
+  // Try sitemap URLs from robots.txt first, then standard locations
+  const sitemapUrls = robotsData.sitemapUrls.length > 0
+    ? robotsData.sitemapUrls
+    : [
+        new URL('/sitemap.xml', baseUrl).href,
+        new URL('/sitemap_index.xml', baseUrl).href,
+      ];
+
+  for (const sitemapUrl of sitemapUrls) {
+    try {
+      const response = await httpGet<string>(sitemapUrl, {
+        timeout: 10000,
+        validateStatus: () => true,
+      });
+
+      if (response.status === 200 && (response.data as string).includes('<urlset') || (response.data as string).includes('<sitemapindex')) {
+        // Count URLs in sitemap
+        const content = response.data as string;
+        const urlMatches = content.match(/<loc>/g);
+        const urlCount = urlMatches ? urlMatches.length : 0;
+
+        // Check if sitemap is referenced in robots.txt
+        if (!robotsData.hasSitemap) {
+          issues.push({
+            ...ISSUE_DEFINITIONS.SITEMAP_NOT_IN_ROBOTS,
+            affectedUrls: [sitemapUrl],
+          });
+        }
+
+        return {
+          issues,
+          data: { exists: true, url: sitemapUrl, urlCount, errors: [] }
+        };
+      }
+    } catch {
+      // Try next URL
+    }
+  }
+
+  // No sitemap found
+  issues.push({
+    ...ISSUE_DEFINITIONS.SITEMAP_MISSING,
+    affectedUrls: [new URL('/sitemap.xml', baseUrl).href],
+  });
+
+  return {
+    issues,
+    data: { exists: false, urlCount: 0, errors: ['No sitemap found'] }
+  };
+}
+
+export async function checkRedirects(url: string): Promise<{ issues: AuditIssue[]; chain: string[]; finalUrl: string }> {
+  const issues: AuditIssue[] = [];
+  const chain: string[] = [url];
+  let currentUrl = url;
+  let maxRedirects = 10;
+
+  while (maxRedirects > 0) {
+    try {
+      const response = await httpGet<string>(currentUrl, {
+        timeout: 10000,
+        maxRedirects: 0,
+        validateStatus: () => true,
+      });
+
+      if (response.status >= 300 && response.status < 400) {
+        const location = response.headers.location;
+        if (!location) break;
+
+        const nextUrl = new URL(location, currentUrl).href;
+
+        // Check for redirect loop
+        if (chain.includes(nextUrl)) {
+          issues.push({
+            ...ISSUE_DEFINITIONS.REDIRECT_LOOP,
+            affectedUrls: chain,
+            details: { chain, loopTo: nextUrl },
+          });
+          break;
+        }
+
+        chain.push(nextUrl);
+        currentUrl = nextUrl;
+        maxRedirects--;
+      } else {
+        break;
+      }
+    } catch {
+      break;
+    }
+  }
+
+  // Check for redirect chain (more than 1 redirect)
+  if (chain.length > 2) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.REDIRECT_CHAIN,
+      affectedUrls: chain,
+      details: { chain, hops: chain.length - 1 },
+    });
+  }
+
+  return { issues, chain, finalUrl: currentUrl };
+}
+
+export async function runCrawlabilityChecks(baseUrl: string): Promise<AuditIssue[]> {
+  const allIssues: AuditIssue[] = [];
+
+  // Check robots.txt
+  const robotsResult = await checkRobotsTxt(baseUrl);
+  allIssues.push(...robotsResult.issues);
+
+  // Check sitemap
+  const sitemapResult = await checkSitemap(baseUrl, robotsResult.data);
+  allIssues.push(...sitemapResult.issues);
+
+  // Check for redirects on the main URL
+  const redirectResult = await checkRedirects(baseUrl);
+  allIssues.push(...redirectResult.issues);
+
+  return allIssues;
+}

package/src/audit/checks/directory-listing.ts

@@ -0,0 +1,172 @@
+/**
+ * Directory Listing Security Check
+ *
+ * Checks if the web server is configured to allow directory listing.
+ * When enabled, accessing a directory without an index file shows
+ * all files in that directory - a security risk.
+ *
+ * Detection methods:
+ * - Check common directories (/images/, /assets/, /uploads/, etc.)
+ * - Look for Apache/Nginx directory listing HTML patterns
+ * - Check response headers
+ */
+
+import { httpGet } from '../../utils/http.js';
+import type { AuditIssue } from '../types.js';
+
+export interface DirectoryListingData {
+  directoryListingEnabled: boolean;
+  vulnerableDirectories: string[];
+  checkedDirectories: string[];
+  serverType?: string;
+}
+
+// Common directories to check for directory listing
+const COMMON_DIRECTORIES = [
+  '/images/',
+  '/img/',
+  '/assets/',
+  '/static/',
+  '/uploads/',
+  '/files/',
+  '/media/',
+  '/css/',
+  '/js/',
+  '/scripts/',
+  '/fonts/',
+  '/backup/',
+  '/temp/',
+  '/tmp/',
+];
+
+// Patterns that indicate directory listing is enabled
+const DIRECTORY_LISTING_PATTERNS = [
+  // Apache
+  /<title>Index of/i,
+  /Parent Directory/i,
+  /<h1>Index of/i,
+  /\[DIR\]/i,
+  /\[PARENTDIR\]/i,
+  // Nginx
+  /<title>Index of.*<\/title>/i,
+  /autoindex/i,
+  // IIS
+  /\[To Parent Directory\]/i,
+  /<title>.*- \/<\/title>/i,
+  // Generic
+  /Directory listing for/i,
+  /<pre>.*<a href="[^"]*">[^<]*<\/a>.*<\/pre>/is,
+];
+
+/**
+ * Check if HTML content indicates directory listing
+ */
+function isDirectoryListing(html: string): boolean {
+  return DIRECTORY_LISTING_PATTERNS.some((pattern) => pattern.test(html));
+}
+
+/**
+ * Detect server type from response
+ */
+function detectServerType(headers: Record<string, string>): string | undefined {
+  const server = headers.server || headers.Server;
+  if (server) {
+    if (/apache/i.test(server)) return 'Apache';
+    if (/nginx/i.test(server)) return 'Nginx';
+    if (/iis/i.test(server)) return 'IIS';
+    if (/cloudflare/i.test(server)) return 'Cloudflare';
+    return server;
+  }
+  return undefined;
+}
+
+/**
+ * Analyze directory listing security
+ */
+export async function analyzeDirectoryListing(url: string): Promise<{ issues: AuditIssue[]; data: DirectoryListingData }> {
+  const issues: AuditIssue[] = [];
+  const baseUrl = new URL(url);
+  const baseOrigin = baseUrl.origin;
+
+  const vulnerableDirectories: string[] = [];
+  const checkedDirectories: string[] = [];
+  let serverType: string | undefined;
+
+  // Check each common directory (parallel with short timeout)
+  const directoriesToCheck = COMMON_DIRECTORIES.slice(0, 6); // Limit to 6 directories
+
+  // Run all directory checks in parallel with 3s timeout each
+  const checkPromises = directoriesToCheck.map(async (dir) => {
+    const dirUrl = `${baseOrigin}${dir}`;
+    checkedDirectories.push(dir);
+
+    try {
+      const response = await httpGet<string>(dirUrl, {
+        timeout: 3000, // Reduced timeout
+        validateStatus: () => true,
+      });
+
+      // Get server type from response
+      if (response.headers) {
+        const detectedServer = detectServerType(response.headers as Record<string, string>);
+        if (detectedServer && !serverType) {
+          serverType = detectedServer;
+        }
+      }
+
+      // Check if response indicates directory listing
+      if (response.status === 200 && typeof response.data === 'string') {
+        if (isDirectoryListing(response.data)) {
+          vulnerableDirectories.push(dir);
+        }
+      }
+    } catch {
+      // Skip failed requests (directory doesn't exist or blocked)
+    }
+  });
+
+  // Wait for all checks with overall timeout
+  await Promise.race([
+    Promise.all(checkPromises),
+    new Promise<void>((resolve) => setTimeout(resolve, 8000)), // 8s overall timeout
+  ]);
+
+  const directoryListingEnabled = vulnerableDirectories.length > 0;
+
+  // Generate issues
+  if (directoryListingEnabled) {
+    issues.push({
+      code: 'DIRECTORY_LISTING_ENABLED',
+      severity: 'warning',
+      category: 'security',
+      title: 'Directory listing enabled',
+      description: `${vulnerableDirectories.length} director${vulnerableDirectories.length === 1 ? 'y' : 'ies'} expose file listings. This reveals your site structure and potentially sensitive files.`,
+      impact:
+        'Attackers can discover backup files, configuration files, or hidden content. This is a security vulnerability and information disclosure risk.',
+      howToFix:
+        serverType === 'Apache'
+          ? 'Add "Options -Indexes" to your .htaccess file or Apache configuration.'
+          : serverType === 'Nginx'
+            ? 'Remove or set "autoindex off" in your Nginx configuration.'
+            : 'Disable directory listing in your web server configuration. Add index files to directories.',
+      affectedUrls: vulnerableDirectories.map((dir) => `${baseOrigin}${dir}`),
+      details: {
+        vulnerableDirectories,
+        serverType,
+        checkedCount: checkedDirectories.length,
+        recommendation:
+          'Ensure all directories have an index.html or index.php, or disable directory listing at the server level.',
+      },
+    });
+  }
+
+  return {
+    issues,
+    data: {
+      directoryListingEnabled,
+      vulnerableDirectories,
+      checkedDirectories,
+      serverType,
+    },
+  };
+}

package/src/audit/checks/dom-analysis.ts

@@ -0,0 +1,191 @@
+// DOM Analysis Checks
+// Checks for DOM size, depth, and render-blocking resources
+
+import * as cheerio from 'cheerio';
+import type { AuditIssue } from '../types.js';
+import { ISSUE_DEFINITIONS } from '../types.js';
+
+const DOM_SIZE_WARNING = 1500;
+const DOM_SIZE_ERROR = 3000;
+const DOM_DEPTH_WARNING = 32;
+
+export interface DOMAnalysisData {
+  nodeCount: number;
+  maxDepth: number;
+  renderBlockingResources: {
+    css: string[];
+    js: string[];
+  };
+  totalRenderBlockingSize: number;
+  criticalPathLength: number;
+}
+
+/**
+ * Count total DOM nodes
+ */
+function countNodes($: cheerio.CheerioAPI): number {
+  let count = 0;
+
+  function traverse(element: unknown): void {
+    count++;
+    const el = element as { children?: unknown[] };
+    if (el.children) {
+      for (const child of el.children) {
+        traverse(child);
+      }
+    }
+  }
+
+  const root = $.root();
+  root.each((_, el) => traverse(el));
+
+  return count;
+}
+
+/**
+ * Calculate maximum DOM depth
+ */
+function calculateMaxDepth($: cheerio.CheerioAPI): number {
+  let maxDepth = 0;
+
+  function traverse(element: unknown, depth: number): void {
+    maxDepth = Math.max(maxDepth, depth);
+    const el = element as { children?: unknown[] };
+    if (el.children) {
+      for (const child of el.children) {
+        traverse(child, depth + 1);
+      }
+    }
+  }
+
+  const root = $.root();
+  root.each((_, el) => traverse(el, 0));
+
+  return maxDepth;
+}
+
+/**
+ * Find render-blocking resources
+ */
+function findRenderBlockingResources($: cheerio.CheerioAPI): { css: string[]; js: string[] } {
+  const css: string[] = [];
+  const js: string[] = [];
+
+  // Find render-blocking CSS (in head, without media="print" or disabled)
+  $('head link[rel="stylesheet"]').each((_, el) => {
+    const href = $(el).attr('href');
+    const media = $(el).attr('media');
+    const disabled = $(el).attr('disabled');
+
+    if (href && media !== 'print' && !disabled) {
+      css.push(href);
+    }
+  });
+
+  // Find inline style tags in head (blocking)
+  $('head style').each((_, el) => {
+    const content = $(el).html();
+    if (content && content.trim().length > 100) {
+      css.push('[inline-style]');
+    }
+  });
+
+  // Find render-blocking JS (in head, without async/defer)
+  $('head script[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    const async = $(el).attr('async');
+    const defer = $(el).attr('defer');
+    const type = $(el).attr('type');
+
+    // Skip module scripts (they're deferred by default) and non-JS
+    if (type === 'module' || (type && !type.includes('javascript'))) {
+      return;
+    }
+
+    if (src && !async && !defer) {
+      js.push(src);
+    }
+  });
+
+  return { css, js };
+}
+
+/**
+ * Analyze DOM structure and render-blocking resources
+ */
+export function analyzeDOMStructure(html: string, url: string): { issues: AuditIssue[]; data: DOMAnalysisData } {
+  const issues: AuditIssue[] = [];
+  const $ = cheerio.load(html);
+
+  // Count nodes
+  const nodeCount = countNodes($);
+
+  // Calculate depth
+  const maxDepth = calculateMaxDepth($);
+
+  // Find render-blocking resources
+  const renderBlockingResources = findRenderBlockingResources($);
+  const criticalPathLength = renderBlockingResources.css.length + renderBlockingResources.js.length;
+
+  // Generate issues
+  if (nodeCount > DOM_SIZE_ERROR) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.DOM_SIZE_EXCESSIVE,
+      severity: 'error',
+      affectedUrls: [url],
+      details: {
+        nodeCount,
+        threshold: DOM_SIZE_WARNING,
+        recommendation: 'Reduce DOM size to under 1500 nodes',
+      },
+    });
+  } else if (nodeCount > DOM_SIZE_WARNING) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.DOM_SIZE_EXCESSIVE,
+      affectedUrls: [url],
+      details: {
+        nodeCount,
+        threshold: DOM_SIZE_WARNING,
+        recommendation: 'Consider simplifying page structure',
+      },
+    });
+  }
+
+  if (maxDepth > DOM_DEPTH_WARNING) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.DOM_DEPTH_EXCESSIVE,
+      affectedUrls: [url],
+      details: {
+        maxDepth,
+        threshold: DOM_DEPTH_WARNING,
+      },
+    });
+  }
+
+  if (criticalPathLength > 0) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.RENDER_BLOCKING_RESOURCES,
+      affectedUrls: [url],
+      details: {
+        cssFiles: renderBlockingResources.css,
+        jsFiles: renderBlockingResources.js,
+        totalCount: criticalPathLength,
+        recommendation:
+          criticalPathLength > 3
+            ? 'Inline critical CSS and defer non-critical resources'
+            : 'Consider using async/defer for scripts',
+      },
+    });
+  }
+
+  return {
+    issues,
+    data: {
+      nodeCount,
+      maxDepth,
+      renderBlockingResources,
+      totalRenderBlockingSize: 0, // Would need to fetch resources to calculate
+      criticalPathLength,
+    },
+  };
+}