@rankcli/agent-runtime 0.0.1

package/src/audit/checks/robots-validation.ts

@@ -0,0 +1,373 @@
+/**
+ * robots.txt Syntax Validation
+ *
+ * Validates robots.txt according to Google's specification:
+ * https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
+ *
+ * Valid directives:
+ * - User-agent: <bot-name>
+ * - Disallow: <path>
+ * - Allow: <path>
+ * - Sitemap: <url>
+ *
+ * Common extensions (non-standard but widely supported):
+ * - Crawl-delay: <seconds>
+ * - Host: <domain> (Yandex)
+ */
+
+import { httpGet } from '../../utils/http.js';
+import type { AuditIssue } from '../types.js';
+
+export interface RobotsValidationData {
+  exists: boolean;
+  isValid: boolean;
+  lineCount: number;
+  errors: RobotsError[];
+  warnings: RobotsWarning[];
+  directives: {
+    userAgents: string[];
+    sitemaps: string[];
+    hasWildcardAgent: boolean;
+    hasCrawlDelay: boolean;
+  };
+}
+
+interface RobotsError {
+  line: number;
+  content: string;
+  message: string;
+}
+
+interface RobotsWarning {
+  line: number;
+  content: string;
+  message: string;
+}
+
+// Standard directives (case-insensitive)
+const STANDARD_DIRECTIVES = [
+  'user-agent',
+  'disallow',
+  'allow',
+  'sitemap',
+];
+
+// Common non-standard but accepted directives
+const EXTENDED_DIRECTIVES = [
+  'crawl-delay',
+  'host',
+  'clean-param', // Yandex
+  'request-rate', // Some crawlers
+];
+
+// All recognized directives
+const ALL_DIRECTIVES = [...STANDARD_DIRECTIVES, ...EXTENDED_DIRECTIVES];
+
+export async function validateRobotsTxt(
+  url: string
+): Promise<{ issues: AuditIssue[]; data: RobotsValidationData }> {
+  const issues: AuditIssue[] = [];
+  const parsedUrl = new URL(url);
+  const robotsUrl = new URL('/robots.txt', parsedUrl.origin).href;
+
+  try {
+    const response = await httpGet<string>(robotsUrl, {
+      timeout: 10000,
+      validateStatus: () => true,
+    });
+
+    if (response.status === 404) {
+      return {
+        issues,
+        data: {
+          exists: false,
+          isValid: true, // Missing is not invalid
+          lineCount: 0,
+          errors: [],
+          warnings: [],
+          directives: {
+            userAgents: [],
+            sitemaps: [],
+            hasWildcardAgent: false,
+            hasCrawlDelay: false,
+          },
+        },
+      };
+    }
+
+    const content = response.data as string;
+    const validation = parseAndValidate(content);
+
+    // Generate issues based on validation
+    if (validation.errors.length > 0) {
+      issues.push({
+        code: 'ROBOTS_TXT_INVALID_SYNTAX',
+        severity: 'warning',
+        category: 'crawlability',
+        title: 'robots.txt contains syntax errors',
+        description: `Found ${validation.errors.length} syntax error(s) in robots.txt. Invalid syntax may cause crawlers to misinterpret your rules.`,
+        impact: 'Search engines may not properly understand your crawling rules, potentially blocking or allowing unintended pages.',
+        howToFix: 'Fix the syntax errors in robots.txt. Each directive should be on its own line with format "Directive: value".',
+        affectedUrls: [robotsUrl],
+        details: {
+          errors: validation.errors.map(e => ({
+            line: e.line,
+            content: e.content,
+            message: e.message,
+          })),
+        },
+      });
+    }
+
+    if (validation.warnings.length > 0) {
+      issues.push({
+        code: 'ROBOTS_TXT_WARNINGS',
+        severity: 'notice',
+        category: 'crawlability',
+        title: 'robots.txt has potential issues',
+        description: `Found ${validation.warnings.length} warning(s) in robots.txt.`,
+        impact: 'Some crawlers may not recognize non-standard directives.',
+        howToFix: 'Review the warnings and update if needed.',
+        affectedUrls: [robotsUrl],
+        details: {
+          warnings: validation.warnings.map(w => ({
+            line: w.line,
+            content: w.content,
+            message: w.message,
+          })),
+        },
+      });
+    }
+
+    return {
+      issues,
+      data: {
+        exists: true,
+        isValid: validation.errors.length === 0,
+        lineCount: content.split('\n').length,
+        errors: validation.errors,
+        warnings: validation.warnings,
+        directives: validation.directives,
+      },
+    };
+  } catch (error) {
+    return {
+      issues,
+      data: {
+        exists: false,
+        isValid: true,
+        lineCount: 0,
+        errors: [],
+        warnings: [],
+        directives: {
+          userAgents: [],
+          sitemaps: [],
+          hasWildcardAgent: false,
+          hasCrawlDelay: false,
+        },
+      },
+    };
+  }
+}
+
+function parseAndValidate(content: string): {
+  errors: RobotsError[];
+  warnings: RobotsWarning[];
+  directives: RobotsValidationData['directives'];
+} {
+  const errors: RobotsError[] = [];
+  const warnings: RobotsWarning[] = [];
+  const userAgents: string[] = [];
+  const sitemaps: string[] = [];
+  let hasWildcardAgent = false;
+  let hasCrawlDelay = false;
+
+  const lines = content.split('\n');
+  let currentUserAgent: string | null = null;
+  let hasDirectiveAfterUserAgent = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const lineNum = i + 1;
+    const rawLine = lines[i];
+    const line = rawLine.trim();
+
+    // Skip empty lines
+    if (line === '') {
+      continue;
+    }
+
+    // Skip comments
+    if (line.startsWith('#')) {
+      continue;
+    }
+
+    // Handle inline comments
+    const commentIndex = line.indexOf('#');
+    const effectiveLine = commentIndex > 0 ? line.substring(0, commentIndex).trim() : line;
+
+    if (effectiveLine === '') {
+      continue;
+    }
+
+    // Check for colon (required separator)
+    const colonIndex = effectiveLine.indexOf(':');
+    if (colonIndex === -1) {
+      errors.push({
+        line: lineNum,
+        content: rawLine,
+        message: 'Missing colon separator. Format should be "Directive: value"',
+      });
+      continue;
+    }
+
+    const directive = effectiveLine.substring(0, colonIndex).trim().toLowerCase();
+    const value = effectiveLine.substring(colonIndex + 1).trim();
+
+    // Check if directive is recognized
+    if (!ALL_DIRECTIVES.includes(directive)) {
+      errors.push({
+        line: lineNum,
+        content: rawLine,
+        message: `Unknown directive "${directive}". Valid directives: User-agent, Disallow, Allow, Sitemap`,
+      });
+      continue;
+    }
+
+    // Check for non-standard directives
+    if (EXTENDED_DIRECTIVES.includes(directive) && !STANDARD_DIRECTIVES.includes(directive)) {
+      warnings.push({
+        line: lineNum,
+        content: rawLine,
+        message: `"${directive}" is a non-standard directive. Not all crawlers support it.`,
+      });
+    }
+
+    // Directive-specific validation
+    switch (directive) {
+      case 'user-agent':
+        if (value === '') {
+          errors.push({
+            line: lineNum,
+            content: rawLine,
+            message: 'User-agent value cannot be empty',
+          });
+        } else {
+          currentUserAgent = value;
+          hasDirectiveAfterUserAgent = false;
+          if (!userAgents.includes(value)) {
+            userAgents.push(value);
+          }
+          if (value === '*') {
+            hasWildcardAgent = true;
+          }
+        }
+        break;
+
+      case 'disallow':
+      case 'allow':
+        if (currentUserAgent === null) {
+          errors.push({
+            line: lineNum,
+            content: rawLine,
+            message: `${directive} must come after a User-agent directive`,
+          });
+        } else {
+          hasDirectiveAfterUserAgent = true;
+          // Value can be empty for Disallow (means allow all)
+          // Check for invalid characters in path
+          if (value && !isValidPath(value)) {
+            warnings.push({
+              line: lineNum,
+              content: rawLine,
+              message: `Path "${value}" contains unusual characters`,
+            });
+          }
+        }
+        break;
+
+      case 'sitemap':
+        // Sitemap can appear anywhere
+        if (value === '') {
+          errors.push({
+            line: lineNum,
+            content: rawLine,
+            message: 'Sitemap URL cannot be empty',
+          });
+        } else if (!isValidUrl(value)) {
+          errors.push({
+            line: lineNum,
+            content: rawLine,
+            message: `Invalid sitemap URL: "${value}"`,
+          });
+        } else {
+          sitemaps.push(value);
+        }
+        break;
+
+      case 'crawl-delay':
+        hasCrawlDelay = true;
+        if (currentUserAgent === null) {
+          warnings.push({
+            line: lineNum,
+            content: rawLine,
+            message: 'Crawl-delay should come after a User-agent directive',
+          });
+        }
+        if (value === '' || isNaN(parseFloat(value))) {
+          errors.push({
+            line: lineNum,
+            content: rawLine,
+            message: 'Crawl-delay must be a number',
+          });
+        }
+        break;
+
+      case 'host':
+        // Yandex-specific, should be a valid domain
+        if (value === '') {
+          errors.push({
+            line: lineNum,
+            content: rawLine,
+            message: 'Host value cannot be empty',
+          });
+        }
+        break;
+    }
+  }
+
+  return {
+    errors,
+    warnings,
+    directives: {
+      userAgents,
+      sitemaps,
+      hasWildcardAgent,
+      hasCrawlDelay,
+    },
+  };
+}
+
+function isValidPath(path: string): boolean {
+  // Basic path validation
+  // Paths should start with / or be * or $
+  if (path === '' || path === '*' || path === '$') {
+    return true;
+  }
+  if (!path.startsWith('/') && !path.startsWith('*')) {
+    return false;
+  }
+  // Check for obviously invalid characters
+  if (/[\x00-\x1f]/.test(path)) {
+    return false;
+  }
+  return true;
+}
+
+function isValidUrl(url: string): boolean {
+  try {
+    new URL(url);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/audit/checks/security-headers.ts

@@ -0,0 +1,172 @@
+// Security Headers Checks
+// Checks for CSP, X-Frame-Options, HSTS, and other security headers
+
+import { httpGet } from '../../utils/http.js';
+import type { AuditIssue } from '../types.js';
+import { ISSUE_DEFINITIONS } from '../types.js';
+
+export interface SecurityHeadersData {
+  https: boolean;
+  headers: {
+    hsts: string | null;
+    csp: string | null;
+    xFrameOptions: string | null;
+    xContentTypeOptions: string | null;
+    referrerPolicy: string | null;
+    permissionsPolicy: string | null;
+  };
+  hstsMaxAge: number | null;
+  hstsIncludesSubdomains: boolean;
+  hstsPreload: boolean;
+}
+
+/**
+ * Parse HSTS header
+ */
+function parseHSTS(header: string | null): {
+  maxAge: number | null;
+  includeSubdomains: boolean;
+  preload: boolean;
+} {
+  if (!header) {
+    return { maxAge: null, includeSubdomains: false, preload: false };
+  }
+
+  const result = { maxAge: null as number | null, includeSubdomains: false, preload: false };
+
+  const maxAgeMatch = header.match(/max-age=(\d+)/i);
+  if (maxAgeMatch) {
+    result.maxAge = parseInt(maxAgeMatch[1], 10);
+  }
+
+  result.includeSubdomains = /includesubdomains/i.test(header);
+  result.preload = /preload/i.test(header);
+
+  return result;
+}
+
+/**
+ * Analyze security headers
+ */
+export async function analyzeSecurityHeaders(url: string): Promise<{ issues: AuditIssue[]; data: SecurityHeadersData }> {
+  const issues: AuditIssue[] = [];
+
+  try {
+    const response = await httpGet<string>(url, {
+      timeout: 15000,
+      validateStatus: () => true,
+      maxRedirects: 5,
+    });
+
+    const headers = response.headers;
+    const isHttps = url.startsWith('https://');
+
+    // Extract security headers (case-insensitive)
+    const getHeader = (name: string): string | null => {
+      const key = Object.keys(headers).find((k) => k.toLowerCase() === name.toLowerCase());
+      return key ? (headers[key] as string) : null;
+    };
+
+    const hsts = getHeader('strict-transport-security');
+    const csp = getHeader('content-security-policy');
+    const xFrameOptions = getHeader('x-frame-options');
+    const xContentTypeOptions = getHeader('x-content-type-options');
+    const referrerPolicy = getHeader('referrer-policy');
+    const permissionsPolicy = getHeader('permissions-policy') || getHeader('feature-policy');
+
+    // Parse HSTS
+    const hstsData = parseHSTS(hsts);
+
+    // Check for HTTPS
+    if (!isHttps) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.NOT_HTTPS,
+        affectedUrls: [url],
+      });
+    }
+
+    // Check for HSTS
+    if (isHttps && !hsts) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.HSTS_MISSING,
+        affectedUrls: [url],
+      });
+    }
+
+    // Check for CSP
+    if (!csp) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.CSP_MISSING,
+        affectedUrls: [url],
+      });
+    }
+
+    // Check for X-Frame-Options
+    if (!xFrameOptions) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.X_FRAME_OPTIONS_MISSING,
+        affectedUrls: [url],
+      });
+    }
+
+    // Check for X-Content-Type-Options
+    if (!xContentTypeOptions) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.X_CONTENT_TYPE_OPTIONS_MISSING,
+        affectedUrls: [url],
+      });
+    }
+
+    // Check for Referrer-Policy
+    if (!referrerPolicy) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.REFERRER_POLICY_MISSING,
+        affectedUrls: [url],
+      });
+    }
+
+    // Check for Permissions-Policy
+    if (!permissionsPolicy) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.PERMISSIONS_POLICY_MISSING,
+        affectedUrls: [url],
+      });
+    }
+
+    return {
+      issues,
+      data: {
+        https: isHttps,
+        headers: {
+          hsts,
+          csp,
+          xFrameOptions,
+          xContentTypeOptions,
+          referrerPolicy,
+          permissionsPolicy,
+        },
+        hstsMaxAge: hstsData.maxAge,
+        hstsIncludesSubdomains: hstsData.includeSubdomains,
+        hstsPreload: hstsData.preload,
+      },
+    };
+  } catch (error) {
+    return {
+      issues,
+      data: {
+        https: url.startsWith('https://'),
+        headers: {
+          hsts: null,
+          csp: null,
+          xFrameOptions: null,
+          xContentTypeOptions: null,
+          referrerPolicy: null,
+          permissionsPolicy: null,
+        },
+        hstsMaxAge: null,
+        hstsIncludesSubdomains: false,
+        hstsPreload: false,
+      },
+    };
+  }
+}

package/src/audit/checks/security.ts

@@ -0,0 +1,144 @@
+import * as cheerio from 'cheerio';
+import type { AuditIssue } from '../types.js';
+import { ISSUE_DEFINITIONS } from '../types.js';
+
+export interface SecurityData {
+  isHttps: boolean;
+  hasMixedContent: boolean;
+  mixedContentUrls: string[];
+  hasHsts: boolean;
+  hstsMaxAge?: number;
+  certificateExpiry?: Date;
+  certificateIssuer?: string;
+}
+
+export async function analyzeSecurity(
+  html: string,
+  url: string,
+  headers: Record<string, string>
+): Promise<{ issues: AuditIssue[]; data: SecurityData }> {
+  const issues: AuditIssue[] = [];
+  const $ = cheerio.load(html);
+  const parsedUrl = new URL(url);
+  const isHttps = parsedUrl.protocol === 'https:';
+
+  // Check for HTTPS
+  if (!isHttps) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.NOT_HTTPS,
+      affectedUrls: [url],
+    });
+  }
+
+  // Check for mixed content (HTTP resources on HTTPS page)
+  const mixedContentUrls: string[] = [];
+  if (isHttps) {
+    // Check scripts
+    $('script[src]').each((_, el) => {
+      const src = $(el).attr('src');
+      if (src?.startsWith('http://')) {
+        mixedContentUrls.push(src);
+      }
+    });
+
+    // Check stylesheets
+    $('link[rel="stylesheet"][href]').each((_, el) => {
+      const href = $(el).attr('href');
+      if (href?.startsWith('http://')) {
+        mixedContentUrls.push(href);
+      }
+    });
+
+    // Check images
+    $('img[src]').each((_, el) => {
+      const src = $(el).attr('src');
+      if (src?.startsWith('http://')) {
+        mixedContentUrls.push(src);
+      }
+    });
+
+    // Check iframes
+    $('iframe[src]').each((_, el) => {
+      const src = $(el).attr('src');
+      if (src?.startsWith('http://')) {
+        mixedContentUrls.push(src);
+      }
+    });
+
+    if (mixedContentUrls.length > 0) {
+      issues.push({
+        ...ISSUE_DEFINITIONS.MIXED_CONTENT,
+        affectedUrls: [url],
+        details: {
+          count: mixedContentUrls.length,
+          resources: mixedContentUrls.slice(0, 5),
+        },
+      });
+    }
+  }
+
+  // Check HSTS header
+  const hstsHeader = headers['strict-transport-security'];
+  const hasHsts = !!hstsHeader;
+  let hstsMaxAge: number | undefined;
+
+  if (hstsHeader) {
+    const maxAgeMatch = hstsHeader.match(/max-age=(\d+)/);
+    if (maxAgeMatch) {
+      hstsMaxAge = parseInt(maxAgeMatch[1], 10);
+    }
+  } else if (isHttps) {
+    issues.push({
+      ...ISSUE_DEFINITIONS.HSTS_MISSING,
+      affectedUrls: [url],
+    });
+  }
+
+  const data: SecurityData = {
+    isHttps,
+    hasMixedContent: mixedContentUrls.length > 0,
+    mixedContentUrls,
+    hasHsts,
+    hstsMaxAge,
+  };
+
+  return { issues, data };
+}
+
+// Check SSL certificate
+// Note: Using fetch for isomorphic support - detailed cert info (expiry, issuer)
+// is not available via fetch API. This only verifies the cert is valid.
+export async function checkCertificate(url: string): Promise<{
+  valid: boolean;
+  expiresAt?: Date;
+  issuer?: string;
+  daysUntilExpiry?: number;
+  issues: AuditIssue[];
+}> {
+  const issues: AuditIssue[] = [];
+  const parsedUrl = new URL(url);
+
+  if (parsedUrl.protocol !== 'https:') {
+    return { valid: false, issues };
+  }
+
+  try {
+    // Use fetch to verify the certificate is valid
+    // This will throw if the certificate is invalid/expired
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 5000);
+
+    await fetch(url, {
+      method: 'HEAD',
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeoutId);
+
+    // Certificate is valid if we got here
+    return { valid: true, issues };
+  } catch (error) {
+    // Connection failed - could be cert issue or network error
+    return { valid: false, issues };
+  }
+}