@rankcli/agent-runtime 0.0.15 → 0.0.16

package/src/audit/checks/robots-validation.ts

@@ -1,373 +0,0 @@
-/**
- * robots.txt Syntax Validation
- *
- * Validates robots.txt according to Google's specification:
- * https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
- *
- * Valid directives:
- * - User-agent: <bot-name>
- * - Disallow: <path>
- * - Allow: <path>
- * - Sitemap: <url>
- *
- * Common extensions (non-standard but widely supported):
- * - Crawl-delay: <seconds>
- * - Host: <domain> (Yandex)
- */
-
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-
-export interface RobotsValidationData {
-  exists: boolean;
-  isValid: boolean;
-  lineCount: number;
-  errors: RobotsError[];
-  warnings: RobotsWarning[];
-  directives: {
-    userAgents: string[];
-    sitemaps: string[];
-    hasWildcardAgent: boolean;
-    hasCrawlDelay: boolean;
-  };
-}
-
-interface RobotsError {
-  line: number;
-  content: string;
-  message: string;
-}
-
-interface RobotsWarning {
-  line: number;
-  content: string;
-  message: string;
-}
-
-// Standard directives (case-insensitive)
-const STANDARD_DIRECTIVES = [
-  'user-agent',
-  'disallow',
-  'allow',
-  'sitemap',
-];
-
-// Common non-standard but accepted directives
-const EXTENDED_DIRECTIVES = [
-  'crawl-delay',
-  'host',
-  'clean-param', // Yandex
-  'request-rate', // Some crawlers
-];
-
-// All recognized directives
-const ALL_DIRECTIVES = [...STANDARD_DIRECTIVES, ...EXTENDED_DIRECTIVES];
-
-export async function validateRobotsTxt(
-  url: string
-): Promise<{ issues: AuditIssue[]; data: RobotsValidationData }> {
-  const issues: AuditIssue[] = [];
-  const parsedUrl = new URL(url);
-  const robotsUrl = new URL('/robots.txt', parsedUrl.origin).href;
-
-  try {
-    const response = await httpGet<string>(robotsUrl, {
-      timeout: 10000,
-      validateStatus: () => true,
-    });
-
-    if (response.status === 404) {
-      return {
-        issues,
-        data: {
-          exists: false,
-          isValid: true, // Missing is not invalid
-          lineCount: 0,
-          errors: [],
-          warnings: [],
-          directives: {
-            userAgents: [],
-            sitemaps: [],
-            hasWildcardAgent: false,
-            hasCrawlDelay: false,
-          },
-        },
-      };
-    }
-
-    const content = response.data as string;
-    const validation = parseAndValidate(content);
-
-    // Generate issues based on validation
-    if (validation.errors.length > 0) {
-      issues.push({
-        code: 'ROBOTS_TXT_INVALID_SYNTAX',
-        severity: 'warning',
-        category: 'crawlability',
-        title: 'robots.txt contains syntax errors',
-        description: `Found ${validation.errors.length} syntax error(s) in robots.txt. Invalid syntax may cause crawlers to misinterpret your rules.`,
-        impact: 'Search engines may not properly understand your crawling rules, potentially blocking or allowing unintended pages.',
-        howToFix: 'Fix the syntax errors in robots.txt. Each directive should be on its own line with format "Directive: value".',
-        affectedUrls: [robotsUrl],
-        details: {
-          errors: validation.errors.map(e => ({
-            line: e.line,
-            content: e.content,
-            message: e.message,
-          })),
-        },
-      });
-    }
-
-    if (validation.warnings.length > 0) {
-      issues.push({
-        code: 'ROBOTS_TXT_WARNINGS',
-        severity: 'notice',
-        category: 'crawlability',
-        title: 'robots.txt has potential issues',
-        description: `Found ${validation.warnings.length} warning(s) in robots.txt.`,
-        impact: 'Some crawlers may not recognize non-standard directives.',
-        howToFix: 'Review the warnings and update if needed.',
-        affectedUrls: [robotsUrl],
-        details: {
-          warnings: validation.warnings.map(w => ({
-            line: w.line,
-            content: w.content,
-            message: w.message,
-          })),
-        },
-      });
-    }
-
-    return {
-      issues,
-      data: {
-        exists: true,
-        isValid: validation.errors.length === 0,
-        lineCount: content.split('\n').length,
-        errors: validation.errors,
-        warnings: validation.warnings,
-        directives: validation.directives,
-      },
-    };
-  } catch (error) {
-    return {
-      issues,
-      data: {
-        exists: false,
-        isValid: true,
-        lineCount: 0,
-        errors: [],
-        warnings: [],
-        directives: {
-          userAgents: [],
-          sitemaps: [],
-          hasWildcardAgent: false,
-          hasCrawlDelay: false,
-        },
-      },
-    };
-  }
-}
-
-function parseAndValidate(content: string): {
-  errors: RobotsError[];
-  warnings: RobotsWarning[];
-  directives: RobotsValidationData['directives'];
-} {
-  const errors: RobotsError[] = [];
-  const warnings: RobotsWarning[] = [];
-  const userAgents: string[] = [];
-  const sitemaps: string[] = [];
-  let hasWildcardAgent = false;
-  let hasCrawlDelay = false;
-
-  const lines = content.split('\n');
-  let currentUserAgent: string | null = null;
-  let hasDirectiveAfterUserAgent = false;
-
-  for (let i = 0; i < lines.length; i++) {
-    const lineNum = i + 1;
-    const rawLine = lines[i];
-    const line = rawLine.trim();
-
-    // Skip empty lines
-    if (line === '') {
-      continue;
-    }
-
-    // Skip comments
-    if (line.startsWith('#')) {
-      continue;
-    }
-
-    // Handle inline comments
-    const commentIndex = line.indexOf('#');
-    const effectiveLine = commentIndex > 0 ? line.substring(0, commentIndex).trim() : line;
-
-    if (effectiveLine === '') {
-      continue;
-    }
-
-    // Check for colon (required separator)
-    const colonIndex = effectiveLine.indexOf(':');
-    if (colonIndex === -1) {
-      errors.push({
-        line: lineNum,
-        content: rawLine,
-        message: 'Missing colon separator. Format should be "Directive: value"',
-      });
-      continue;
-    }
-
-    const directive = effectiveLine.substring(0, colonIndex).trim().toLowerCase();
-    const value = effectiveLine.substring(colonIndex + 1).trim();
-
-    // Check if directive is recognized
-    if (!ALL_DIRECTIVES.includes(directive)) {
-      errors.push({
-        line: lineNum,
-        content: rawLine,
-        message: `Unknown directive "${directive}". Valid directives: User-agent, Disallow, Allow, Sitemap`,
-      });
-      continue;
-    }
-
-    // Check for non-standard directives
-    if (EXTENDED_DIRECTIVES.includes(directive) && !STANDARD_DIRECTIVES.includes(directive)) {
-      warnings.push({
-        line: lineNum,
-        content: rawLine,
-        message: `"${directive}" is a non-standard directive. Not all crawlers support it.`,
-      });
-    }
-
-    // Directive-specific validation
-    switch (directive) {
-      case 'user-agent':
-        if (value === '') {
-          errors.push({
-            line: lineNum,
-            content: rawLine,
-            message: 'User-agent value cannot be empty',
-          });
-        } else {
-          currentUserAgent = value;
-          hasDirectiveAfterUserAgent = false;
-          if (!userAgents.includes(value)) {
-            userAgents.push(value);
-          }
-          if (value === '*') {
-            hasWildcardAgent = true;
-          }
-        }
-        break;
-
-      case 'disallow':
-      case 'allow':
-        if (currentUserAgent === null) {
-          errors.push({
-            line: lineNum,
-            content: rawLine,
-            message: `${directive} must come after a User-agent directive`,
-          });
-        } else {
-          hasDirectiveAfterUserAgent = true;
-          // Value can be empty for Disallow (means allow all)
-          // Check for invalid characters in path
-          if (value && !isValidPath(value)) {
-            warnings.push({
-              line: lineNum,
-              content: rawLine,
-              message: `Path "${value}" contains unusual characters`,
-            });
-          }
-        }
-        break;
-
-      case 'sitemap':
-        // Sitemap can appear anywhere
-        if (value === '') {
-          errors.push({
-            line: lineNum,
-            content: rawLine,
-            message: 'Sitemap URL cannot be empty',
-          });
-        } else if (!isValidUrl(value)) {
-          errors.push({
-            line: lineNum,
-            content: rawLine,
-            message: `Invalid sitemap URL: "${value}"`,
-          });
-        } else {
-          sitemaps.push(value);
-        }
-        break;
-
-      case 'crawl-delay':
-        hasCrawlDelay = true;
-        if (currentUserAgent === null) {
-          warnings.push({
-            line: lineNum,
-            content: rawLine,
-            message: 'Crawl-delay should come after a User-agent directive',
-          });
-        }
-        if (value === '' || isNaN(parseFloat(value))) {
-          errors.push({
-            line: lineNum,
-            content: rawLine,
-            message: 'Crawl-delay must be a number',
-          });
-        }
-        break;
-
-      case 'host':
-        // Yandex-specific, should be a valid domain
-        if (value === '') {
-          errors.push({
-            line: lineNum,
-            content: rawLine,
-            message: 'Host value cannot be empty',
-          });
-        }
-        break;
-    }
-  }
-
-  return {
-    errors,
-    warnings,
-    directives: {
-      userAgents,
-      sitemaps,
-      hasWildcardAgent,
-      hasCrawlDelay,
-    },
-  };
-}
-
-function isValidPath(path: string): boolean {
-  // Basic path validation
-  // Paths should start with / or be * or $
-  if (path === '' || path === '*' || path === '$') {
-    return true;
-  }
-  if (!path.startsWith('/') && !path.startsWith('*')) {
-    return false;
-  }
-  // Check for obviously invalid characters
-  if (/[\x00-\x1f]/.test(path)) {
-    return false;
-  }
-  return true;
-}
-
-function isValidUrl(url: string): boolean {
-  try {
-    new URL(url);
-    return true;
-  } catch {
-    return false;
-  }
-}

package/src/audit/checks/security-headers.ts

@@ -1,188 +0,0 @@
-// Security Headers Checks
-// Checks for CSP, X-Frame-Options, HSTS, and other security headers
-
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-import { ISSUE_DEFINITIONS } from '../types.js';
-
-export interface SecurityHeadersData {
-  https: boolean;
-  headers: {
-    hsts: string | null;
-    csp: string | null;
-    xFrameOptions: string | null;
-    xContentTypeOptions: string | null;
-    referrerPolicy: string | null;
-    permissionsPolicy: string | null;
-  };
-  hstsMaxAge: number | null;
-  hstsIncludesSubdomains: boolean;
-  hstsPreload: boolean;
-}
-
-/**
- * Parse HSTS header
- */
-function parseHSTS(header: string | null): {
-  maxAge: number | null;
-  includeSubdomains: boolean;
-  preload: boolean;
-} {
-  if (!header) {
-    return { maxAge: null, includeSubdomains: false, preload: false };
-  }
-
-  const result = { maxAge: null as number | null, includeSubdomains: false, preload: false };
-
-  const maxAgeMatch = header.match(/max-age=(\d+)/i);
-  if (maxAgeMatch) {
-    result.maxAge = parseInt(maxAgeMatch[1], 10);
-  }
-
-  result.includeSubdomains = /includesubdomains/i.test(header);
-  result.preload = /preload/i.test(header);
-
-  return result;
-}
-
-/**
- * Analyze security headers
- */
-export async function analyzeSecurityHeaders(url: string): Promise<{ issues: AuditIssue[]; data: SecurityHeadersData }> {
-  const issues: AuditIssue[] = [];
-
-  try {
-    const response = await httpGet<string>(url, {
-      timeout: 15000,
-      validateStatus: () => true,
-      maxRedirects: 5,
-    });
-
-    const headers = response.headers;
-    // Use URL parsing for robust protocol detection (handles case variations like Https://)
-    let isHttps = false;
-    try {
-      const parsedUrl = new URL(url);
-      isHttps = parsedUrl.protocol === 'https:';
-    } catch {
-      // If URL parsing fails, fall back to startsWith check
-      isHttps = url.toLowerCase().startsWith('https://');
-    }
-
-    // Extract security headers (case-insensitive)
-    const getHeader = (name: string): string | null => {
-      const key = Object.keys(headers).find((k) => k.toLowerCase() === name.toLowerCase());
-      return key ? (headers[key] as string) : null;
-    };
-
-    const hsts = getHeader('strict-transport-security');
-    const csp = getHeader('content-security-policy');
-    const xFrameOptions = getHeader('x-frame-options');
-    const xContentTypeOptions = getHeader('x-content-type-options');
-    const referrerPolicy = getHeader('referrer-policy');
-    const permissionsPolicy = getHeader('permissions-policy') || getHeader('feature-policy');
-
-    // Parse HSTS
-    const hstsData = parseHSTS(hsts);
-
-    // Check for HTTPS
-    if (!isHttps) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.NOT_HTTPS,
-        affectedUrls: [url],
-      });
-    }
-
-    // Check for HSTS
-    if (isHttps && !hsts) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.HSTS_MISSING,
-        affectedUrls: [url],
-      });
-    }
-
-    // Check for CSP
-    if (!csp) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.CSP_MISSING,
-        affectedUrls: [url],
-      });
-    }
-
-    // Check for X-Frame-Options
-    if (!xFrameOptions) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.X_FRAME_OPTIONS_MISSING,
-        affectedUrls: [url],
-      });
-    }
-
-    // Check for X-Content-Type-Options
-    if (!xContentTypeOptions) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.X_CONTENT_TYPE_OPTIONS_MISSING,
-        affectedUrls: [url],
-      });
-    }
-
-    // Check for Referrer-Policy
-    if (!referrerPolicy) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.REFERRER_POLICY_MISSING,
-        affectedUrls: [url],
-      });
-    }
-
-    // Check for Permissions-Policy
-    if (!permissionsPolicy) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.PERMISSIONS_POLICY_MISSING,
-        affectedUrls: [url],
-      });
-    }
-
-    return {
-      issues,
-      data: {
-        https: isHttps,
-        headers: {
-          hsts,
-          csp,
-          xFrameOptions,
-          xContentTypeOptions,
-          referrerPolicy,
-          permissionsPolicy,
-        },
-        hstsMaxAge: hstsData.maxAge,
-        hstsIncludesSubdomains: hstsData.includeSubdomains,
-        hstsPreload: hstsData.preload,
-      },
-    };
-  } catch (error) {
-    // Use URL parsing for robust protocol detection in error case too
-    let isHttps = false;
-    try {
-      const parsedUrl = new URL(url);
-      isHttps = parsedUrl.protocol === 'https:';
-    } catch {
-      isHttps = url.toLowerCase().startsWith('https://');
-    }
-    return {
-      issues,
-      data: {
-        https: isHttps,
-        headers: {
-          hsts: null,
-          csp: null,
-          xFrameOptions: null,
-          xContentTypeOptions: null,
-          referrerPolicy: null,
-          permissionsPolicy: null,
-        },
-        hstsMaxAge: null,
-        hstsIncludesSubdomains: false,
-        hstsPreload: false,
-      },
-    };
-  }
-}

package/src/audit/checks/security.ts

@@ -1,144 +0,0 @@
-import * as cheerio from 'cheerio';
-import type { AuditIssue } from '../types.js';
-import { ISSUE_DEFINITIONS } from '../types.js';
-
-export interface SecurityData {
-  isHttps: boolean;
-  hasMixedContent: boolean;
-  mixedContentUrls: string[];
-  hasHsts: boolean;
-  hstsMaxAge?: number;
-  certificateExpiry?: Date;
-  certificateIssuer?: string;
-}
-
-export async function analyzeSecurity(
-  html: string,
-  url: string,
-  headers: Record<string, string>
-): Promise<{ issues: AuditIssue[]; data: SecurityData }> {
-  const issues: AuditIssue[] = [];
-  const $ = cheerio.load(html);
-  const parsedUrl = new URL(url);
-  const isHttps = parsedUrl.protocol === 'https:';
-
-  // Check for HTTPS
-  if (!isHttps) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.NOT_HTTPS,
-      affectedUrls: [url],
-    });
-  }
-
-  // Check for mixed content (HTTP resources on HTTPS page)
-  const mixedContentUrls: string[] = [];
-  if (isHttps) {
-    // Check scripts
-    $('script[src]').each((_, el) => {
-      const src = $(el).attr('src');
-      if (src?.startsWith('http://')) {
-        mixedContentUrls.push(src);
-      }
-    });
-
-    // Check stylesheets
-    $('link[rel="stylesheet"][href]').each((_, el) => {
-      const href = $(el).attr('href');
-      if (href?.startsWith('http://')) {
-        mixedContentUrls.push(href);
-      }
-    });
-
-    // Check images
-    $('img[src]').each((_, el) => {
-      const src = $(el).attr('src');
-      if (src?.startsWith('http://')) {
-        mixedContentUrls.push(src);
-      }
-    });
-
-    // Check iframes
-    $('iframe[src]').each((_, el) => {
-      const src = $(el).attr('src');
-      if (src?.startsWith('http://')) {
-        mixedContentUrls.push(src);
-      }
-    });
-
-    if (mixedContentUrls.length > 0) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.MIXED_CONTENT,
-        affectedUrls: [url],
-        details: {
-          count: mixedContentUrls.length,
-          resources: mixedContentUrls.slice(0, 5),
-        },
-      });
-    }
-  }
-
-  // Check HSTS header
-  const hstsHeader = headers['strict-transport-security'];
-  const hasHsts = !!hstsHeader;
-  let hstsMaxAge: number | undefined;
-
-  if (hstsHeader) {
-    const maxAgeMatch = hstsHeader.match(/max-age=(\d+)/);
-    if (maxAgeMatch) {
-      hstsMaxAge = parseInt(maxAgeMatch[1], 10);
-    }
-  } else if (isHttps) {
-    issues.push({
-      ...ISSUE_DEFINITIONS.HSTS_MISSING,
-      affectedUrls: [url],
-    });
-  }
-
-  const data: SecurityData = {
-    isHttps,
-    hasMixedContent: mixedContentUrls.length > 0,
-    mixedContentUrls,
-    hasHsts,
-    hstsMaxAge,
-  };
-
-  return { issues, data };
-}
-
-// Check SSL certificate
-// Note: Using fetch for isomorphic support - detailed cert info (expiry, issuer)
-// is not available via fetch API. This only verifies the cert is valid.
-export async function checkCertificate(url: string): Promise<{
-  valid: boolean;
-  expiresAt?: Date;
-  issuer?: string;
-  daysUntilExpiry?: number;
-  issues: AuditIssue[];
-}> {
-  const issues: AuditIssue[] = [];
-  const parsedUrl = new URL(url);
-
-  if (parsedUrl.protocol !== 'https:') {
-    return { valid: false, issues };
-  }
-
-  try {
-    // Use fetch to verify the certificate is valid
-    // This will throw if the certificate is invalid/expired
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 5000);
-
-    await fetch(url, {
-      method: 'HEAD',
-      signal: controller.signal,
-    });
-
-    clearTimeout(timeoutId);
-
-    // Certificate is valid if we got here
-    return { valid: true, issues };
-  } catch (error) {
-    // Connection failed - could be cert issue or network error
-    return { valid: false, issues };
-  }
-}