@rankcli/agent-runtime 0.0.15 → 0.0.16

package/src/audit/checks/caching-headers.ts

@@ -1,305 +0,0 @@
-/**
- * Browser Caching Headers Check
- *
- * Validates HTTP caching headers for optimal performance:
- * - Cache-Control header
- * - Expires header
- * - ETag header
- * - Last-Modified header
- *
- * Proper caching improves page load times for repeat visitors
- * and reduces server load.
- */
-
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-
-export interface CachingHeadersData {
-  htmlCaching: {
-    cacheControl?: string;
-    expires?: string;
-    etag?: string;
-    lastModified?: string;
-    maxAge?: number;
-    isPublic: boolean;
-    isPrivate: boolean;
-    noStore: boolean;
-    noCache: boolean;
-  };
-  staticAssets: AssetCachingInfo[];
-  issues: {
-    noCaching: number;
-    shortTtl: number;
-    properlyConfigured: number;
-  };
-}
-
-interface AssetCachingInfo {
-  url: string;
-  type: string;
-  cacheControl?: string;
-  maxAge?: number;
-  hasCaching: boolean;
-}
-
-// Recommended cache durations (in seconds)
-const CACHE_RECOMMENDATIONS = {
-  html: 0, // HTML should generally not be cached or have short TTL
-  css: 31536000, // 1 year for versioned assets
-  js: 31536000,
-  images: 31536000,
-  fonts: 31536000,
-  minimum: 86400, // 1 day minimum for static assets
-};
-
-export async function analyzeCachingHeaders(
-  url: string,
-  headers?: Record<string, string>
-): Promise<{ issues: AuditIssue[]; data: CachingHeadersData }> {
-  const issues: AuditIssue[] = [];
-
-  // Analyze HTML page headers
-  let htmlHeaders = headers;
-  if (!htmlHeaders) {
-    try {
-      const response = await httpGet<string>(url, {
-        timeout: 10000,
-        validateStatus: () => true,
-      });
-      htmlHeaders = response.headers as Record<string, string>;
-    } catch {
-      htmlHeaders = {};
-    }
-  }
-
-  const htmlCaching = parseCacheHeaders(htmlHeaders);
-
-  // Check HTML caching (should be minimal or no caching)
-  if (htmlCaching.maxAge && htmlCaching.maxAge > 3600) {
-    issues.push({
-      code: 'CACHING_HTML_TOO_LONG',
-      severity: 'notice',
-      category: 'performance',
-      title: 'HTML page has long cache duration',
-      description: `HTML page has Cache-Control max-age of ${htmlCaching.maxAge} seconds (${Math.round(htmlCaching.maxAge / 3600)} hours). HTML content changes frequently.`,
-      impact: 'Users may see stale content. Content updates may not appear immediately.',
-      howToFix: 'Set shorter cache duration for HTML: Cache-Control: no-cache or max-age=0, must-revalidate',
-      affectedUrls: [url],
-    });
-  }
-
-  // Sample static assets to check their caching
-  const staticAssets: AssetCachingInfo[] = [];
-  const assetUrls = await getStaticAssetUrls(url);
-
-  let noCaching = 0;
-  let shortTtl = 0;
-  let properlyConfigured = 0;
-
-  // Check up to 5 static assets
-  const assetsToCheck = assetUrls.slice(0, 5);
-
-  for (const assetUrl of assetsToCheck) {
-    try {
-      const assetResponse = await httpGet<string>(assetUrl, {
-        timeout: 5000,
-        validateStatus: () => true,
-      });
-
-      const assetCaching = parseCacheHeaders(assetResponse.headers as Record<string, string>);
-      const type = getAssetType(assetUrl);
-
-      const assetInfo: AssetCachingInfo = {
-        url: assetUrl,
-        type,
-        cacheControl: assetCaching.cacheControl,
-        maxAge: assetCaching.maxAge,
-        hasCaching: Boolean(assetCaching.cacheControl || assetCaching.expires),
-      };
-
-      staticAssets.push(assetInfo);
-
-      if (!assetInfo.hasCaching) {
-        noCaching++;
-      } else if (assetCaching.maxAge && assetCaching.maxAge < CACHE_RECOMMENDATIONS.minimum) {
-        shortTtl++;
-      } else {
-        properlyConfigured++;
-      }
-    } catch {
-      // Asset check failed, skip
-    }
-  }
-
-  // Generate issues for static assets
-  if (noCaching > 0) {
-    issues.push({
-      code: 'CACHING_STATIC_NO_CACHE',
-      severity: 'warning',
-      category: 'performance',
-      title: 'Static assets missing cache headers',
-      description: `${noCaching} static asset(s) have no Cache-Control header. This forces browsers to re-download on every visit.`,
-      impact: 'Slower page loads for repeat visitors. Increased bandwidth usage.',
-      howToFix: 'Add Cache-Control headers to static assets. For versioned files (with hash in filename), use: Cache-Control: public, max-age=31536000, immutable',
-      affectedUrls: staticAssets.filter(a => !a.hasCaching).map(a => a.url),
-      details: {
-        recommendation: {
-          'versioned assets (with hash)': 'Cache-Control: public, max-age=31536000, immutable',
-          'non-versioned assets': 'Cache-Control: public, max-age=86400',
-        },
-      },
-    });
-  }
-
-  if (shortTtl > 0) {
-    issues.push({
-      code: 'CACHING_STATIC_SHORT_TTL',
-      severity: 'notice',
-      category: 'performance',
-      title: 'Static assets have short cache duration',
-      description: `${shortTtl} static asset(s) have cache duration under 1 day. Consider longer caching for better performance.`,
-      impact: 'Assets are re-downloaded more often than necessary.',
-      howToFix: 'Increase max-age for static assets. Use versioned filenames (hash in filename) to enable long-term caching.',
-      affectedUrls: staticAssets.filter(a => a.maxAge && a.maxAge < CACHE_RECOMMENDATIONS.minimum).map(a => a.url),
-    });
-  }
-
-  // Check for missing ETag or Last-Modified
-  if (!htmlCaching.etag && !htmlCaching.lastModified) {
-    issues.push({
-      code: 'CACHING_NO_VALIDATORS',
-      severity: 'notice',
-      category: 'performance',
-      title: 'Missing cache validators (ETag/Last-Modified)',
-      description: 'Page is missing both ETag and Last-Modified headers. These allow efficient cache validation.',
-      impact: 'Browsers cannot efficiently validate if cached content is still fresh.',
-      howToFix: 'Configure your server to send ETag or Last-Modified headers. Most web servers do this automatically for static files.',
-      affectedUrls: [url],
-    });
-  }
-
-  return {
-    issues,
-    data: {
-      htmlCaching,
-      staticAssets,
-      issues: {
-        noCaching,
-        shortTtl,
-        properlyConfigured,
-      },
-    },
-  };
-}
-
-function parseCacheHeaders(headers: Record<string, string>): CachingHeadersData['htmlCaching'] {
-  const cacheControl = headers['cache-control'] || headers['Cache-Control'];
-  const expires = headers['expires'] || headers['Expires'];
-  const etag = headers['etag'] || headers['ETag'];
-  const lastModified = headers['last-modified'] || headers['Last-Modified'];
-
-  let maxAge: number | undefined;
-  let isPublic = false;
-  let isPrivate = false;
-  let noStore = false;
-  let noCache = false;
-
-  if (cacheControl) {
-    // Parse max-age
-    const maxAgeMatch = cacheControl.match(/max-age=(\d+)/i);
-    if (maxAgeMatch) {
-      maxAge = parseInt(maxAgeMatch[1], 10);
-    }
-
-    // Parse directives
-    const lowerCC = cacheControl.toLowerCase();
-    isPublic = lowerCC.includes('public');
-    isPrivate = lowerCC.includes('private');
-    noStore = lowerCC.includes('no-store');
-    noCache = lowerCC.includes('no-cache');
-  }
-
-  return {
-    cacheControl,
-    expires,
-    etag,
-    lastModified,
-    maxAge,
-    isPublic,
-    isPrivate,
-    noStore,
-    noCache,
-  };
-}
-
-async function getStaticAssetUrls(pageUrl: string): Promise<string[]> {
-  const assets: string[] = [];
-
-  try {
-    const response = await httpGet<string>(pageUrl, {
-      timeout: 10000,
-      validateStatus: () => true,
-    });
-
-    const html = response.data as string;
-    const baseUrl = new URL(pageUrl);
-
-    // Extract CSS files
-    const cssMatches = html.match(/href="([^"]+\.css[^"]*)"/gi);
-    if (cssMatches) {
-      for (const match of cssMatches.slice(0, 2)) {
-        const href = match.match(/href="([^"]+)"/)?.[1];
-        if (href) {
-          try {
-            assets.push(new URL(href, baseUrl).href);
-          } catch {
-            // Invalid URL
-          }
-        }
-      }
-    }
-
-    // Extract JS files
-    const jsMatches = html.match(/src="([^"]+\.js[^"]*)"/gi);
-    if (jsMatches) {
-      for (const match of jsMatches.slice(0, 2)) {
-        const src = match.match(/src="([^"]+)"/)?.[1];
-        if (src) {
-          try {
-            assets.push(new URL(src, baseUrl).href);
-          } catch {
-            // Invalid URL
-          }
-        }
-      }
-    }
-
-    // Extract image files
-    const imgMatches = html.match(/src="([^"]+\.(png|jpg|jpeg|gif|webp|svg)[^"]*)"/gi);
-    if (imgMatches) {
-      for (const match of imgMatches.slice(0, 1)) {
-        const src = match.match(/src="([^"]+)"/)?.[1];
-        if (src) {
-          try {
-            assets.push(new URL(src, baseUrl).href);
-          } catch {
-            // Invalid URL
-          }
-        }
-      }
-    }
-  } catch {
-    // Failed to fetch page
-  }
-
-  return assets;
-}
-
-function getAssetType(url: string): string {
-  const lowerUrl = url.toLowerCase();
-  if (lowerUrl.includes('.css')) return 'css';
-  if (lowerUrl.includes('.js')) return 'javascript';
-  if (lowerUrl.match(/\.(png|jpg|jpeg|gif|webp|svg|ico)/)) return 'image';
-  if (lowerUrl.match(/\.(woff|woff2|ttf|otf|eot)/)) return 'font';
-  return 'other';
-}

package/src/audit/checks/canonical-advanced.ts

@@ -1,150 +0,0 @@
-import * as cheerio from 'cheerio';
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-import { ISSUE_DEFINITIONS } from '../types.js';
-
-export interface CanonicalData {
-  canonical?: string;
-  isSelfReferencing: boolean;
-  isCrossDomain: boolean;
-  canonicalChain?: string[];
-  finalCanonical?: string;
-}
-
-export async function analyzeCanonicalAdvanced(
-  html: string,
-  url: string,
-  options: { checkChain?: boolean; sitemapUrls?: string[] } = {}
-): Promise<{ issues: AuditIssue[]; data: CanonicalData }> {
-  const issues: AuditIssue[] = [];
-  const $ = cheerio.load(html);
-
-  const canonicalLink = $('link[rel="canonical"]').attr('href');
-  const baseUrl = new URL(url);
-
-  const data: CanonicalData = {
-    canonical: canonicalLink,
-    isSelfReferencing: false,
-    isCrossDomain: false,
-  };
-
-  if (!canonicalLink) {
-    return { issues, data };
-  }
-
-  // Resolve canonical URL
-  let canonicalUrl: URL;
-  try {
-    canonicalUrl = new URL(canonicalLink, url);
-  } catch {
-    return { issues, data };
-  }
-
-  // Check if self-referencing
-  const normalizedUrl = normalizeUrl(url);
-  const normalizedCanonical = normalizeUrl(canonicalUrl.href);
-  data.isSelfReferencing = normalizedUrl === normalizedCanonical;
-
-  // Check for cross-domain canonical
-  if (canonicalUrl.hostname !== baseUrl.hostname) {
-    data.isCrossDomain = true;
-    issues.push({
-      ...ISSUE_DEFINITIONS.CANONICAL_CROSS_DOMAIN,
-      affectedUrls: [url],
-      details: {
-        canonical: canonicalUrl.href,
-        sourceDomain: baseUrl.hostname,
-        targetDomain: canonicalUrl.hostname,
-      },
-    });
-  }
-
-  // Check for canonical chain if enabled
-  if (options.checkChain && !data.isSelfReferencing) {
-    const chain = await followCanonicalChain(canonicalUrl.href, [url]);
-    if (chain.length > 2) {
-      data.canonicalChain = chain;
-      data.finalCanonical = chain[chain.length - 1];
-      issues.push({
-        ...ISSUE_DEFINITIONS.CANONICAL_CHAIN,
-        affectedUrls: [url],
-        details: {
-          chain,
-          depth: chain.length,
-        },
-      });
-    }
-  }
-
-  // Check if canonical matches sitemap
-  if (options.sitemapUrls && options.sitemapUrls.length > 0) {
-    const sitemapNormalized = options.sitemapUrls.map(normalizeUrl);
-    const urlInSitemap = sitemapNormalized.includes(normalizedUrl);
-    const canonicalInSitemap = sitemapNormalized.includes(normalizedCanonical);
-
-    if (urlInSitemap && !canonicalInSitemap && !data.isSelfReferencing) {
-      issues.push({
-        ...ISSUE_DEFINITIONS.CANONICAL_MISMATCH_SITEMAP,
-        affectedUrls: [url],
-        details: {
-          pageUrl: url,
-          canonical: canonicalUrl.href,
-          message: 'Page is in sitemap but its canonical URL is not',
-        },
-      });
-    }
-  }
-
-  return { issues, data };
-}
-
-async function followCanonicalChain(
-  url: string,
-  chain: string[] = [],
-  maxDepth: number = 5
-): Promise<string[]> {
-  if (chain.length >= maxDepth) return chain;
-
-  try {
-    const response = await httpGet<string>(url, {
-      timeout: 5000,
-      maxRedirects: 5,
-      validateStatus: (status) => status < 400,
-    });
-
-    const $ = cheerio.load(response.data);
-    const nextCanonical = $('link[rel="canonical"]').attr('href');
-
-    if (!nextCanonical) {
-      return [...chain, url];
-    }
-
-    const resolvedCanonical = new URL(nextCanonical, url).href;
-    const normalizedCurrent = normalizeUrl(url);
-    const normalizedNext = normalizeUrl(resolvedCanonical);
-
-    // Self-referencing, end of chain
-    if (normalizedCurrent === normalizedNext) {
-      return [...chain, url];
-    }
-
-    // Circular reference detection
-    if (chain.some(u => normalizeUrl(u) === normalizedNext)) {
-      return [...chain, url, resolvedCanonical + ' (circular)'];
-    }
-
-    return followCanonicalChain(resolvedCanonical, [...chain, url], maxDepth);
-  } catch {
-    return [...chain, url];
-  }
-}
-
-function normalizeUrl(urlStr: string): string {
-  try {
-    const parsed = new URL(urlStr);
-    // Remove trailing slash, lowercase hostname
-    return `${parsed.protocol}//${parsed.hostname.toLowerCase()}${parsed.pathname.replace(/\/$/, '')}`;
-  } catch {
-    return urlStr.toLowerCase().replace(/\/$/, '');
-  }
-}

package/src/audit/checks/canonical-domain.ts

@@ -1,196 +0,0 @@
-/**
- * Canonical Domain Check
- *
- * Verifies that the site properly handles www vs non-www versions
- * to avoid duplicate content issues.
- *
- * Best practice: One version should redirect to the other (301 redirect).
- * Both versions serving content = duplicate content problem.
- */
-
-import { httpGet } from '../../utils/http.js';
-import type { AuditIssue } from '../types.js';
-
-export interface CanonicalDomainData {
-  primaryDomain: string;
-  wwwVersion: {
-    url: string;
-    status: number | null;
-    redirectsTo?: string;
-    accessible: boolean;
-  };
-  nonWwwVersion: {
-    url: string;
-    status: number | null;
-    redirectsTo?: string;
-    accessible: boolean;
-  };
-  hasProperRedirect: boolean;
-  preferredVersion: 'www' | 'non-www' | 'unknown';
-  httpsRedirect: {
-    httpToHttps: boolean;
-    httpsStatus: number | null;
-  };
-}
-
-export async function analyzeCanonicalDomain(
-  url: string
-): Promise<{ issues: AuditIssue[]; data: CanonicalDomainData }> {
-  const issues: AuditIssue[] = [];
-  const parsedUrl = new URL(url);
-  const hostname = parsedUrl.hostname;
-
-  // Determine www and non-www versions
-  const isWww = hostname.startsWith('www.');
-  const wwwHostname = isWww ? hostname : `www.${hostname}`;
-  const nonWwwHostname = isWww ? hostname.substring(4) : hostname;
-
-  // Skip if hostname has multiple subdomains (e.g., api.staging.example.com)
-  const parts = nonWwwHostname.split('.');
-  if (parts.length > 2 && !nonWwwHostname.match(/\.(co|com|org|net|gov)\.[a-z]{2}$/i)) {
-    // This is a subdomain, not a main domain
-    return {
-      issues: [],
-      data: {
-        primaryDomain: hostname,
-        wwwVersion: { url: '', status: null, accessible: false },
-        nonWwwVersion: { url: '', status: null, accessible: false },
-        hasProperRedirect: true, // N/A for subdomains
-        preferredVersion: 'unknown',
-        httpsRedirect: { httpToHttps: false, httpsStatus: null },
-      },
-    };
-  }
-
-  const wwwUrl = `https://${wwwHostname}${parsedUrl.pathname}`;
-  const nonWwwUrl = `https://${nonWwwHostname}${parsedUrl.pathname}`;
-
-  // Test both versions
-  const [wwwResult, nonWwwResult] = await Promise.all([
-    testUrl(wwwUrl),
-    testUrl(nonWwwUrl),
-  ]);
-
-  // Determine if proper redirect exists
-  let hasProperRedirect = false;
-  let preferredVersion: 'www' | 'non-www' | 'unknown' = 'unknown';
-
-  // Check if www redirects to non-www
-  if (wwwResult.status && wwwResult.status >= 300 && wwwResult.status < 400) {
-    if (wwwResult.redirectsTo?.includes(nonWwwHostname)) {
-      hasProperRedirect = true;
-      preferredVersion = 'non-www';
-    }
-  }
-
-  // Check if non-www redirects to www
-  if (nonWwwResult.status && nonWwwResult.status >= 300 && nonWwwResult.status < 400) {
-    if (nonWwwResult.redirectsTo?.includes(wwwHostname)) {
-      hasProperRedirect = true;
-      preferredVersion = 'www';
-    }
-  }
-
-  // Both accessible without redirect = problem
-  const bothAccessible =
-    wwwResult.accessible &&
-    nonWwwResult.accessible &&
-    wwwResult.status === 200 &&
-    nonWwwResult.status === 200;
-
-  if (bothAccessible && !hasProperRedirect) {
-    issues.push({
-      code: 'CANONICAL_WWW_DUPLICATE',
-      severity: 'warning',
-      category: 'crawlability',
-      title: 'Both www and non-www versions accessible',
-      description: `Both ${wwwUrl} and ${nonWwwUrl} return 200 OK. This creates duplicate content.`,
-      impact: 'Search engines may index both versions, diluting page authority and causing ranking issues.',
-      howToFix: 'Set up a 301 redirect from one version to the other. Most sites prefer non-www, but either works. Configure at server/CDN level.',
-      affectedUrls: [wwwUrl, nonWwwUrl],
-      details: {
-        wwwStatus: wwwResult.status,
-        nonWwwStatus: nonWwwResult.status,
-        recommendation: 'Choose one version as canonical and 301 redirect the other.',
-      },
-    });
-  }
-
-  // Check HTTP to HTTPS redirect
-  const httpUrl = `http://${hostname}${parsedUrl.pathname}`;
-  const httpResult = await testUrl(httpUrl);
-
-  let httpToHttps = false;
-  if (httpResult.status && httpResult.status >= 300 && httpResult.status < 400) {
-    if (httpResult.redirectsTo?.startsWith('https://')) {
-      httpToHttps = true;
-    }
-  }
-
-  if (!httpToHttps && httpResult.accessible && httpResult.status === 200) {
-    issues.push({
-      code: 'CANONICAL_NO_HTTPS_REDIRECT',
-      severity: 'warning',
-      category: 'security',
-      title: 'HTTP does not redirect to HTTPS',
-      description: 'The HTTP version of the site is accessible without redirecting to HTTPS.',
-      impact: 'Duplicate content issues and security concerns. Users may access insecure version.',
-      howToFix: 'Configure a 301 redirect from HTTP to HTTPS at the server or CDN level.',
-      affectedUrls: [httpUrl],
-    });
-  }
-
-  return {
-    issues,
-    data: {
-      primaryDomain: hostname,
-      wwwVersion: {
-        url: wwwUrl,
-        status: wwwResult.status,
-        redirectsTo: wwwResult.redirectsTo,
-        accessible: wwwResult.accessible,
-      },
-      nonWwwVersion: {
-        url: nonWwwUrl,
-        status: nonWwwResult.status,
-        redirectsTo: nonWwwResult.redirectsTo,
-        accessible: nonWwwResult.accessible,
-      },
-      hasProperRedirect,
-      preferredVersion,
-      httpsRedirect: {
-        httpToHttps,
-        httpsStatus: httpResult.status,
-      },
-    },
-  };
-}
-
-async function testUrl(url: string): Promise<{
-  status: number | null;
-  redirectsTo?: string;
-  accessible: boolean;
-}> {
-  try {
-    const response = await httpGet<string>(url, {
-      timeout: 10000,
-      maxRedirects: 0, // Don't follow redirects
-      validateStatus: () => true,
-    });
-
-    const status = response.status;
-    const location = response.headers.location || response.headers.Location;
-
-    return {
-      status,
-      redirectsTo: location,
-      accessible: status !== null && status < 500,
-    };
-  } catch (error) {
-    // DNS resolution failure, connection refused, etc.
-    return {
-      status: null,
-      accessible: false,
-    };
-  }
-}