@rankcli/agent-runtime 0.0.12 → 0.0.14

package/src/audit/checks/client-rendering.ts

@@ -312,17 +312,17 @@ function generateRecommendations(
 
     if (framework === 'React') {
       recommendations.push(
-        'Quick fix: Add react-snap to pre-render pages at build time (npm install -D react-snap, add "postbuild": "react-snap" to scripts)'
+        'Quick fix: Use Vike (vike.dev) for SSR/SSG - works with Vite, minimal config needed'
       );
       recommendations.push(
-        'Alternative: Use Vike (vite-plugin-ssr) for SSR/SSG without changing frameworks'
+        'Alternative: Migrate to Next.js or Remix for built-in SSR/SSG support'
       );
     } else if (framework === 'Vue') {
       recommendations.push(
-        'Quick fix: Add prerender-spa-plugin to pre-render pages at build time'
+        'Quick fix: Use Vike (vike.dev) for SSR/SSG - works with Vite, minimal config needed'
       );
       recommendations.push(
-        'Alternative: Use Vike (vite-plugin-ssr) for SSR/SSG without changing frameworks'
+        'Alternative: Migrate to Nuxt for built-in SSR/SSG support'
       );
     } else if (framework === 'Angular Universal') {
       recommendations.push(
@@ -330,7 +330,7 @@ function generateRecommendations(
       );
     } else {
       recommendations.push(
-        'Quick fix: Use a pre-rendering tool like react-snap or prerender-spa-plugin'
+        'Quick fix: Use Vike (vike.dev) for SSR/SSG with any Vite-based framework'
       );
     }
 
@@ -368,10 +368,10 @@ export function analyzeClientRendering(
   // Critical: Client-side only rendering
   if (analysis.renderingMethod === 'csr' && analysis.confidence !== 'low') {
     const howToFixByFramework = analysis.frameworkDetected === 'React'
-      ? 'Add react-snap to pre-render pages: npm install -D react-snap, then add "postbuild": "react-snap" to package.json scripts. No code changes needed.'
+      ? 'Use Vike (vike.dev) to add SSR/SSG to your Vite React app: npm install vike vike-react, then follow the setup guide. Alternatively, migrate to Next.js or Remix.'
       : analysis.frameworkDetected === 'Vue'
-      ? 'Add prerender-spa-plugin to pre-render pages at build time. Alternatively, use Vike for SSR/SSG.'
-      : 'Pre-render your pages using react-snap, prerender-spa-plugin, or similar build-time tools.';
+      ? 'Use Vike (vike.dev) to add SSR/SSG to your Vite Vue app: npm install vike vike-vue, then follow the setup guide. Alternatively, migrate to Nuxt.'
+      : 'Use Vike (vike.dev) for SSR/SSG with Vite-based frameworks, or use a framework with built-in SSR support (Next.js, Nuxt, Remix, SvelteKit).';
 
     issues.push({
       code: 'CLIENT_SIDE_RENDERING',
@@ -431,8 +431,8 @@ export function analyzeClientRendering(
         'Single Page Applications without SSR have slower time-to-content for search crawlers.',
       howToFix:
         analysis.frameworkDetected === 'React'
-          ? 'Quick fix: Add react-snap (npm install -D react-snap) to pre-render at build time. Alternative: Use Vike for SSR/SSG.'
-          : 'Quick fix: Add prerender-spa-plugin to pre-render at build time. Alternative: Use Vike for SSR/SSG.',
+          ? 'Add SSR/SSG using Vike (vike.dev): npm install vike vike-react. Alternative: migrate to Next.js or Remix.'
+          : 'Add SSR/SSG using Vike (vike.dev): npm install vike vike-vue. Alternative: migrate to Nuxt.',
       affectedUrls: [url],
       details: {
         framework: analysis.frameworkDetected,

package/src/audit/checks/security-headers.ts

@@ -59,7 +59,15 @@ export async function analyzeSecurityHeaders(url: string): Promise<{ issues: Aud
     });
 
     const headers = response.headers;
-    const isHttps = url.startsWith('https://');
+    // Use URL parsing for robust protocol detection (handles case variations like Https://)
+    let isHttps = false;
+    try {
+      const parsedUrl = new URL(url);
+      isHttps = parsedUrl.protocol === 'https:';
+    } catch {
+      // If URL parsing fails, fall back to startsWith check
+      isHttps = url.toLowerCase().startsWith('https://');
+    }
 
     // Extract security headers (case-insensitive)
     const getHeader = (name: string): string | null => {
@@ -151,10 +159,18 @@ export async function analyzeSecurityHeaders(url: string): Promise<{ issues: Aud
       },
     };
   } catch (error) {
+    // Use URL parsing for robust protocol detection in error case too
+    let isHttps = false;
+    try {
+      const parsedUrl = new URL(url);
+      isHttps = parsedUrl.protocol === 'https:';
+    } catch {
+      isHttps = url.toLowerCase().startsWith('https://');
+    }
     return {
       issues,
       data: {
-        https: url.startsWith('https://'),
+        https: isHttps,
         headers: {
           hsts: null,
           csp: null,

package/src/audit/engine.ts

@@ -123,6 +123,7 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
       timeout: 30000,
       validateStatus: () => true,
     }).catch(err => {
+      console.error('Main page fetch failed:', err instanceof Error ? { message: err.message, cause: (err as NodeJS.ErrnoException).cause } : err);
       return { error: err, data: '', headers: {} as Record<string, string> };
     }),
   ]);
@@ -131,15 +132,21 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
 
   // Check if fetch failed
   if ('error' in fetchResult) {
+    const err = fetchResult.error;
+    const errorMsg = err instanceof Error ? err.message : 'Unknown error';
+    const errorCause = err instanceof Error && (err as NodeJS.ErrnoException).cause 
+      ? ` (${String((err as NodeJS.ErrnoException).cause)})` 
+      : '';
     allIssues.push({
       code: 'FETCH_ERROR',
       severity: 'error',
-      category: 'crawlability',
-      title: 'Failed to fetch page',
-      description: `Could not fetch the page: ${fetchResult.error instanceof Error ? fetchResult.error.message : 'Unknown error'}`,
-      impact: 'Cannot perform full audit without page content.',
-      howToFix: 'Ensure the URL is accessible and the server is responding.',
+      category: 'indexability', // Not crawlability - robots/sitemap checks may have succeeded
+      title: 'Page fetch failed',
+      description: `Could not load page content: ${errorMsg}${errorCause}. Only robots.txt and sitemap checks were performed.`,
+      impact: 'Cannot perform full SEO audit without page HTML content.',
+      howToFix: 'Verify the URL is accessible, the server is responding, and there are no firewall/geo blocks.',
       affectedUrls: [url],
+      details: { error: errorMsg, cause: errorCause || undefined },
     });
     return createReport(url, domain, allIssues, pages);
   }

package/src/index.ts

@@ -62,6 +62,9 @@ export * from './reports/index.js';
 // GEO (Generative Engine Optimization) tracking
 export * from './geo/index.js';
 
+// Keyword Rank Tracking
+export * from './ranking/index.js';
+
 // Advanced Analyzers (GEO, CWV, Security, Schema, Images, Links, Mobile)
 export * as analyzers from './analyzers/index.js';
 

package/src/ranking/index.ts

@@ -0,0 +1,5 @@
+// Ranking Module - Keyword position tracking and SERP analysis
+
+export * from './types.js';
+export * from './serp-client.js';
+export * from './tracker.js';

package/src/ranking/serp-client.ts

@@ -0,0 +1,348 @@
+// SERP Client - Check keyword rankings via API or scraping
+
+import type { RankCheckOptions, RankCheckResult, SerpFeature, CompetitorUrl } from './types.js';
+
+export interface SerpClientConfig {
+  provider: 'valueserp' | 'serpapi' | 'direct';
+  apiKey?: string;
+  proxyUrl?: string;
+}
+
+export class SerpClient {
+  private config: SerpClientConfig;
+
+  constructor(config: SerpClientConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Check ranking for a single keyword
+   */
+  async checkRank(options: RankCheckOptions): Promise<RankCheckResult[]> {
+    const results: RankCheckResult[] = [];
+
+    for (const keyword of options.keywords) {
+      try {
+        const result = await this.checkSingleKeyword({
+          ...options,
+          keyword,
+        });
+        results.push(result);
+      } catch (error) {
+        // Log error but continue with other keywords
+        console.error(`Error checking rank for "${keyword}":`, error);
+        results.push({
+          keyword,
+          position: null,
+          url: null,
+          serpFeatures: [],
+          topResults: [],
+          checkedAt: new Date(),
+        });
+      }
+    }
+
+    return results;
+  }
+
+  private async checkSingleKeyword(options: RankCheckOptions & { keyword: string }): Promise<RankCheckResult> {
+    switch (this.config.provider) {
+      case 'valueserp':
+        return this.checkViaValueSerp(options);
+      case 'serpapi':
+        return this.checkViaSerpApi(options);
+      case 'direct':
+      default:
+        return this.checkViaDirect(options);
+    }
+  }
+
+  /**
+   * ValueSERP API implementation
+   * Docs: https://www.valueserp.com/docs
+   */
+  private async checkViaValueSerp(options: RankCheckOptions & { keyword: string }): Promise<RankCheckResult> {
+    if (!this.config.apiKey) {
+      throw new Error('ValueSERP API key required');
+    }
+
+    const params = new URLSearchParams({
+      api_key: this.config.apiKey,
+      q: options.keyword,
+      location: options.country || 'United States',
+      google_domain: options.country === 'US' ? 'google.com' : `google.${options.country?.toLowerCase() || 'com'}`,
+      gl: options.country || 'us',
+      hl: options.language || 'en',
+      device: options.device || 'desktop',
+      num: '100', // Get top 100 results
+    });
+
+    const response = await fetch(`https://api.valueserp.com/search?${params}`);
+    
+    if (!response.ok) {
+      throw new Error(`ValueSERP API error: ${response.status}`);
+    }
+
+    const data = await response.json() as ValueSerpResponse;
+    
+    return this.parseValueSerpResponse(data, options.domain, options.keyword);
+  }
+
+  /**
+   * SerpAPI implementation (alternative)
+   * Docs: https://serpapi.com/search-api
+   */
+  private async checkViaSerpApi(options: RankCheckOptions & { keyword: string }): Promise<RankCheckResult> {
+    if (!this.config.apiKey) {
+      throw new Error('SerpAPI key required');
+    }
+
+    const params = new URLSearchParams({
+      api_key: this.config.apiKey,
+      q: options.keyword,
+      location: options.country === 'US' ? 'United States' : options.country || 'United States',
+      gl: options.country?.toLowerCase() || 'us',
+      hl: options.language || 'en',
+      device: options.device || 'desktop',
+      num: '100',
+    });
+
+    const response = await fetch(`https://serpapi.com/search?${params}`);
+    
+    if (!response.ok) {
+      throw new Error(`SerpAPI error: ${response.status}`);
+    }
+
+    const data = await response.json() as SerpApiResponse;
+    
+    return this.parseSerpApiResponse(data, options.domain, options.keyword);
+  }
+
+  /**
+   * Direct scraping fallback (rate-limited, use with caution)
+   */
+  private async checkViaDirect(options: RankCheckOptions & { keyword: string }): Promise<RankCheckResult> {
+    const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(options.keyword)}&num=100`;
+    
+    const headers: Record<string, string> = {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'Accept-Language': options.language || 'en-US,en;q=0.9',
+    };
+
+    const response = await fetch(searchUrl, { headers });
+    
+    if (!response.ok) {
+      throw new Error(`Direct search failed: ${response.status}`);
+    }
+
+    const html = await response.text();
+    return this.parseDirectSearchResults(html, options.domain, options.keyword);
+  }
+
+  private parseValueSerpResponse(data: ValueSerpResponse, domain: string, keyword: string): RankCheckResult {
+    const topResults: CompetitorUrl[] = [];
+    const serpFeatures: SerpFeature[] = [];
+    let position: number | null = null;
+    let url: string | null = null;
+
+    // Parse organic results
+    if (data.organic_results) {
+      for (let i = 0; i < data.organic_results.length; i++) {
+        const result = data.organic_results[i];
+        const resultDomain = this.extractDomain(result.link);
+        
+        topResults.push({
+          position: i + 1,
+          url: result.link,
+          domain: resultDomain,
+          title: result.title,
+        });
+
+        // Check if this is our domain
+        if (this.domainMatches(resultDomain, domain) && position === null) {
+          position = i + 1;
+          url = result.link;
+        }
+      }
+    }
+
+    // Parse SERP features
+    if (data.answer_box) {
+      serpFeatures.push({
+        type: 'featured_snippet',
+        position: 0,
+        hasOwnSite: this.domainMatches(this.extractDomain(data.answer_box.link || ''), domain),
+      });
+    }
+
+    if (data.people_also_ask) {
+      serpFeatures.push({
+        type: 'people_also_ask',
+        hasOwnSite: data.people_also_ask.some(
+          (paa: { link?: string }) => this.domainMatches(this.extractDomain(paa.link || ''), domain)
+        ),
+      });
+    }
+
+    if (data.local_results) {
+      serpFeatures.push({ type: 'local_pack' });
+    }
+
+    if (data.knowledge_graph) {
+      serpFeatures.push({ type: 'knowledge_panel' });
+    }
+
+    return {
+      keyword,
+      position,
+      url,
+      serpFeatures,
+      topResults: topResults.slice(0, 10),
+      checkedAt: new Date(),
+    };
+  }
+
+  private parseSerpApiResponse(data: SerpApiResponse, domain: string, keyword: string): RankCheckResult {
+    const topResults: CompetitorUrl[] = [];
+    const serpFeatures: SerpFeature[] = [];
+    let position: number | null = null;
+    let url: string | null = null;
+
+    // Parse organic results
+    if (data.organic_results) {
+      for (let i = 0; i < data.organic_results.length; i++) {
+        const result = data.organic_results[i];
+        const resultDomain = this.extractDomain(result.link);
+        
+        topResults.push({
+          position: result.position || i + 1,
+          url: result.link,
+          domain: resultDomain,
+          title: result.title,
+        });
+
+        if (this.domainMatches(resultDomain, domain) && position === null) {
+          position = result.position || i + 1;
+          url = result.link;
+        }
+      }
+    }
+
+    // Parse SERP features
+    if (data.answer_box) {
+      serpFeatures.push({
+        type: 'featured_snippet',
+        position: 0,
+      });
+    }
+
+    if (data.related_questions) {
+      serpFeatures.push({ type: 'people_also_ask' });
+    }
+
+    return {
+      keyword,
+      position,
+      url,
+      serpFeatures,
+      topResults: topResults.slice(0, 10),
+      checkedAt: new Date(),
+    };
+  }
+
+  private parseDirectSearchResults(html: string, domain: string, keyword: string): RankCheckResult {
+    const topResults: CompetitorUrl[] = [];
+    let position: number | null = null;
+    let url: string | null = null;
+
+    // Simple regex-based parsing (not perfect but works for basic cases)
+    const linkRegex = /<a[^>]+href="\/url\?q=([^"&]+)/g;
+    let match;
+    let index = 0;
+
+    while ((match = linkRegex.exec(html)) !== null && index < 100) {
+      try {
+        const decodedUrl = decodeURIComponent(match[1]);
+        const resultDomain = this.extractDomain(decodedUrl);
+        
+        // Skip Google's own URLs
+        if (resultDomain.includes('google.com') || resultDomain.includes('gstatic.com')) {
+          continue;
+        }
+
+        index++;
+        topResults.push({
+          position: index,
+          url: decodedUrl,
+          domain: resultDomain,
+        });
+
+        if (this.domainMatches(resultDomain, domain) && position === null) {
+          position = index;
+          url = decodedUrl;
+        }
+      } catch {
+        // Skip malformed URLs
+      }
+    }
+
+    return {
+      keyword,
+      position,
+      url,
+      serpFeatures: [], // Direct scraping doesn't easily extract SERP features
+      topResults: topResults.slice(0, 10),
+      checkedAt: new Date(),
+    };
+  }
+
+  private extractDomain(url: string): string {
+    try {
+      const parsed = new URL(url);
+      return parsed.hostname.replace(/^www\./, '');
+    } catch {
+      return '';
+    }
+  }
+
+  private domainMatches(resultDomain: string, targetDomain: string): boolean {
+    const normalizedResult = resultDomain.toLowerCase().replace(/^www\./, '');
+    const normalizedTarget = targetDomain.toLowerCase().replace(/^www\./, '');
+    
+    // Exact match or subdomain match
+    return normalizedResult === normalizedTarget || 
+           normalizedResult.endsWith(`.${normalizedTarget}`);
+  }
+}
+
+// ValueSERP response types
+interface ValueSerpResponse {
+  organic_results?: Array<{
+    link: string;
+    title: string;
+    snippet?: string;
+  }>;
+  answer_box?: {
+    link?: string;
+    title?: string;
+  };
+  people_also_ask?: Array<{
+    question: string;
+    link?: string;
+  }>;
+  local_results?: Array<unknown>;
+  knowledge_graph?: unknown;
+}
+
+// SerpAPI response types
+interface SerpApiResponse {
+  organic_results?: Array<{
+    position: number;
+    link: string;
+    title: string;
+    snippet?: string;
+  }>;
+  answer_box?: unknown;
+  related_questions?: Array<unknown>;
+}