@rankcli/agent-runtime 0.0.13 → 0.0.15

package/src/audit/engine.ts

@@ -56,6 +56,7 @@ import { analyzeFreshnessSignals } from './checks/freshness-signals.js';
 // AI Search Optimization checks (2026 - ChatGPT, AI Overviews)
 import { analyzeBingOptimization } from './checks/bing-optimization.js';
 import { analyzeAIContentStructure } from './checks/ai-content-structure.js';
+import { analyzeRAGChunkReadiness } from './checks/rag-chunk-readiness.js';
 import { analyzeCitationQuality } from './checks/citation-quality.js';
 import { analyzeAnswerConciseness } from './checks/answer-conciseness.js';
 // AI SEO Skills checks (brand narrative, citation worthiness, review ecosystem)
@@ -109,10 +110,13 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
   const parsedUrl = new URL(url);
   const domain = parsedUrl.hostname;
 
-  console.log(`\n🔍 Running comprehensive SEO audit on ${url}...\n`);
+  // Progress messages go to stderr, not stdout: stdout is reserved for the
+  // audit's actual JSON output (`rankcli audit --output json > file.json`),
+  // and printing progress via console.log here corrupts that redirect.
+  console.error(`\n🔍 Running comprehensive SEO audit on ${url}...\n`);
 
   // ========== PHASE 1: CRAWLABILITY + FETCH (PARALLEL) ==========
-  console.log('📋 Phase 1: Crawlability checks + page fetch (parallel)...');
+  console.error('📋 Phase 1: Crawlability checks + page fetch (parallel)...');
 
   const [crawlabilityResult, fetchResult] = await Promise.all([
     runCrawlabilityChecks(url).catch(err => {
@@ -123,6 +127,7 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
       timeout: 30000,
       validateStatus: () => true,
     }).catch(err => {
+      console.error('Main page fetch failed:', err instanceof Error ? { message: err.message, cause: (err as NodeJS.ErrnoException).cause } : err);
       return { error: err, data: '', headers: {} as Record<string, string> };
     }),
   ]);
@@ -131,15 +136,21 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
 
   // Check if fetch failed
   if ('error' in fetchResult) {
+    const err = fetchResult.error;
+    const errorMsg = err instanceof Error ? err.message : 'Unknown error';
+    const errorCause = err instanceof Error && (err as NodeJS.ErrnoException).cause 
+      ? ` (${String((err as NodeJS.ErrnoException).cause)})` 
+      : '';
     allIssues.push({
       code: 'FETCH_ERROR',
       severity: 'error',
-      category: 'crawlability',
-      title: 'Failed to fetch page',
-      description: `Could not fetch the page: ${fetchResult.error instanceof Error ? fetchResult.error.message : 'Unknown error'}`,
-      impact: 'Cannot perform full audit without page content.',
-      howToFix: 'Ensure the URL is accessible and the server is responding.',
+      category: 'indexability', // Not crawlability - robots/sitemap checks may have succeeded
+      title: 'Page fetch failed',
+      description: `Could not load page content: ${errorMsg}${errorCause}. Only robots.txt and sitemap checks were performed.`,
+      impact: 'Cannot perform full SEO audit without page HTML content.',
+      howToFix: 'Verify the URL is accessible, the server is responding, and there are no firewall/geo blocks.',
       affectedUrls: [url],
+      details: { error: errorMsg, cause: errorCause || undefined },
     });
     return createReport(url, domain, allIssues, pages);
   }
@@ -148,7 +159,7 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
   const headers = fetchResult.headers;
 
   // ========== PHASE 2: SYNCHRONOUS HTML CHECKS (PARALLEL) ==========
-  console.log(`📝 Phase 2: Running synchronous HTML checks (tier: ${tier}, limit: ${checksLimit})...`);
+  console.error(`📝 Phase 2: Running synchronous HTML checks (tier: ${tier}, limit: ${checksLimit})...`);
 
   // ===== CORE CHECKS (all tiers) =====
   const onPageResult = analyzeOnPage(html, url);
@@ -199,6 +210,7 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
   const qdfFreshnessResult = runPremiumChecks ? analyzeFreshnessSignals(html, url) : { issues: [], data: {} };
   // AI Search Optimization (2026) - Premium only
   const aiContentStructureResult = runPremiumChecks ? analyzeAIContentStructure(html, url) : { issues: [], data: {} };
+  const ragChunkReadinessResult = runPremiumChecks ? analyzeRAGChunkReadiness(html, url) : { issues: [], data: {} };
   const citationQualityResult = runPremiumChecks ? analyzeCitationQuality(html, url) : { issues: [], data: {} };
   const answerConcisenessResult = runPremiumChecks ? analyzeAnswerConciseness(html, url) : { issues: [], data: {} };
   // AI SEO Skills - Premium only
@@ -240,6 +252,7 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
     ...entityResult.issues,
     ...qdfFreshnessResult.issues,
     ...aiContentStructureResult.issues,
+    ...ragChunkReadinessResult.issues,
     ...citationQualityResult.issues,
     ...answerConcisenessResult.issues,
     ...brandMentionResult.issues,
@@ -267,7 +280,7 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
   }
 
   // ========== PHASE 3: ASYNC CHECKS (PARALLEL) ==========
-  console.log('🔗 Phase 3: Running async checks (parallel)...');
+  console.error('🔗 Phase 3: Running async checks (parallel)...');
 
   // Helper to safely run async checks with error handling and timeout
   const safeAsync = async <T>(
@@ -411,7 +424,7 @@ export async function runFullAudit(options: AuditOptions): Promise<AuditReport>
     issues: allIssues.map(i => i.code),
   });
 
-  console.log('\n✅ Audit complete!\n');
+  console.error('\n✅ Audit complete!\n');
 
   return createReport(url, domain, allIssues, pages);
 }

package/src/audit/types.ts

@@ -994,6 +994,30 @@ export const ISSUE_DEFINITIONS: Record<string, IssueDefinition> = {
     impact: 'Your content will not be used for Bard/Gemini AI training (regular search unaffected).',
     howToFix: 'Remove "User-agent: Google-Extended Disallow: /" if you want Google AI visibility.',
   },
+  CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS: {
+    code: 'CLOUDFLARE_AI_CRAWLER_GATE_AMBIGUOUS',
+    severity: 'warning',
+    category: 'ai-readiness',
+    title: 'No explicit AI crawler rules on a Cloudflare-fronted site',
+    description:
+      'This site appears to be served through Cloudflare but robots.txt has no explicit Allow/Disallow rules for AI crawlers. Cloudflare blocks "mixed-use" AI crawlers by default on ad-hosting zones starting September 15, 2026, and is rolling out Pay Per Crawl / Pay Per Use gating beyond that.',
+    impact:
+      'Without an explicit rule, whether AI crawlers (and future citation opportunities in ChatGPT, Claude, Perplexity, etc.) can reach this site now depends on Cloudflare account-level bot-management defaults, not on this codebase — a silent, invisible-to-git failure mode.',
+    howToFix:
+      'Add explicit User-agent rules for GPTBot, Claude-Web/ClaudeBot, PerplexityBot, and Google-Extended in robots.txt, and confirm the matching allow/block posture in the Cloudflare dashboard under Bot Management → AI Crawl Control. If you\'d rather charge than block, Cloudflare\'s Pay Per Crawl (AI Crawl Control → Payments tab) lets you set a per-crawl price instead of a flat allow/deny — worth a look if you get meaningful AI-crawler traffic.',
+  },
+  NO_AGENT_EXPERIENCE_SURFACE: {
+    code: 'NO_AGENT_EXPERIENCE_SURFACE',
+    severity: 'notice',
+    category: 'ai-readiness',
+    title: 'No agent-facing discovery surface found',
+    description:
+      'None of the emerging AI-agent discovery conventions were found: llms-full.txt, SKILL.md, a discoverable MCP server, or an OpenAPI spec. This is a genuinely new, low-adoption category as of 2026 — most sites have none of these yet — so this is an opportunity, not a compliance failure.',
+    impact:
+      'AI agents (not just chatbots answering questions, but agents that browse and act on a user\'s behalf) increasingly prefer sites that expose machine-readable capabilities over ones that require scraping and guessing at HTML structure. Being an early, discoverable site in this space is a low-competition differentiator right now.',
+    howToFix:
+      'Start with whichever fits your site: llms-full.txt (a single Markdown dump of your key content — measured as fetched roughly 2x more than plain llms.txt), a SKILL.md capability manifest, or an OpenAPI spec at /openapi.json if you already have an API. A discoverable MCP server is the highest-effort, highest-payoff option if your product has one to expose.',
+  },
   HIGH_JS_RENDERING_RATIO: {
     code: 'HIGH_JS_RENDERING_RATIO',
     severity: 'warning',

package/src/index.ts

@@ -62,6 +62,9 @@ export * from './reports/index.js';
 // GEO (Generative Engine Optimization) tracking
 export * from './geo/index.js';
 
+// Keyword Rank Tracking
+export * from './ranking/index.js';
+
 // Advanced Analyzers (GEO, CWV, Security, Schema, Images, Links, Mobile)
 export * as analyzers from './analyzers/index.js';
 

package/src/ranking/index.ts

@@ -0,0 +1,5 @@
+// Ranking Module - Keyword position tracking and SERP analysis
+
+export * from './types.js';
+export * from './serp-client.js';
+export * from './tracker.js';

package/src/ranking/serp-client.ts

@@ -0,0 +1,348 @@
+// SERP Client - Check keyword rankings via API or scraping
+
+import type { RankCheckOptions, RankCheckResult, SerpFeature, CompetitorUrl } from './types.js';
+
+export interface SerpClientConfig {
+  provider: 'valueserp' | 'serpapi' | 'direct';
+  apiKey?: string;
+  proxyUrl?: string;
+}
+
+export class SerpClient {
+  private config: SerpClientConfig;
+
+  constructor(config: SerpClientConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Check ranking for a single keyword
+   */
+  async checkRank(options: RankCheckOptions): Promise<RankCheckResult[]> {
+    const results: RankCheckResult[] = [];
+
+    for (const keyword of options.keywords) {
+      try {
+        const result = await this.checkSingleKeyword({
+          ...options,
+          keyword,
+        });
+        results.push(result);
+      } catch (error) {
+        // Log error but continue with other keywords
+        console.error(`Error checking rank for "${keyword}":`, error);
+        results.push({
+          keyword,
+          position: null,
+          url: null,
+          serpFeatures: [],
+          topResults: [],
+          checkedAt: new Date(),
+        });
+      }
+    }
+
+    return results;
+  }
+
+  private async checkSingleKeyword(options: RankCheckOptions & { keyword: string }): Promise<RankCheckResult> {
+    switch (this.config.provider) {
+      case 'valueserp':
+        return this.checkViaValueSerp(options);
+      case 'serpapi':
+        return this.checkViaSerpApi(options);
+      case 'direct':
+      default:
+        return this.checkViaDirect(options);
+    }
+  }
+
+  /**
+   * ValueSERP API implementation
+   * Docs: https://www.valueserp.com/docs
+   */
+  private async checkViaValueSerp(options: RankCheckOptions & { keyword: string }): Promise<RankCheckResult> {
+    if (!this.config.apiKey) {
+      throw new Error('ValueSERP API key required');
+    }
+
+    const params = new URLSearchParams({
+      api_key: this.config.apiKey,
+      q: options.keyword,
+      location: options.country || 'United States',
+      google_domain: options.country === 'US' ? 'google.com' : `google.${options.country?.toLowerCase() || 'com'}`,
+      gl: options.country || 'us',
+      hl: options.language || 'en',
+      device: options.device || 'desktop',
+      num: '100', // Get top 100 results
+    });
+
+    const response = await fetch(`https://api.valueserp.com/search?${params}`);
+    
+    if (!response.ok) {
+      throw new Error(`ValueSERP API error: ${response.status}`);
+    }
+
+    const data = await response.json() as ValueSerpResponse;
+    
+    return this.parseValueSerpResponse(data, options.domain, options.keyword);
+  }
+
+  /**
+   * SerpAPI implementation (alternative)
+   * Docs: https://serpapi.com/search-api
+   */
+  private async checkViaSerpApi(options: RankCheckOptions & { keyword: string }): Promise<RankCheckResult> {
+    if (!this.config.apiKey) {
+      throw new Error('SerpAPI key required');
+    }
+
+    const params = new URLSearchParams({
+      api_key: this.config.apiKey,
+      q: options.keyword,
+      location: options.country === 'US' ? 'United States' : options.country || 'United States',
+      gl: options.country?.toLowerCase() || 'us',
+      hl: options.language || 'en',
+      device: options.device || 'desktop',
+      num: '100',
+    });
+
+    const response = await fetch(`https://serpapi.com/search?${params}`);
+    
+    if (!response.ok) {
+      throw new Error(`SerpAPI error: ${response.status}`);
+    }
+
+    const data = await response.json() as SerpApiResponse;
+    
+    return this.parseSerpApiResponse(data, options.domain, options.keyword);
+  }
+
+  /**
+   * Direct scraping fallback (rate-limited, use with caution)
+   */
+  private async checkViaDirect(options: RankCheckOptions & { keyword: string }): Promise<RankCheckResult> {
+    const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(options.keyword)}&num=100`;
+    
+    const headers: Record<string, string> = {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'Accept-Language': options.language || 'en-US,en;q=0.9',
+    };
+
+    const response = await fetch(searchUrl, { headers });
+    
+    if (!response.ok) {
+      throw new Error(`Direct search failed: ${response.status}`);
+    }
+
+    const html = await response.text();
+    return this.parseDirectSearchResults(html, options.domain, options.keyword);
+  }
+
+  private parseValueSerpResponse(data: ValueSerpResponse, domain: string, keyword: string): RankCheckResult {
+    const topResults: CompetitorUrl[] = [];
+    const serpFeatures: SerpFeature[] = [];
+    let position: number | null = null;
+    let url: string | null = null;
+
+    // Parse organic results
+    if (data.organic_results) {
+      for (let i = 0; i < data.organic_results.length; i++) {
+        const result = data.organic_results[i];
+        const resultDomain = this.extractDomain(result.link);
+        
+        topResults.push({
+          position: i + 1,
+          url: result.link,
+          domain: resultDomain,
+          title: result.title,
+        });
+
+        // Check if this is our domain
+        if (this.domainMatches(resultDomain, domain) && position === null) {
+          position = i + 1;
+          url = result.link;
+        }
+      }
+    }
+
+    // Parse SERP features
+    if (data.answer_box) {
+      serpFeatures.push({
+        type: 'featured_snippet',
+        position: 0,
+        hasOwnSite: this.domainMatches(this.extractDomain(data.answer_box.link || ''), domain),
+      });
+    }
+
+    if (data.people_also_ask) {
+      serpFeatures.push({
+        type: 'people_also_ask',
+        hasOwnSite: data.people_also_ask.some(
+          (paa: { link?: string }) => this.domainMatches(this.extractDomain(paa.link || ''), domain)
+        ),
+      });
+    }
+
+    if (data.local_results) {
+      serpFeatures.push({ type: 'local_pack' });
+    }
+
+    if (data.knowledge_graph) {
+      serpFeatures.push({ type: 'knowledge_panel' });
+    }
+
+    return {
+      keyword,
+      position,
+      url,
+      serpFeatures,
+      topResults: topResults.slice(0, 10),
+      checkedAt: new Date(),
+    };
+  }
+
+  private parseSerpApiResponse(data: SerpApiResponse, domain: string, keyword: string): RankCheckResult {
+    const topResults: CompetitorUrl[] = [];
+    const serpFeatures: SerpFeature[] = [];
+    let position: number | null = null;
+    let url: string | null = null;
+
+    // Parse organic results
+    if (data.organic_results) {
+      for (let i = 0; i < data.organic_results.length; i++) {
+        const result = data.organic_results[i];
+        const resultDomain = this.extractDomain(result.link);
+        
+        topResults.push({
+          position: result.position || i + 1,
+          url: result.link,
+          domain: resultDomain,
+          title: result.title,
+        });
+
+        if (this.domainMatches(resultDomain, domain) && position === null) {
+          position = result.position || i + 1;
+          url = result.link;
+        }
+      }
+    }
+
+    // Parse SERP features
+    if (data.answer_box) {
+      serpFeatures.push({
+        type: 'featured_snippet',
+        position: 0,
+      });
+    }
+
+    if (data.related_questions) {
+      serpFeatures.push({ type: 'people_also_ask' });
+    }
+
+    return {
+      keyword,
+      position,
+      url,
+      serpFeatures,
+      topResults: topResults.slice(0, 10),
+      checkedAt: new Date(),
+    };
+  }
+
+  private parseDirectSearchResults(html: string, domain: string, keyword: string): RankCheckResult {
+    const topResults: CompetitorUrl[] = [];
+    let position: number | null = null;
+    let url: string | null = null;
+
+    // Simple regex-based parsing (not perfect but works for basic cases)
+    const linkRegex = /<a[^>]+href="\/url\?q=([^"&]+)/g;
+    let match;
+    let index = 0;
+
+    while ((match = linkRegex.exec(html)) !== null && index < 100) {
+      try {
+        const decodedUrl = decodeURIComponent(match[1]);
+        const resultDomain = this.extractDomain(decodedUrl);
+        
+        // Skip Google's own URLs
+        if (resultDomain.includes('google.com') || resultDomain.includes('gstatic.com')) {
+          continue;
+        }
+
+        index++;
+        topResults.push({
+          position: index,
+          url: decodedUrl,
+          domain: resultDomain,
+        });
+
+        if (this.domainMatches(resultDomain, domain) && position === null) {
+          position = index;
+          url = decodedUrl;
+        }
+      } catch {
+        // Skip malformed URLs
+      }
+    }
+
+    return {
+      keyword,
+      position,
+      url,
+      serpFeatures: [], // Direct scraping doesn't easily extract SERP features
+      topResults: topResults.slice(0, 10),
+      checkedAt: new Date(),
+    };
+  }
+
+  private extractDomain(url: string): string {
+    try {
+      const parsed = new URL(url);
+      return parsed.hostname.replace(/^www\./, '');
+    } catch {
+      return '';
+    }
+  }
+
+  private domainMatches(resultDomain: string, targetDomain: string): boolean {
+    const normalizedResult = resultDomain.toLowerCase().replace(/^www\./, '');
+    const normalizedTarget = targetDomain.toLowerCase().replace(/^www\./, '');
+    
+    // Exact match or subdomain match
+    return normalizedResult === normalizedTarget || 
+           normalizedResult.endsWith(`.${normalizedTarget}`);
+  }
+}
+
+// ValueSERP response types
+interface ValueSerpResponse {
+  organic_results?: Array<{
+    link: string;
+    title: string;
+    snippet?: string;
+  }>;
+  answer_box?: {
+    link?: string;
+    title?: string;
+  };
+  people_also_ask?: Array<{
+    question: string;
+    link?: string;
+  }>;
+  local_results?: Array<unknown>;
+  knowledge_graph?: unknown;
+}
+
+// SerpAPI response types
+interface SerpApiResponse {
+  organic_results?: Array<{
+    position: number;
+    link: string;
+    title: string;
+    snippet?: string;
+  }>;
+  answer_box?: unknown;
+  related_questions?: Array<unknown>;
+}