@rankcli/agent-runtime 0.0.1

package/src/keywords/sources/competitor-analysis.ts

@@ -0,0 +1,374 @@
+// Competitor Keyword Analysis (SpyFu Kombat-style)
+// Analyze competitor rankings to find keyword gaps and opportunities
+
+import { httpGet } from '../../utils/http.js';
+import * as cheerio from 'cheerio';
+import type { KeywordData } from '../types.js';
+
+const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
+
+export interface CompetitorKeywordResult {
+  // Keywords all competitors rank for (high priority - proven market)
+  coreKeywords: KeywordData[];
+  // Keywords competitors rank for but you don't (opportunities)
+  missingKeywords: KeywordData[];
+  // Keywords only you rank for (potential advantages)
+  uniqueKeywords: KeywordData[];
+  // All keywords any competitor ranks for
+  keywordUniverse: KeywordData[];
+  // Overlap analysis
+  competitorOverlap: CompetitorOverlap[];
+}
+
+export interface CompetitorOverlap {
+  competitor: string;
+  sharedKeywords: number;
+  uniqueToThem: number;
+  uniqueToYou: number;
+}
+
+export interface CompetitorData {
+  domain: string;
+  keywords: Set<string>;
+  rankings: Map<string, number>; // keyword -> position
+}
+
+/**
+ * Scrape competitor rankings from Google SERP for seed keywords
+ * This discovers what keywords competitors appear for
+ */
+export async function discoverCompetitorKeywords(
+  yourDomain: string,
+  seedKeywords: string[],
+  competitors?: string[]
+): Promise<CompetitorKeywordResult> {
+  const yourKeywords = new Set<string>();
+  const competitorKeywordsMap = new Map<string, Set<string>>();
+  const allKeywords = new Map<string, KeywordData>();
+
+  // Initialize competitor tracking
+  const discoveredCompetitors = new Set<string>(competitors || []);
+
+  console.log('  → Analyzing SERP results for competitor keywords...');
+
+  // For each seed keyword, check who ranks and for what related terms
+  for (const seed of seedKeywords) {
+    try {
+      // Get SERP results
+      const serpData = await analyzeSERP(seed);
+
+      // Track who ranks for this keyword
+      for (const result of serpData.results) {
+        const domain = extractDomain(result.url);
+
+        if (domain === yourDomain || domain.includes(yourDomain) || yourDomain.includes(domain)) {
+          yourKeywords.add(seed);
+        } else {
+          // Discovered competitor
+          discoveredCompetitors.add(domain);
+          if (!competitorKeywordsMap.has(domain)) {
+            competitorKeywordsMap.set(domain, new Set());
+          }
+          competitorKeywordsMap.get(domain)!.add(seed);
+        }
+
+        // Extract keywords from competitor titles/descriptions
+        const extractedKeywords = extractKeywordsFromText(result.title + ' ' + result.description);
+        for (const kw of extractedKeywords) {
+          if (!allKeywords.has(kw)) {
+            allKeywords.set(kw, {
+              keyword: kw,
+              searchVolume: 0,
+              keywordDifficulty: 0,
+              source: 'competitor' as const,
+            });
+          }
+
+          // Track which domains have this keyword
+          if (domain === yourDomain || domain.includes(yourDomain)) {
+            yourKeywords.add(kw);
+          } else {
+            competitorKeywordsMap.get(domain)?.add(kw);
+          }
+        }
+      }
+
+      // Also get related searches for more keyword ideas
+      for (const related of serpData.relatedSearches) {
+        if (!allKeywords.has(related)) {
+          allKeywords.set(related, {
+            keyword: related,
+            searchVolume: 0,
+            keywordDifficulty: 0,
+            source: 'competitor' as const,
+          });
+        }
+      }
+
+      // Avoid rate limiting
+      await sleep(500);
+    } catch (error) {
+      // Continue with other seeds
+    }
+  }
+
+  // Calculate keyword categories
+  const coreKeywords: KeywordData[] = [];
+  const missingKeywords: KeywordData[] = [];
+  const uniqueKeywords: KeywordData[] = [];
+  const keywordUniverse: KeywordData[] = [];
+
+  // Analyze each keyword
+  for (const [keyword, data] of allKeywords) {
+    const competitorCount = countCompetitorsWithKeyword(keyword, competitorKeywordsMap);
+    const youHaveIt = yourKeywords.has(keyword);
+
+    keywordUniverse.push(data);
+
+    if (competitorCount >= 2 && youHaveIt) {
+      // Core keyword - everyone ranks
+      coreKeywords.push(data);
+    } else if (competitorCount >= 1 && !youHaveIt) {
+      // Missing keyword - competitors have it, you don't
+      missingKeywords.push(data);
+    } else if (competitorCount === 0 && youHaveIt) {
+      // Unique keyword - only you have it
+      uniqueKeywords.push(data);
+    }
+  }
+
+  // Calculate competitor overlap
+  const competitorOverlap: CompetitorOverlap[] = [];
+  for (const [domain, keywords] of competitorKeywordsMap) {
+    const shared = [...keywords].filter(k => yourKeywords.has(k)).length;
+    const uniqueToThem = [...keywords].filter(k => !yourKeywords.has(k)).length;
+    const uniqueToYou = [...yourKeywords].filter(k => !keywords.has(k)).length;
+
+    competitorOverlap.push({
+      competitor: domain,
+      sharedKeywords: shared,
+      uniqueToThem,
+      uniqueToYou,
+    });
+  }
+
+  // Sort by number of keywords they have that you don't
+  competitorOverlap.sort((a, b) => b.uniqueToThem - a.uniqueToThem);
+
+  return {
+    coreKeywords,
+    missingKeywords,
+    uniqueKeywords,
+    keywordUniverse,
+    competitorOverlap,
+  };
+}
+
+interface SERPResult {
+  url: string;
+  title: string;
+  description: string;
+  position: number;
+}
+
+interface SERPData {
+  results: SERPResult[];
+  relatedSearches: string[];
+  peopleAlsoAsk: string[];
+}
+
+async function analyzeSERP(query: string): Promise<SERPData> {
+  try {
+    const response = await httpGet<string>('https://www.google.com/search', {
+      params: { q: query, num: 20 },
+      headers: { 'User-Agent': USER_AGENT },
+      timeout: 10000,
+    });
+
+    const $ = cheerio.load(response.data);
+    const results: SERPResult[] = [];
+    const relatedSearches: string[] = [];
+    const peopleAlsoAsk: string[] = [];
+
+    // Extract search results
+    let position = 1;
+    $('div.g').each((_, el) => {
+      const link = $(el).find('a').first();
+      const url = link.attr('href');
+      const title = $(el).find('h3').text().trim();
+      const description = $(el).find('.VwiC3b').text().trim() ||
+                         $(el).find('[data-content-feature]').text().trim();
+
+      if (url && url.startsWith('http') && title) {
+        results.push({ url, title, description, position });
+        position++;
+      }
+    });
+
+    // Extract related searches
+    $('div[data-ved] a').each((_, el) => {
+      const href = $(el).attr('href');
+      if (href?.includes('/search?q=')) {
+        const text = $(el).text().trim();
+        if (text.length > 3 && text.length < 100 && !text.includes('http')) {
+          relatedSearches.push(text);
+        }
+      }
+    });
+
+    // Extract People Also Ask
+    $('[data-sgrd]').each((_, el) => {
+      const text = $(el).text().trim();
+      if (text.endsWith('?') && text.length > 10) {
+        peopleAlsoAsk.push(text);
+      }
+    });
+
+    return {
+      results,
+      relatedSearches: [...new Set(relatedSearches)],
+      peopleAlsoAsk: [...new Set(peopleAlsoAsk)],
+    };
+  } catch {
+    return { results: [], relatedSearches: [], peopleAlsoAsk: [] };
+  }
+}
+
+function extractDomain(url: string): string {
+  try {
+    const parsed = new URL(url);
+    return parsed.hostname.replace(/^www\./, '');
+  } catch {
+    return '';
+  }
+}
+
+function extractKeywordsFromText(text: string): string[] {
+  // Extract 2-4 word phrases that could be keywords
+  const words = text.toLowerCase()
+    .replace(/[^\w\s]/g, ' ')
+    .split(/\s+/)
+    .filter(w => w.length > 2);
+
+  const keywords: string[] = [];
+
+  // Single words (4+ chars)
+  words.filter(w => w.length >= 4).forEach(w => keywords.push(w));
+
+  // 2-word phrases
+  for (let i = 0; i < words.length - 1; i++) {
+    const phrase = `${words[i]} ${words[i + 1]}`;
+    if (phrase.length >= 5 && phrase.length <= 50) {
+      keywords.push(phrase);
+    }
+  }
+
+  // 3-word phrases
+  for (let i = 0; i < words.length - 2; i++) {
+    const phrase = `${words[i]} ${words[i + 1]} ${words[i + 2]}`;
+    if (phrase.length >= 8 && phrase.length <= 60) {
+      keywords.push(phrase);
+    }
+  }
+
+  return [...new Set(keywords)];
+}
+
+function countCompetitorsWithKeyword(
+  keyword: string,
+  competitorKeywordsMap: Map<string, Set<string>>
+): number {
+  let count = 0;
+  for (const [, keywords] of competitorKeywordsMap) {
+    if (keywords.has(keyword)) {
+      count++;
+    }
+  }
+  return count;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Format competitor analysis report
+ */
+export function formatCompetitorReport(result: CompetitorKeywordResult): string {
+  const lines: string[] = [];
+
+  lines.push('');
+  lines.push('═'.repeat(70));
+  lines.push('  COMPETITOR KEYWORD ANALYSIS');
+  lines.push('═'.repeat(70));
+  lines.push('');
+
+  // Competitor overlap summary
+  if (result.competitorOverlap.length > 0) {
+    lines.push('📊 COMPETITOR OVERLAP');
+    lines.push('─'.repeat(70));
+    lines.push('  Domain                          Shared  Gaps  Your Wins');
+    lines.push('  ' + '─'.repeat(60));
+
+    for (const comp of result.competitorOverlap.slice(0, 5)) {
+      const domain = comp.competitor.substring(0, 30).padEnd(32);
+      const shared = String(comp.sharedKeywords).padStart(6);
+      const gaps = String(comp.uniqueToThem).padStart(6);
+      const wins = String(comp.uniqueToYou).padStart(8);
+      lines.push(`  ${domain} ${shared} ${gaps} ${wins}`);
+    }
+    lines.push('');
+  }
+
+  // Missing keywords (opportunities)
+  if (result.missingKeywords.length > 0) {
+    lines.push('🔴 KEYWORD GAPS (Competitors Rank, You Don\'t)');
+    lines.push('─'.repeat(70));
+    lines.push('  These are high-priority opportunities - competitors have proven');
+    lines.push('  these keywords work in your market.');
+    lines.push('');
+
+    for (const kw of result.missingKeywords.slice(0, 10)) {
+      lines.push(`  • ${kw.keyword}`);
+    }
+    lines.push('');
+  }
+
+  // Core keywords
+  if (result.coreKeywords.length > 0) {
+    lines.push('🟢 CORE KEYWORDS (All Competitors Rank)');
+    lines.push('─'.repeat(70));
+    lines.push('  These keywords define your market. You should rank for these.');
+    lines.push('');
+
+    for (const kw of result.coreKeywords.slice(0, 10)) {
+      lines.push(`  • ${kw.keyword}`);
+    }
+    lines.push('');
+  }
+
+  // Unique keywords
+  if (result.uniqueKeywords.length > 0) {
+    lines.push('🔵 YOUR UNIQUE KEYWORDS (Only You Rank)');
+    lines.push('─'.repeat(70));
+    lines.push('  Potential competitive advantages or niche opportunities.');
+    lines.push('');
+
+    for (const kw of result.uniqueKeywords.slice(0, 10)) {
+      lines.push(`  • ${kw.keyword}`);
+    }
+    lines.push('');
+  }
+
+  // Summary stats
+  lines.push('📈 SUMMARY');
+  lines.push('─'.repeat(70));
+  lines.push(`  Total Keywords Analyzed: ${result.keywordUniverse.length}`);
+  lines.push(`  Core Keywords:          ${result.coreKeywords.length}`);
+  lines.push(`  Keyword Gaps:           ${result.missingKeywords.length}`);
+  lines.push(`  Your Unique Keywords:   ${result.uniqueKeywords.length}`);
+  lines.push('');
+  lines.push('═'.repeat(70));
+
+  return lines.join('\n');
+}

package/src/keywords/sources/dataforseo.ts

@@ -0,0 +1,206 @@
+// DataForSEO API Integration (Budget-Friendly Paid Option)
+// Pricing: ~$0.002 per request, $50 minimum deposit
+
+import { httpGet, httpPost } from '../../utils/http.js';
+import type { KeywordData } from '../types.js';
+
+const DATAFORSEO_BASE_URL = 'https://api.dataforseo.com/v3';
+
+export interface DataForSEOCredentials {
+  login: string;
+  password: string;
+}
+
+export interface DataForSEOKeywordResult {
+  keyword: string;
+  search_volume: number;
+  keyword_difficulty: number;
+  cpc: number;
+  competition: number;
+  competition_level: string;
+  monthly_searches?: Array<{ month: string; search_volume: number }>;
+}
+
+// Get keyword data with search volume and difficulty
+export async function getKeywordData(
+  keywords: string[],
+  credentials: DataForSEOCredentials,
+  location: number = 2840 // US
+): Promise<KeywordData[]> {
+  const auth = Buffer.from(`${credentials.login}:${credentials.password}`).toString('base64');
+
+  try {
+    const response = await httpPost(
+      `${DATAFORSEO_BASE_URL}/keywords_data/google_ads/search_volume/live`,
+      [
+        {
+          keywords,
+          location_code: location,
+          language_code: 'en',
+        },
+      ],
+      {
+        headers: {
+          Authorization: `Basic ${auth}`,
+          'Content-Type': 'application/json',
+        },
+        timeout: 30000,
+      }
+    );
+
+    const data = response.data as { tasks?: Array<{ result?: DataForSEOKeywordResult[] }> };
+    const results: DataForSEOKeywordResult[] = data?.tasks?.[0]?.result || [];
+
+    return results.map((r) => ({
+      keyword: r.keyword,
+      searchVolume: r.search_volume || 0,
+      keywordDifficulty: r.keyword_difficulty || estimateKdFromCompetition(r.competition),
+      cpc: r.cpc,
+      source: 'dataforseo' as const,
+    }));
+  } catch (error) {
+    console.error('DataForSEO API error:', error);
+    throw new Error(`DataForSEO API failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+
+// Get keyword suggestions from seed keyword
+export async function getKeywordSuggestions(
+  seedKeyword: string,
+  credentials: DataForSEOCredentials,
+  location: number = 2840,
+  limit: number = 100
+): Promise<KeywordData[]> {
+  const auth = Buffer.from(`${credentials.login}:${credentials.password}`).toString('base64');
+
+  try {
+    const response = await httpPost(
+      `${DATAFORSEO_BASE_URL}/dataforseo_labs/google/keyword_suggestions/live`,
+      [
+        {
+          keyword: seedKeyword,
+          location_code: location,
+          language_code: 'en',
+          include_seed_keyword: true,
+          limit,
+        },
+      ],
+      {
+        headers: {
+          Authorization: `Basic ${auth}`,
+          'Content-Type': 'application/json',
+        },
+        timeout: 30000,
+      }
+    );
+
+    const data = response.data as { tasks?: Array<{ result?: Array<{ items?: unknown[] }> }> };
+    const results = data?.tasks?.[0]?.result?.[0]?.items || [];
+
+    return results.map((item) => {
+      const i = item as { keyword_data: DataForSEOKeywordResult };
+      return {
+        keyword: i.keyword_data.keyword,
+        searchVolume: i.keyword_data.search_volume || 0,
+        keywordDifficulty: i.keyword_data.keyword_difficulty || 0,
+        cpc: i.keyword_data.cpc,
+        source: 'dataforseo' as const,
+      };
+    });
+  } catch (error) {
+    console.error('DataForSEO suggestions error:', error);
+    throw new Error(`DataForSEO API failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+  }
+}
+
+// Get related keywords
+export async function getRelatedKeywords(
+  seedKeyword: string,
+  credentials: DataForSEOCredentials,
+  location: number = 2840,
+  limit: number = 50
+): Promise<KeywordData[]> {
+  const auth = Buffer.from(`${credentials.login}:${credentials.password}`).toString('base64');
+
+  try {
+    const response = await httpPost(
+      `${DATAFORSEO_BASE_URL}/dataforseo_labs/google/related_keywords/live`,
+      [
+        {
+          keyword: seedKeyword,
+          location_code: location,
+          language_code: 'en',
+          limit,
+        },
+      ],
+      {
+        headers: {
+          Authorization: `Basic ${auth}`,
+          'Content-Type': 'application/json',
+        },
+        timeout: 30000,
+      }
+    );
+
+    const data = response.data as { tasks?: Array<{ result?: Array<{ items?: unknown[] }> }> };
+    const results = data?.tasks?.[0]?.result?.[0]?.items || [];
+
+    return results.map((item) => {
+      const i = item as { keyword_data: DataForSEOKeywordResult };
+      return {
+        keyword: i.keyword_data.keyword,
+        searchVolume: i.keyword_data.search_volume || 0,
+        keywordDifficulty: i.keyword_data.keyword_difficulty || 0,
+        cpc: i.keyword_data.cpc,
+        source: 'dataforseo' as const,
+      };
+    });
+  } catch (error) {
+    console.error('DataForSEO related keywords error:', error);
+    return [];
+  }
+}
+
+// Estimate KD from competition value (0-1)
+function estimateKdFromCompetition(competition: number): number {
+  // Competition is Google Ads competition (0-1)
+  // Map to KD scale (0-100)
+  return Math.round(competition * 100);
+}
+
+// Location codes reference
+export const LOCATION_CODES = {
+  US: 2840,
+  UK: 2826,
+  CA: 2124,
+  AU: 2036,
+  DE: 2276,
+  FR: 2250,
+  ES: 2724,
+  IT: 2380,
+  BR: 2076,
+  IN: 2356,
+  JP: 2392,
+};
+
+// Check API balance
+export async function checkBalance(credentials: DataForSEOCredentials): Promise<number> {
+  const auth = Buffer.from(`${credentials.login}:${credentials.password}`).toString('base64');
+
+  try {
+    const response = await httpGet<string>(
+      `${DATAFORSEO_BASE_URL}/appendix/user_data`,
+      {
+        headers: {
+          Authorization: `Basic ${auth}`,
+        },
+      }
+    );
+
+    const data = typeof response.data === 'string' ? JSON.parse(response.data) : response.data;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return (data as any)?.tasks?.[0]?.result?.[0]?.money?.balance || 0;
+  } catch {
+    return 0;
+  }
+}