@rankcli/agent-runtime 0.0.15 → 0.0.16

package/src/keywords/sources/competitor-analysis.ts

@@ -1,374 +0,0 @@
-// Competitor Keyword Analysis (SpyFu Kombat-style)
-// Analyze competitor rankings to find keyword gaps and opportunities
-
-import { httpGet } from '../../utils/http.js';
-import * as cheerio from 'cheerio';
-import type { KeywordData } from '../types.js';
-
-const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
-
-export interface CompetitorKeywordResult {
-  // Keywords all competitors rank for (high priority - proven market)
-  coreKeywords: KeywordData[];
-  // Keywords competitors rank for but you don't (opportunities)
-  missingKeywords: KeywordData[];
-  // Keywords only you rank for (potential advantages)
-  uniqueKeywords: KeywordData[];
-  // All keywords any competitor ranks for
-  keywordUniverse: KeywordData[];
-  // Overlap analysis
-  competitorOverlap: CompetitorOverlap[];
-}
-
-export interface CompetitorOverlap {
-  competitor: string;
-  sharedKeywords: number;
-  uniqueToThem: number;
-  uniqueToYou: number;
-}
-
-export interface CompetitorData {
-  domain: string;
-  keywords: Set<string>;
-  rankings: Map<string, number>; // keyword -> position
-}
-
-/**
- * Scrape competitor rankings from Google SERP for seed keywords
- * This discovers what keywords competitors appear for
- */
-export async function discoverCompetitorKeywords(
-  yourDomain: string,
-  seedKeywords: string[],
-  competitors?: string[]
-): Promise<CompetitorKeywordResult> {
-  const yourKeywords = new Set<string>();
-  const competitorKeywordsMap = new Map<string, Set<string>>();
-  const allKeywords = new Map<string, KeywordData>();
-
-  // Initialize competitor tracking
-  const discoveredCompetitors = new Set<string>(competitors || []);
-
-  console.log('  → Analyzing SERP results for competitor keywords...');
-
-  // For each seed keyword, check who ranks and for what related terms
-  for (const seed of seedKeywords) {
-    try {
-      // Get SERP results
-      const serpData = await analyzeSERP(seed);
-
-      // Track who ranks for this keyword
-      for (const result of serpData.results) {
-        const domain = extractDomain(result.url);
-
-        if (domain === yourDomain || domain.includes(yourDomain) || yourDomain.includes(domain)) {
-          yourKeywords.add(seed);
-        } else {
-          // Discovered competitor
-          discoveredCompetitors.add(domain);
-          if (!competitorKeywordsMap.has(domain)) {
-            competitorKeywordsMap.set(domain, new Set());
-          }
-          competitorKeywordsMap.get(domain)!.add(seed);
-        }
-
-        // Extract keywords from competitor titles/descriptions
-        const extractedKeywords = extractKeywordsFromText(result.title + ' ' + result.description);
-        for (const kw of extractedKeywords) {
-          if (!allKeywords.has(kw)) {
-            allKeywords.set(kw, {
-              keyword: kw,
-              searchVolume: 0,
-              keywordDifficulty: 0,
-              source: 'competitor' as const,
-            });
-          }
-
-          // Track which domains have this keyword
-          if (domain === yourDomain || domain.includes(yourDomain)) {
-            yourKeywords.add(kw);
-          } else {
-            competitorKeywordsMap.get(domain)?.add(kw);
-          }
-        }
-      }
-
-      // Also get related searches for more keyword ideas
-      for (const related of serpData.relatedSearches) {
-        if (!allKeywords.has(related)) {
-          allKeywords.set(related, {
-            keyword: related,
-            searchVolume: 0,
-            keywordDifficulty: 0,
-            source: 'competitor' as const,
-          });
-        }
-      }
-
-      // Avoid rate limiting
-      await sleep(500);
-    } catch (error) {
-      // Continue with other seeds
-    }
-  }
-
-  // Calculate keyword categories
-  const coreKeywords: KeywordData[] = [];
-  const missingKeywords: KeywordData[] = [];
-  const uniqueKeywords: KeywordData[] = [];
-  const keywordUniverse: KeywordData[] = [];
-
-  // Analyze each keyword
-  for (const [keyword, data] of allKeywords) {
-    const competitorCount = countCompetitorsWithKeyword(keyword, competitorKeywordsMap);
-    const youHaveIt = yourKeywords.has(keyword);
-
-    keywordUniverse.push(data);
-
-    if (competitorCount >= 2 && youHaveIt) {
-      // Core keyword - everyone ranks
-      coreKeywords.push(data);
-    } else if (competitorCount >= 1 && !youHaveIt) {
-      // Missing keyword - competitors have it, you don't
-      missingKeywords.push(data);
-    } else if (competitorCount === 0 && youHaveIt) {
-      // Unique keyword - only you have it
-      uniqueKeywords.push(data);
-    }
-  }
-
-  // Calculate competitor overlap
-  const competitorOverlap: CompetitorOverlap[] = [];
-  for (const [domain, keywords] of competitorKeywordsMap) {
-    const shared = [...keywords].filter(k => yourKeywords.has(k)).length;
-    const uniqueToThem = [...keywords].filter(k => !yourKeywords.has(k)).length;
-    const uniqueToYou = [...yourKeywords].filter(k => !keywords.has(k)).length;
-
-    competitorOverlap.push({
-      competitor: domain,
-      sharedKeywords: shared,
-      uniqueToThem,
-      uniqueToYou,
-    });
-  }
-
-  // Sort by number of keywords they have that you don't
-  competitorOverlap.sort((a, b) => b.uniqueToThem - a.uniqueToThem);
-
-  return {
-    coreKeywords,
-    missingKeywords,
-    uniqueKeywords,
-    keywordUniverse,
-    competitorOverlap,
-  };
-}
-
-interface SERPResult {
-  url: string;
-  title: string;
-  description: string;
-  position: number;
-}
-
-interface SERPData {
-  results: SERPResult[];
-  relatedSearches: string[];
-  peopleAlsoAsk: string[];
-}
-
-async function analyzeSERP(query: string): Promise<SERPData> {
-  try {
-    const response = await httpGet<string>('https://www.google.com/search', {
-      params: { q: query, num: 20 },
-      headers: { 'User-Agent': USER_AGENT },
-      timeout: 10000,
-    });
-
-    const $ = cheerio.load(response.data);
-    const results: SERPResult[] = [];
-    const relatedSearches: string[] = [];
-    const peopleAlsoAsk: string[] = [];
-
-    // Extract search results
-    let position = 1;
-    $('div.g').each((_, el) => {
-      const link = $(el).find('a').first();
-      const url = link.attr('href');
-      const title = $(el).find('h3').text().trim();
-      const description = $(el).find('.VwiC3b').text().trim() ||
-                         $(el).find('[data-content-feature]').text().trim();
-
-      if (url && url.startsWith('http') && title) {
-        results.push({ url, title, description, position });
-        position++;
-      }
-    });
-
-    // Extract related searches
-    $('div[data-ved] a').each((_, el) => {
-      const href = $(el).attr('href');
-      if (href?.includes('/search?q=')) {
-        const text = $(el).text().trim();
-        if (text.length > 3 && text.length < 100 && !text.includes('http')) {
-          relatedSearches.push(text);
-        }
-      }
-    });
-
-    // Extract People Also Ask
-    $('[data-sgrd]').each((_, el) => {
-      const text = $(el).text().trim();
-      if (text.endsWith('?') && text.length > 10) {
-        peopleAlsoAsk.push(text);
-      }
-    });
-
-    return {
-      results,
-      relatedSearches: [...new Set(relatedSearches)],
-      peopleAlsoAsk: [...new Set(peopleAlsoAsk)],
-    };
-  } catch {
-    return { results: [], relatedSearches: [], peopleAlsoAsk: [] };
-  }
-}
-
-function extractDomain(url: string): string {
-  try {
-    const parsed = new URL(url);
-    return parsed.hostname.replace(/^www\./, '');
-  } catch {
-    return '';
-  }
-}
-
-function extractKeywordsFromText(text: string): string[] {
-  // Extract 2-4 word phrases that could be keywords
-  const words = text.toLowerCase()
-    .replace(/[^\w\s]/g, ' ')
-    .split(/\s+/)
-    .filter(w => w.length > 2);
-
-  const keywords: string[] = [];
-
-  // Single words (4+ chars)
-  words.filter(w => w.length >= 4).forEach(w => keywords.push(w));
-
-  // 2-word phrases
-  for (let i = 0; i < words.length - 1; i++) {
-    const phrase = `${words[i]} ${words[i + 1]}`;
-    if (phrase.length >= 5 && phrase.length <= 50) {
-      keywords.push(phrase);
-    }
-  }
-
-  // 3-word phrases
-  for (let i = 0; i < words.length - 2; i++) {
-    const phrase = `${words[i]} ${words[i + 1]} ${words[i + 2]}`;
-    if (phrase.length >= 8 && phrase.length <= 60) {
-      keywords.push(phrase);
-    }
-  }
-
-  return [...new Set(keywords)];
-}
-
-function countCompetitorsWithKeyword(
-  keyword: string,
-  competitorKeywordsMap: Map<string, Set<string>>
-): number {
-  let count = 0;
-  for (const [, keywords] of competitorKeywordsMap) {
-    if (keywords.has(keyword)) {
-      count++;
-    }
-  }
-  return count;
-}
-
-function sleep(ms: number): Promise<void> {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-/**
- * Format competitor analysis report
- */
-export function formatCompetitorReport(result: CompetitorKeywordResult): string {
-  const lines: string[] = [];
-
-  lines.push('');
-  lines.push('═'.repeat(70));
-  lines.push('  COMPETITOR KEYWORD ANALYSIS');
-  lines.push('═'.repeat(70));
-  lines.push('');
-
-  // Competitor overlap summary
-  if (result.competitorOverlap.length > 0) {
-    lines.push('📊 COMPETITOR OVERLAP');
-    lines.push('─'.repeat(70));
-    lines.push('  Domain                          Shared  Gaps  Your Wins');
-    lines.push('  ' + '─'.repeat(60));
-
-    for (const comp of result.competitorOverlap.slice(0, 5)) {
-      const domain = comp.competitor.substring(0, 30).padEnd(32);
-      const shared = String(comp.sharedKeywords).padStart(6);
-      const gaps = String(comp.uniqueToThem).padStart(6);
-      const wins = String(comp.uniqueToYou).padStart(8);
-      lines.push(`  ${domain} ${shared} ${gaps} ${wins}`);
-    }
-    lines.push('');
-  }
-
-  // Missing keywords (opportunities)
-  if (result.missingKeywords.length > 0) {
-    lines.push('🔴 KEYWORD GAPS (Competitors Rank, You Don\'t)');
-    lines.push('─'.repeat(70));
-    lines.push('  These are high-priority opportunities - competitors have proven');
-    lines.push('  these keywords work in your market.');
-    lines.push('');
-
-    for (const kw of result.missingKeywords.slice(0, 10)) {
-      lines.push(`  • ${kw.keyword}`);
-    }
-    lines.push('');
-  }
-
-  // Core keywords
-  if (result.coreKeywords.length > 0) {
-    lines.push('🟢 CORE KEYWORDS (All Competitors Rank)');
-    lines.push('─'.repeat(70));
-    lines.push('  These keywords define your market. You should rank for these.');
-    lines.push('');
-
-    for (const kw of result.coreKeywords.slice(0, 10)) {
-      lines.push(`  • ${kw.keyword}`);
-    }
-    lines.push('');
-  }
-
-  // Unique keywords
-  if (result.uniqueKeywords.length > 0) {
-    lines.push('🔵 YOUR UNIQUE KEYWORDS (Only You Rank)');
-    lines.push('─'.repeat(70));
-    lines.push('  Potential competitive advantages or niche opportunities.');
-    lines.push('');
-
-    for (const kw of result.uniqueKeywords.slice(0, 10)) {
-      lines.push(`  • ${kw.keyword}`);
-    }
-    lines.push('');
-  }
-
-  // Summary stats
-  lines.push('📈 SUMMARY');
-  lines.push('─'.repeat(70));
-  lines.push(`  Total Keywords Analyzed: ${result.keywordUniverse.length}`);
-  lines.push(`  Core Keywords:          ${result.coreKeywords.length}`);
-  lines.push(`  Keyword Gaps:           ${result.missingKeywords.length}`);
-  lines.push(`  Your Unique Keywords:   ${result.uniqueKeywords.length}`);
-  lines.push('');
-  lines.push('═'.repeat(70));
-
-  return lines.join('\n');
-}

package/src/keywords/sources/dataforseo.ts

@@ -1,206 +0,0 @@
-// DataForSEO API Integration (Budget-Friendly Paid Option)
-// Pricing: ~$0.002 per request, $50 minimum deposit
-
-import { httpGet, httpPost } from '../../utils/http.js';
-import type { KeywordData } from '../types.js';
-
-const DATAFORSEO_BASE_URL = 'https://api.dataforseo.com/v3';
-
-export interface DataForSEOCredentials {
-  login: string;
-  password: string;
-}
-
-export interface DataForSEOKeywordResult {
-  keyword: string;
-  search_volume: number;
-  keyword_difficulty: number;
-  cpc: number;
-  competition: number;
-  competition_level: string;
-  monthly_searches?: Array<{ month: string; search_volume: number }>;
-}
-
-// Get keyword data with search volume and difficulty
-export async function getKeywordData(
-  keywords: string[],
-  credentials: DataForSEOCredentials,
-  location: number = 2840 // US
-): Promise<KeywordData[]> {
-  const auth = Buffer.from(`${credentials.login}:${credentials.password}`).toString('base64');
-
-  try {
-    const response = await httpPost(
-      `${DATAFORSEO_BASE_URL}/keywords_data/google_ads/search_volume/live`,
-      [
-        {
-          keywords,
-          location_code: location,
-          language_code: 'en',
-        },
-      ],
-      {
-        headers: {
-          Authorization: `Basic ${auth}`,
-          'Content-Type': 'application/json',
-        },
-        timeout: 30000,
-      }
-    );
-
-    const data = response.data as { tasks?: Array<{ result?: DataForSEOKeywordResult[] }> };
-    const results: DataForSEOKeywordResult[] = data?.tasks?.[0]?.result || [];
-
-    return results.map((r) => ({
-      keyword: r.keyword,
-      searchVolume: r.search_volume || 0,
-      keywordDifficulty: r.keyword_difficulty || estimateKdFromCompetition(r.competition),
-      cpc: r.cpc,
-      source: 'dataforseo' as const,
-    }));
-  } catch (error) {
-    console.error('DataForSEO API error:', error);
-    throw new Error(`DataForSEO API failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
-  }
-}
-
-// Get keyword suggestions from seed keyword
-export async function getKeywordSuggestions(
-  seedKeyword: string,
-  credentials: DataForSEOCredentials,
-  location: number = 2840,
-  limit: number = 100
-): Promise<KeywordData[]> {
-  const auth = Buffer.from(`${credentials.login}:${credentials.password}`).toString('base64');
-
-  try {
-    const response = await httpPost(
-      `${DATAFORSEO_BASE_URL}/dataforseo_labs/google/keyword_suggestions/live`,
-      [
-        {
-          keyword: seedKeyword,
-          location_code: location,
-          language_code: 'en',
-          include_seed_keyword: true,
-          limit,
-        },
-      ],
-      {
-        headers: {
-          Authorization: `Basic ${auth}`,
-          'Content-Type': 'application/json',
-        },
-        timeout: 30000,
-      }
-    );
-
-    const data = response.data as { tasks?: Array<{ result?: Array<{ items?: unknown[] }> }> };
-    const results = data?.tasks?.[0]?.result?.[0]?.items || [];
-
-    return results.map((item) => {
-      const i = item as { keyword_data: DataForSEOKeywordResult };
-      return {
-        keyword: i.keyword_data.keyword,
-        searchVolume: i.keyword_data.search_volume || 0,
-        keywordDifficulty: i.keyword_data.keyword_difficulty || 0,
-        cpc: i.keyword_data.cpc,
-        source: 'dataforseo' as const,
-      };
-    });
-  } catch (error) {
-    console.error('DataForSEO suggestions error:', error);
-    throw new Error(`DataForSEO API failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
-  }
-}
-
-// Get related keywords
-export async function getRelatedKeywords(
-  seedKeyword: string,
-  credentials: DataForSEOCredentials,
-  location: number = 2840,
-  limit: number = 50
-): Promise<KeywordData[]> {
-  const auth = Buffer.from(`${credentials.login}:${credentials.password}`).toString('base64');
-
-  try {
-    const response = await httpPost(
-      `${DATAFORSEO_BASE_URL}/dataforseo_labs/google/related_keywords/live`,
-      [
-        {
-          keyword: seedKeyword,
-          location_code: location,
-          language_code: 'en',
-          limit,
-        },
-      ],
-      {
-        headers: {
-          Authorization: `Basic ${auth}`,
-          'Content-Type': 'application/json',
-        },
-        timeout: 30000,
-      }
-    );
-
-    const data = response.data as { tasks?: Array<{ result?: Array<{ items?: unknown[] }> }> };
-    const results = data?.tasks?.[0]?.result?.[0]?.items || [];
-
-    return results.map((item) => {
-      const i = item as { keyword_data: DataForSEOKeywordResult };
-      return {
-        keyword: i.keyword_data.keyword,
-        searchVolume: i.keyword_data.search_volume || 0,
-        keywordDifficulty: i.keyword_data.keyword_difficulty || 0,
-        cpc: i.keyword_data.cpc,
-        source: 'dataforseo' as const,
-      };
-    });
-  } catch (error) {
-    console.error('DataForSEO related keywords error:', error);
-    return [];
-  }
-}
-
-// Estimate KD from competition value (0-1)
-function estimateKdFromCompetition(competition: number): number {
-  // Competition is Google Ads competition (0-1)
-  // Map to KD scale (0-100)
-  return Math.round(competition * 100);
-}
-
-// Location codes reference
-export const LOCATION_CODES = {
-  US: 2840,
-  UK: 2826,
-  CA: 2124,
-  AU: 2036,
-  DE: 2276,
-  FR: 2250,
-  ES: 2724,
-  IT: 2380,
-  BR: 2076,
-  IN: 2356,
-  JP: 2392,
-};
-
-// Check API balance
-export async function checkBalance(credentials: DataForSEOCredentials): Promise<number> {
-  const auth = Buffer.from(`${credentials.login}:${credentials.password}`).toString('base64');
-
-  try {
-    const response = await httpGet<string>(
-      `${DATAFORSEO_BASE_URL}/appendix/user_data`,
-      {
-        headers: {
-          Authorization: `Basic ${auth}`,
-        },
-      }
-    );
-
-    const data = typeof response.data === 'string' ? JSON.parse(response.data) : response.data;
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    return (data as any)?.tasks?.[0]?.result?.[0]?.money?.balance || 0;
-  } catch {
-    return 0;
-  }
-}