@rankcli/agent-runtime 0.0.15 → 0.0.17

package/src/keywords/sources/free-sources.ts

@@ -1,294 +0,0 @@
-// Free Keyword Data Sources
-// Prioritize these over paid APIs
-
-import { httpGet } from '../../utils/http.js';
-import * as cheerio from 'cheerio';
-import type { KeywordData } from '../types.js';
-
-const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
-
-// 1. Google "People Also Ask" Scraper
-export async function getPeopleAlsoAsk(query: string): Promise<string[]> {
-  try {
-    const response = await httpGet<string>('https://www.google.com/search', {
-      params: { q: query },
-      headers: { 'User-Agent': USER_AGENT },
-      timeout: 10000,
-    });
-
-    const $ = cheerio.load(response.data);
-    const questions: string[] = [];
-
-    // PAA boxes have specific data attributes
-    $('[data-sgrd]').each((_, el) => {
-      const text = $(el).text().trim();
-      if (text.endsWith('?') && text.length > 10) {
-        questions.push(text);
-      }
-    });
-
-    // Alternative selector
-    $('div[jsname="Cpkphb"]').each((_, el) => {
-      const text = $(el).text().trim();
-      if (text.length > 10 && text.length < 200) {
-        questions.push(text);
-      }
-    });
-
-    return [...new Set(questions)].slice(0, 10);
-  } catch {
-    return [];
-  }
-}
-
-// 2. Google Related Searches Scraper
-export async function getRelatedSearches(query: string): Promise<string[]> {
-  try {
-    const response = await httpGet<string>('https://www.google.com/search', {
-      params: { q: query },
-      headers: { 'User-Agent': USER_AGENT },
-      timeout: 10000,
-    });
-
-    const $ = cheerio.load(response.data);
-    const related: string[] = [];
-
-    // Related searches at bottom of SERP
-    $('div[data-ved] a').each((_, el) => {
-      const href = $(el).attr('href');
-      if (href?.includes('/search?q=')) {
-        const text = $(el).text().trim();
-        if (text.length > 3 && text.length < 100 && !text.includes('http')) {
-          related.push(text);
-        }
-      }
-    });
-
-    return [...new Set(related)].slice(0, 8);
-  } catch {
-    return [];
-  }
-}
-
-// 3. Wikipedia Topics for Long-Tail Ideas
-export async function getWikipediaTopics(query: string): Promise<string[]> {
-  try {
-    const response = await httpGet<[string, string[], string[], string[]]>('https://en.wikipedia.org/w/api.php', {
-      params: {
-        action: 'opensearch',
-        search: query,
-        limit: 10,
-        namespace: 0,
-        format: 'json',
-      },
-      timeout: 5000,
-    });
-
-    // Response: [query, [titles], [descriptions], [urls]]
-    return response.data[1] || [];
-  } catch {
-    return [];
-  }
-}
-
-// 4. AnswerThePublic-style Question Generator
-export function generateQuestions(keyword: string): string[] {
-  const questionPrefixes = [
-    'what is', 'what are', 'what does',
-    'how to', 'how do', 'how does', 'how can',
-    'why is', 'why do', 'why does', 'why are',
-    'when to', 'when should', 'when do',
-    'where to', 'where can', 'where do',
-    'which', 'who', 'can you', 'is it',
-    'are there', 'does', 'do',
-  ];
-
-  const questions: string[] = [];
-
-  for (const prefix of questionPrefixes) {
-    questions.push(`${prefix} ${keyword}`);
-  }
-
-  return questions;
-}
-
-// 5. Modifier-based Keyword Expansion
-export function expandWithModifiers(keyword: string): string[] {
-  const modifiers = {
-    commercial: ['best', 'top', 'review', 'comparison', 'vs', 'alternative', 'pricing'],
-    informational: ['how to', 'what is', 'guide', 'tutorial', 'examples', 'tips'],
-    temporal: ['2024', '2025', 'new', 'latest', 'updated'],
-    intent: ['free', 'online', 'cheap', 'premium', 'professional'],
-    action: ['download', 'buy', 'get', 'create', 'make', 'build', 'use'],
-  };
-
-  const expanded: string[] = [];
-
-  for (const [, mods] of Object.entries(modifiers)) {
-    for (const mod of mods) {
-      expanded.push(`${mod} ${keyword}`);
-      expanded.push(`${keyword} ${mod}`);
-    }
-  }
-
-  return expanded;
-}
-
-// 6. Competitor Title Analyzer (from SERP)
-export async function analyzeCompetitorTitles(query: string): Promise<{ titles: string[]; keywords: string[] }> {
-  try {
-    const response = await httpGet<string>('https://www.google.com/search', {
-      params: { q: query, num: 10 },
-      headers: { 'User-Agent': USER_AGENT },
-      timeout: 10000,
-    });
-
-    const $ = cheerio.load(response.data);
-    const titles: string[] = [];
-    const keywords: Set<string> = new Set();
-
-    // Extract titles from search results
-    $('h3').each((_, el) => {
-      const title = $(el).text().trim();
-      if (title.length > 10 && title.length < 100) {
-        titles.push(title);
-
-        // Extract potential keywords from titles
-        const words = title.toLowerCase().split(/[\s\-|:]+/);
-        words.filter(w => w.length > 3).forEach(w => keywords.add(w));
-      }
-    });
-
-    return {
-      titles: titles.slice(0, 10),
-      keywords: Array.from(keywords).slice(0, 20),
-    };
-  } catch {
-    return { titles: [], keywords: [] };
-  }
-}
-
-// 7. Combine All Free Sources
-export async function getAllFreeKeywordIdeas(
-  seedKeyword: string,
-  options: { includeQuestions?: boolean; includeModifiers?: boolean } = {}
-): Promise<KeywordData[]> {
-  const { includeQuestions = true, includeModifiers = true } = options;
-
-  const allKeywords: string[] = [];
-  const seen = new Set<string>();
-
-  const addKeyword = (kw: string) => {
-    const normalized = kw.toLowerCase().trim();
-    if (!seen.has(normalized) && normalized.length > 2) {
-      seen.add(normalized);
-      allKeywords.push(kw);
-    }
-  };
-
-  // Run sources in parallel where possible
-  console.log('  → Fetching from Google autocomplete...');
-  const [paa, related, wiki, competitors] = await Promise.all([
-    getPeopleAlsoAsk(seedKeyword).catch(() => []),
-    getRelatedSearches(seedKeyword).catch(() => []),
-    getWikipediaTopics(seedKeyword).catch(() => []),
-    analyzeCompetitorTitles(seedKeyword).catch(() => ({ titles: [], keywords: [] })),
-  ]);
-
-  // Add PAA questions
-  paa.forEach(addKeyword);
-
-  // Add related searches
-  related.forEach(addKeyword);
-
-  // Add Wikipedia topics
-  wiki.forEach(addKeyword);
-
-  // Add competitor keywords
-  competitors.keywords.forEach(addKeyword);
-
-  // Add generated questions
-  if (includeQuestions) {
-    generateQuestions(seedKeyword).forEach(addKeyword);
-  }
-
-  // Add modifier expansions
-  if (includeModifiers) {
-    expandWithModifiers(seedKeyword).forEach(addKeyword);
-  }
-
-  // Convert to KeywordData format
-  return allKeywords.map((kw) => ({
-    keyword: kw,
-    searchVolume: 0, // Will be estimated
-    keywordDifficulty: 0, // Will be estimated
-    source: 'autocomplete' as const,
-  }));
-}
-
-// 8. Estimate metrics based on keyword characteristics
-export function estimateMetrics(keyword: string): { volume: number; difficulty: number } {
-  const words = keyword.split(' ').length;
-  const hasQuestion = /^(how|what|why|when|where|who|which|can|is|are|do|does)/i.test(keyword);
-  const hasCommercial = /\b(best|top|review|buy|price|cheap|free|vs|alternative)\b/i.test(keyword);
-  const hasYear = /\b20\d{2}\b/.test(keyword);
-
-  // Base estimates
-  let volume = 500;
-  let difficulty = 30;
-
-  // Adjust based on word count (longer = less volume, less competition)
-  if (words >= 5) {
-    volume = 50;
-    difficulty = 10;
-  } else if (words >= 4) {
-    volume = 150;
-    difficulty = 15;
-  } else if (words >= 3) {
-    volume = 300;
-    difficulty = 20;
-  } else if (words === 2) {
-    volume = 800;
-    difficulty = 35;
-  } else {
-    volume = 2000;
-    difficulty = 50;
-  }
-
-  // Adjust for question keywords (usually lower competition)
-  if (hasQuestion) {
-    difficulty -= 5;
-    volume *= 0.8;
-  }
-
-  // Adjust for commercial intent (usually higher competition)
-  if (hasCommercial) {
-    difficulty += 10;
-    volume *= 1.2;
-  }
-
-  // Year keywords are often seasonal
-  if (hasYear) {
-    volume *= 1.5;
-  }
-
-  return {
-    volume: Math.round(Math.max(10, volume)),
-    difficulty: Math.round(Math.max(1, Math.min(100, difficulty))),
-  };
-}
-
-// 9. Apply estimates to keyword list
-export function applyEstimates(keywords: KeywordData[]): KeywordData[] {
-  return keywords.map((kw) => {
-    if (kw.searchVolume === 0 || kw.keywordDifficulty === 0) {
-      const estimates = estimateMetrics(kw.keyword);
-      return {
-        ...kw,
-        searchVolume: kw.searchVolume || estimates.volume,
-        keywordDifficulty: kw.keywordDifficulty || estimates.difficulty,
-      };
-    }
-    return kw;
-  });
-}

package/src/keywords/sources/gsc.ts

@@ -1,123 +0,0 @@
-// Google Search Console Integration (Free - Your Own Data)
-
-import type { KeywordData, GSCQueryData } from '../types.js';
-
-// Note: GSC API requires OAuth2 authentication
-// This module provides the structure and helpers for GSC integration
-
-export interface GSCCredentials {
-  clientId: string;
-  clientSecret: string;
-  refreshToken: string;
-}
-
-export interface GSCQueryOptions {
-  siteUrl: string;
-  startDate: string;
-  endDate: string;
-  dimensions?: string[];
-  rowLimit?: number;
-  startRow?: number;
-}
-
-// Transform GSC data to our KeywordData format
-export function transformGSCData(gscData: GSCQueryData[]): KeywordData[] {
-  return gscData.map((row) => ({
-    keyword: row.query,
-    searchVolume: estimateVolumeFromImpressions(row.impressions),
-    keywordDifficulty: estimateKdFromPosition(row.position),
-    source: 'gsc' as const,
-    currentRanking: row.position,
-    impressions: row.impressions,
-    clicks: row.clicks,
-    ctr: row.ctr,
-  }));
-}
-
-// Estimate search volume from impressions
-// If you're ranking #1-3, impressions ≈ 60-90% of total volume
-// If you're ranking #4-10, impressions ≈ 10-30% of total volume
-function estimateVolumeFromImpressions(impressions: number): number {
-  // Rough estimate: multiply by factor based on assumed position
-  // This is a simplified calculation
-  return Math.round(impressions * 1.5);
-}
-
-// Estimate KD from current ranking position
-// If you're already ranking, KD is within your reach
-function estimateKdFromPosition(position: number): number {
-  if (position <= 3) return 5; // Already ranking well = easy for you
-  if (position <= 10) return 15; // First page = achievable
-  if (position <= 20) return 25; // Second page = moderate
-  if (position <= 50) return 35; // You're in the game
-  return 45; // Need more effort
-}
-
-// Find keywords with high impressions but low clicks (optimization opportunities)
-export function findCTROpportunities(gscData: GSCQueryData[]): GSCQueryData[] {
-  return gscData
-    .filter((row) => {
-      // High impressions, low CTR, good position
-      return row.impressions > 100 && row.ctr < 0.03 && row.position <= 20;
-    })
-    .sort((a, b) => b.impressions - a.impressions);
-}
-
-// Find keywords where you're close to page 1
-export function findAlmostPage1Keywords(gscData: GSCQueryData[]): GSCQueryData[] {
-  return gscData
-    .filter((row) => row.position > 10 && row.position <= 20 && row.impressions > 50)
-    .sort((a, b) => a.position - b.position);
-}
-
-// Find your best performing keywords
-export function findTopPerformers(gscData: GSCQueryData[]): GSCQueryData[] {
-  return gscData
-    .filter((row) => row.position <= 10 && row.clicks > 10)
-    .sort((a, b) => b.clicks - a.clicks);
-}
-
-// Generate GSC API request body
-export function buildGSCRequest(options: GSCQueryOptions): object {
-  return {
-    startDate: options.startDate,
-    endDate: options.endDate,
-    dimensions: options.dimensions || ['query'],
-    rowLimit: options.rowLimit || 1000,
-    startRow: options.startRow || 0,
-  };
-}
-
-// Parse date range for last N days
-export function getDateRange(days: number = 28): { startDate: string; endDate: string } {
-  const endDate = new Date();
-  endDate.setDate(endDate.getDate() - 3); // GSC data is delayed by ~3 days
-
-  const startDate = new Date(endDate);
-  startDate.setDate(startDate.getDate() - days);
-
-  return {
-    startDate: startDate.toISOString().split('T')[0],
-    endDate: endDate.toISOString().split('T')[0],
-  };
-}
-
-// Example response structure for reference
-export const GSC_EXAMPLE_RESPONSE = {
-  rows: [
-    {
-      keys: ['mermaid animation'],
-      clicks: 45,
-      impressions: 1200,
-      ctr: 0.0375,
-      position: 4.2,
-    },
-    {
-      keys: ['animate diagrams'],
-      clicks: 12,
-      impressions: 800,
-      ctr: 0.015,
-      position: 8.5,
-    },
-  ],
-};