@rankcli/agent-runtime 0.0.1

package/src/keywords/engine.ts

@@ -0,0 +1,359 @@
+// Main Keyword Research Engine
+
+import * as cheerio from 'cheerio';
+import { httpGet } from '../utils/http.js';
+import type {
+  SiteProfile,
+  KeywordData,
+  KeywordResearchResult,
+} from './types.js';
+import { prioritizeKeywords } from './prioritizer.js';
+import {
+  getExpandedSuggestions,
+  enrichKeywordsWithEstimates,
+} from './sources/autocomplete.js';
+import {
+  getAllFreeKeywordIdeas,
+  applyEstimates,
+} from './sources/free-sources.js';
+import {
+  getKeywordSuggestions,
+  getRelatedKeywords,
+  type DataForSEOCredentials,
+} from './sources/dataforseo.js';
+
+const USER_AGENT = 'SEO-Autopilot/1.0 (+https://seo-autopilot.dev)';
+
+export interface KeywordResearchOptions {
+  seedKeywords: string[];
+  siteProfile: SiteProfile;
+  url?: string; // To fetch current meta for suggestions
+  dataForSEOCredentials?: DataForSEOCredentials;
+  maxKeywords?: number;
+}
+
+export interface PageMeta {
+  title?: string;
+  description?: string;
+  h1?: string;
+  url: string;
+}
+
+export async function runKeywordResearch(
+  options: KeywordResearchOptions
+): Promise<KeywordResearchResult> {
+  const {
+    seedKeywords,
+    siteProfile,
+    url,
+    dataForSEOCredentials,
+    maxKeywords = 50,
+  } = options;
+
+  console.log('\n🔍 Starting keyword research...\n');
+
+  // Fetch current page meta if URL provided
+  let pageMeta: PageMeta | undefined;
+  if (url) {
+    console.log('📥 Fetching current page meta...');
+    pageMeta = await fetchPageMeta(url);
+  }
+
+  // Collect keywords from different sources
+  let allKeywords: KeywordData[] = [];
+
+  // Source 1: Free Sources (Autocomplete, PAA, Related, Wikipedia, Competitors)
+  console.log('🔤 Getting keyword ideas from free sources...');
+  for (const seed of seedKeywords) {
+    // Expanded autocomplete (alphabet soup technique)
+    const autocompleteSuggestions = await getExpandedSuggestions(seed, {
+      language: 'en',
+      country: siteProfile.targetGeo === 'us' ? 'us' : siteProfile.targetGeo,
+    });
+    allKeywords.push(...autocompleteSuggestions);
+
+    // All other free sources (PAA, Related, Wikipedia, Modifiers)
+    const freeIdeas = await getAllFreeKeywordIdeas(seed, {
+      includeQuestions: true,
+      includeModifiers: true,
+    });
+    allKeywords.push(...freeIdeas);
+  }
+
+  // Source 2: DataForSEO (Premium - only if credentials provided)
+  if (dataForSEOCredentials) {
+    console.log('💎 Getting premium keyword data from DataForSEO...');
+    try {
+      for (const seed of seedKeywords) {
+        const suggestions = await getKeywordSuggestions(
+          seed,
+          dataForSEOCredentials,
+          2840, // US
+          30
+        );
+        allKeywords.push(...suggestions);
+
+        const related = await getRelatedKeywords(
+          seed,
+          dataForSEOCredentials,
+          2840,
+          20
+        );
+        allKeywords.push(...related);
+      }
+    } catch (error) {
+      console.log('⚠️  DataForSEO fetch failed, using free estimates');
+    }
+  }
+
+  // Remove duplicates
+  const uniqueKeywords = deduplicateKeywords(allKeywords);
+
+  // Enrich keywords without data with estimates (free estimation)
+  const enrichedKeywords = applyEstimates(enrichKeywordsWithEstimates(uniqueKeywords));
+
+  // Filter to relevant keywords
+  const relevantKeywords = filterRelevantKeywords(enrichedKeywords, seedKeywords);
+
+  // Limit to max keywords
+  const limitedKeywords = relevantKeywords.slice(0, maxKeywords);
+
+  console.log(`\n📋 Found ${limitedKeywords.length} keyword opportunities\n`);
+
+  // Prioritize and categorize
+  const result = prioritizeKeywords(
+    limitedKeywords,
+    siteProfile,
+    pageMeta
+      ? {
+          title: pageMeta.title,
+          description: pageMeta.description,
+          h1: pageMeta.h1,
+        }
+      : undefined
+  );
+
+  return result;
+}
+
+async function fetchPageMeta(url: string): Promise<PageMeta> {
+  try {
+    const response = await httpGet<string>(url, {
+      
+      timeout: 10000,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    return {
+      url,
+      title: $('title').text().trim() || undefined,
+      description: $('meta[name="description"]').attr('content')?.trim() || undefined,
+      h1: $('h1').first().text().trim() || undefined,
+    };
+  } catch {
+    return { url };
+  }
+}
+
+function deduplicateKeywords(keywords: KeywordData[]): KeywordData[] {
+  const seen = new Map<string, KeywordData>();
+
+  for (const kw of keywords) {
+    const key = kw.keyword.toLowerCase().trim();
+    const existing = seen.get(key);
+
+    // Keep the one with more data (higher volume or from better source)
+    if (!existing || kw.searchVolume > existing.searchVolume) {
+      seen.set(key, kw);
+    }
+  }
+
+  return Array.from(seen.values());
+}
+
+function filterRelevantKeywords(
+  keywords: KeywordData[],
+  seedKeywords: string[]
+): KeywordData[] {
+  const seeds = seedKeywords.map((s) => s.toLowerCase());
+
+  return keywords.filter((kw) => {
+    const kwLower = kw.keyword.toLowerCase();
+
+    // Keep if contains any seed keyword
+    if (seeds.some((seed) => kwLower.includes(seed) || seed.includes(kwLower))) {
+      return true;
+    }
+
+    // Keep if related to common patterns
+    const relatedPatterns = [
+      'how to',
+      'best',
+      'free',
+      'online',
+      'tool',
+      'app',
+      'software',
+      'tutorial',
+      'guide',
+    ];
+
+    return relatedPatterns.some((pattern) => kwLower.includes(pattern));
+  });
+}
+
+// Format results for console output
+export function formatKeywordReport(result: KeywordResearchResult): string {
+  const lines: string[] = [];
+
+  lines.push('');
+  lines.push('═'.repeat(70));
+  lines.push('  KEYWORD RESEARCH REPORT');
+  lines.push('═'.repeat(70));
+  lines.push('');
+
+  // Site profile summary
+  lines.push('📋 SITE PROFILE');
+  lines.push('─'.repeat(70));
+  lines.push(`  Domain:    ${result.siteProfile.domain}`);
+  lines.push(`  Age:       ${result.siteProfile.domainAge}`);
+  lines.push(`  Backlinks: ${result.siteProfile.backlinkCount}`);
+  lines.push(`  Goal:      ${result.siteProfile.businessGoal}`);
+  lines.push(`  Max KD:    ${result.maxKdThreshold} (based on your profile)`);
+  lines.push('');
+
+  // Quick wins
+  if (result.quickWins.length > 0) {
+    lines.push('🟢 QUICK WINS (Target First!)');
+    lines.push('─'.repeat(70));
+    lines.push('  These keywords have low competition and you can rank quickly.');
+    lines.push('');
+
+    const table = formatKeywordTable(result.quickWins.slice(0, 10));
+    lines.push(table);
+    lines.push('');
+  }
+
+  // Medium term
+  if (result.mediumTerm.length > 0) {
+    lines.push('🟡 MEDIUM TERM OPPORTUNITIES');
+    lines.push('─'.repeat(70));
+    lines.push('  Build some authority first, then target these keywords.');
+    lines.push('');
+
+    const table = formatKeywordTable(result.mediumTerm.slice(0, 5));
+    lines.push(table);
+    lines.push('');
+  }
+
+  // Long term
+  if (result.longTerm.length > 0) {
+    lines.push('🔴 LONG TERM (High Competition)');
+    lines.push('─'.repeat(70));
+    lines.push('  Save these for when you have more domain authority.');
+    lines.push('');
+
+    const table = formatKeywordTable(result.longTerm.slice(0, 3));
+    lines.push(table);
+    lines.push('');
+  }
+
+  // Recommendations
+  if (result.recommendations.length > 0) {
+    lines.push('💡 RECOMMENDATIONS');
+    lines.push('─'.repeat(70));
+    for (const rec of result.recommendations) {
+      lines.push(`  • ${rec}`);
+    }
+    lines.push('');
+  }
+
+  // Top actions
+  const topActions = result.quickWins
+    .slice(0, 3)
+    .filter((kw) => kw.suggestedAction.suggestedValue);
+
+  if (topActions.length > 0) {
+    lines.push('🎯 SUGGESTED CHANGES');
+    lines.push('─'.repeat(70));
+
+    for (const kw of topActions) {
+      lines.push(`  Keyword: "${kw.keyword}"`);
+      lines.push(`  Action:  ${kw.suggestedAction.description}`);
+      if (kw.suggestedAction.currentValue) {
+        lines.push(`  Current: ${kw.suggestedAction.currentValue}`);
+      }
+      if (kw.suggestedAction.suggestedValue) {
+        lines.push(`  Suggest: ${kw.suggestedAction.suggestedValue}`);
+      }
+      lines.push('');
+    }
+  }
+
+  lines.push('═'.repeat(70));
+
+  return lines.join('\n');
+}
+
+function formatKeywordTable(keywords: KeywordData[]): string {
+  const lines: string[] = [];
+
+  // Header
+  lines.push('  Keyword                           Vol    KD   Score  Action');
+  lines.push('  ' + '─'.repeat(65));
+
+  for (const kw of keywords) {
+    const keyword = kw.keyword.substring(0, 30).padEnd(32);
+    const volume = String(kw.searchVolume).padStart(6);
+    const kd = String(kw.keywordDifficulty).padStart(4);
+    const score = 'priorityScore' in kw ? String((kw as any).priorityScore).padStart(5) : '  -  ';
+    const action =
+      'suggestedAction' in kw
+        ? (kw as any).suggestedAction.type.substring(0, 15)
+        : '';
+
+    lines.push(`  ${keyword} ${volume} ${kd} ${score}  ${action}`);
+  }
+
+  return lines.join('\n');
+}
+
+// Extract seed keywords from page content
+export async function extractSeedKeywords(url: string): Promise<string[]> {
+  try {
+    const response = await httpGet<string>(url, {
+      
+      timeout: 10000,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const seeds: Set<string> = new Set();
+
+    // From title
+    const title = $('title').text().toLowerCase();
+    const titleWords = title.split(/[\s\-|:]+/).filter((w) => w.length > 3);
+    titleWords.forEach((w) => seeds.add(w));
+
+    // From H1
+    const h1 = $('h1').first().text().toLowerCase();
+    const h1Words = h1.split(/[\s\-|:]+/).filter((w) => w.length > 3);
+    h1Words.forEach((w) => seeds.add(w));
+
+    // From meta keywords (if present)
+    const metaKeywords = $('meta[name="keywords"]').attr('content');
+    if (metaKeywords) {
+      metaKeywords.split(',').forEach((k) => seeds.add(k.trim().toLowerCase()));
+    }
+
+    // From description
+    const desc = $('meta[name="description"]').attr('content')?.toLowerCase() || '';
+    const descWords = desc.split(/\s+/).filter((w) => w.length > 5);
+    descWords.slice(0, 5).forEach((w) => seeds.add(w));
+
+    return Array.from(seeds).slice(0, 10);
+  } catch {
+    return [];
+  }
+}

package/src/keywords/index.ts

@@ -0,0 +1,151 @@
+// Keyword Research Module
+
+// Types
+export * from './types.js';
+
+// Main engine (legacy)
+export {
+  runKeywordResearch,
+  formatKeywordReport,
+  extractSeedKeywords,
+  type KeywordResearchOptions,
+  type PageMeta,
+} from './engine.js';
+
+// AI-Powered Keyword Research (new)
+export {
+  runAIKeywordResearch,
+  type AIKeywordResearchOptions,
+  type PlanTier,
+  type FreeKeywordResult,
+  type PaidKeywordResult,
+  type KeywordOpportunity,
+  type FreeToolIdea,
+  type CompetitiveInsight,
+  type UncertaintyAssessment,
+  type KeywordRecommendation,
+} from './ai-keyword-engine.js';
+
+// Site Crawler
+export {
+  crawlSite,
+  extractKeyPhrases,
+  type CrawledPage,
+  type SiteCrawlResult,
+} from './site-crawler.js';
+
+// AI Summarizer
+export {
+  summarizeSite,
+  createFallbackSummary,
+  enhanceSummaryWithCompetitors,
+  generateUncertaintyQuestions,
+  type SiteSummary,
+  type SummarizerOptions,
+} from './ai-summarizer.js';
+
+// NLP Analysis
+export {
+  runNLPAnalysis,
+  calculateTFIDF as calculateKeywordTFIDF,
+  extractNgrams,
+  calculateBM25,
+  clusterKeywordsByEmbedding,
+  extractEntityPhrases,
+  extractTopics,
+  tokenize,
+  type TFIDFResult,
+  type NGram,
+  type KeywordCluster,
+  type TopicModel,
+  type NLPAnalysisResult,
+} from './nlp-analysis.js';
+
+// Wizard
+export {
+  startWizardSession,
+  generateWizardQuestions,
+  processWizardResponse,
+  getNextQuestion,
+  completeWizard,
+  wizardResponsesToContext,
+  formatWizardProgress,
+  type WizardQuestion,
+  type WizardResponse,
+  type WizardSession,
+  type WizardResult,
+} from './wizard.js';
+
+// CI Mode
+export {
+  runCIKeywordResearch,
+  formatCIResult,
+  type CIKeywordOptions,
+  type CIKeywordResult,
+  type CIAction,
+} from './ci-mode.js';
+
+// Prioritization
+export { prioritizeKeywords } from './prioritizer.js';
+
+// Data sources
+export {
+  getAutocompleteSuggestions,
+  getExpandedSuggestions,
+  enrichKeywordsWithEstimates,
+} from './sources/autocomplete.js';
+
+export {
+  transformGSCData,
+  findCTROpportunities,
+  findAlmostPage1Keywords,
+  findTopPerformers,
+  buildGSCRequest,
+  getDateRange,
+} from './sources/gsc.js';
+
+export {
+  getKeywordData,
+  getKeywordSuggestions,
+  getRelatedKeywords,
+  checkBalance,
+  LOCATION_CODES,
+  type DataForSEOCredentials,
+} from './sources/dataforseo.js';
+
+// Competitor Analysis (SpyFu Kombat-style)
+export {
+  discoverCompetitorKeywords,
+  formatCompetitorReport,
+  type CompetitorKeywordResult,
+  type CompetitorOverlap,
+} from './sources/competitor-analysis.js';
+
+// Topic Grouping/Clustering
+export {
+  groupKeywordsByTopic,
+  formatTopicReport,
+  type KeywordTopic,
+  type TopicClusterResult,
+  type ContentRecommendation,
+} from './topic-grouping.js';
+
+// Competitive Search (free APIs: Brave, Serper, GitHub, npm)
+export {
+  searchCompetitors,
+  searchFormatConverters,
+  searchHackerNews,
+  type CompetitiveSearchResult,
+  type CompetitorTool,
+  type CompetitiveSearchOptions,
+} from './sources/competitive-search.js';
+
+// LLM Judge for Tool Feasibility
+export {
+  evaluateToolFeasibility,
+  enhanceToolIdea,
+  evaluateAndEnhanceToolIdeas,
+  type ToolFeasibilityScore,
+  type EnhancedToolIdea,
+  type LLMJudgeOptions,
+} from './llm-judge.js';