@rankcli/agent-runtime 0.0.15 → 0.0.17

package/src/keywords/ai-keyword-engine.ts

@@ -1,1062 +0,0 @@
-/**
- * AI-Powered Keyword Research Engine
- *
- * Tiered access:
- * - FREE: Basic keyword extraction + score teaser
- * - PAID: Full NLP, clustering, embeddings, competitive analysis
- *
- * Based on Bhanu's "Engineering as Marketing" strategy:
- * 1. Find low KD keywords (< 10)
- * 2. High volume (1000+)
- * 3. Relevant to product
- * 4. Easy to build free tools for
- */
-
-import OpenAI from 'openai';
-import { crawlSite, extractKeyPhrases, type SiteCrawlResult } from './site-crawler.js';
-import { summarizeSite, generateUncertaintyQuestions, type SiteSummary } from './ai-summarizer.js';
-import {
-  runNLPAnalysis,
-  calculateTFIDF,
-  extractNgrams,
-  tokenize,
-  type NLPAnalysisResult,
-} from './nlp-analysis.js';
-import { getExpandedSuggestions, enrichKeywordsWithEstimates } from './sources/autocomplete.js';
-import { getAllFreeKeywordIdeas, applyEstimates } from './sources/free-sources.js';
-import type { KeywordData, SiteProfile } from './types.js';
-import { prioritizeKeywords } from './prioritizer.js';
-import {
-  searchCompetitors,
-  searchFormatConverters,
-  searchHackerNews,
-  type CompetitiveSearchResult,
-  type CompetitorTool,
-  type CompetitiveSearchOptions,
-} from './sources/competitive-search.js';
-import {
-  evaluateAndEnhanceToolIdeas,
-  type EnhancedToolIdea,
-  type LLMJudgeOptions,
-} from './llm-judge.js';
-
-export type PlanTier = 'free' | 'solo' | 'pro' | 'agency';
-
-export interface AIKeywordResearchOptions {
-  url: string;
-  tier: PlanTier;
-  openaiApiKey?: string;
-  maxPages?: number;
-  userContext?: {
-    productDescription?: string;
-    targetAudience?: string;
-    competitors?: string[];
-    mainProblem?: string;
-    differentiator?: string;
-  };
-  wizardResponses?: Array<{ id: string; answer: string }>;
-  ciMode?: boolean;
-  /** API keys for competitive search (optional but recommended) */
-  competitiveSearchKeys?: {
-    braveApiKey?: string;
-    serperApiKey?: string;
-    githubToken?: string;
-  };
-  /** Enable enhanced tool ideas with LLM judge (default: true for paid tiers) */
-  enableEnhancedToolIdeas?: boolean;
-}
-
-export interface FreeKeywordResult {
-  tier: 'free';
-  /** Basic keyword suggestions (limited) */
-  keywords: Array<{
-    keyword: string;
-    estimatedVolume: 'low' | 'medium' | 'high';
-    estimatedDifficulty: 'low' | 'medium' | 'high';
-    relevance: number;
-  }>;
-  /** Overall keyword opportunity score (teaser) */
-  opportunityScore: number;
-  /** Teaser message */
-  teaserMessage: string;
-  /** What they'd get with upgrade */
-  upgradeFeatures: string[];
-  /** Basic site understanding */
-  siteCategory: string;
-  /** Detected keywords count */
-  totalKeywordsFound: number;
-}
-
-export interface PaidKeywordResult {
-  tier: PlanTier;
-  /** Site summary from AI */
-  siteSummary: SiteSummary;
-  /** Full keyword opportunities */
-  opportunities: KeywordOpportunity[];
-  /** NLP analysis results */
-  nlpAnalysis: NLPAnalysisResult;
-  /** Keyword clusters */
-  clusters: Array<{
-    name: string;
-    keywords: string[];
-    totalVolume: number;
-    avgDifficulty: number;
-  }>;
-  /** Recommended free tools to build */
-  freeToolIdeas: FreeToolIdea[];
-  /** Competitive positioning */
-  competitiveInsights?: CompetitiveInsight[];
-  /** Uncertainty assessment */
-  uncertainty: UncertaintyAssessment;
-  /** Prioritized recommendations */
-  recommendations: KeywordRecommendation[];
-}
-
-export interface KeywordOpportunity {
-  keyword: string;
-  volume: number;
-  difficulty: number;
-  relevance: number;
-  opportunityScore: number;
-  intent: 'informational' | 'commercial' | 'transactional' | 'navigational';
-  suggestedContentType: 'free-tool' | 'landing-page' | 'blog-post' | 'comparison' | 'guide';
-  suggestedCTA: string;
-  rationale: string;
-  cluster?: string;
-}
-
-export interface FreeToolIdea {
-  keyword: string;
-  toolName: string;
-  toolDescription: string;
-  estimatedEffort: 'low' | 'medium' | 'high';
-  ctaConnection: string;
-  volume: number;
-  difficulty: number;
-  priority: number;
-  /** Enhanced fields from LLM judge */
-  inputFormat?: string;
-  outputFormat?: string;
-  productTieIn?: string;
-  ctaText?: string;
-  deepLinkPattern?: string;
-  competitors?: {
-    hasFreeAlternative: boolean;
-    topCompetitor?: string;
-    ourAdvantage: string;
-  };
-  implementationHints?: {
-    suggestedLibraries: string[];
-    estimatedComplexity: 'trivial' | 'simple' | 'moderate' | 'complex';
-    canUseExistingOSS: boolean;
-    ossLibrary?: string;
-  };
-  feasibilityScore?: {
-    feasibility: number;
-    specificity: number;
-    productFit: number;
-    marketOpportunity: number;
-    overallScore: number;
-    passes: boolean;
-    reasoning: string;
-  };
-}
-
-export interface CompetitiveInsight {
-  competitor: string;
-  theirFocus: 'broad' | 'niche';
-  ourAdvantage?: string;
-  keywordsTheyRankFor: string[];
-  keywordsWeCanTarget: string[];
-}
-
-export interface UncertaintyAssessment {
-  overallConfidence: number;
-  siteUnderstanding: number;
-  audienceClarity: number;
-  competitiveClarity: number;
-  canAutomate: boolean;
-  questionsToAsk: Array<{
-    id: string;
-    question: string;
-    impact: number;
-    options?: string[];
-  }>;
-  explanation: string;
-}
-
-export interface KeywordRecommendation {
-  priority: number;
-  keyword: string;
-  action: 'build-tool' | 'create-page' | 'write-content' | 'skip' | 'needs-input';
-  effort: 'low' | 'medium' | 'high';
-  impact: 'low' | 'medium' | 'high';
-  rationale: string;
-}
-
-/**
- * Run keyword research based on tier
- */
-export async function runAIKeywordResearch(
-  options: AIKeywordResearchOptions
-): Promise<FreeKeywordResult | PaidKeywordResult> {
-  const { url, tier, openaiApiKey, maxPages = 20 } = options;
-
-  console.log(`\n🔬 Starting AI Keyword Research (${tier} tier)...\n`);
-
-  // Step 1: Crawl the site
-  const crawlResult = await crawlSite(url, { maxPages });
-
-  // Route based on tier
-  if (tier === 'free') {
-    return runFreeKeywordResearch(crawlResult, options);
-  } else {
-    if (!openaiApiKey) {
-      throw new Error('OpenAI API key required for paid tier keyword research');
-    }
-    return runPaidKeywordResearch(crawlResult, options);
-  }
-}
-
-/**
- * FREE TIER: Basic keyword extraction with teaser
- */
-async function runFreeKeywordResearch(
-  crawlResult: SiteCrawlResult,
-  options: AIKeywordResearchOptions
-): Promise<FreeKeywordResult> {
-  console.log('📊 Running free tier analysis...');
-
-  // Basic TF-IDF analysis (no AI)
-  const documents = crawlResult.pages.map((p) => p.mainContent);
-  const tfidfResults = calculateTFIDF(documents);
-
-  // Extract key phrases
-  const keyPhrases = extractKeyPhrases(crawlResult);
-
-  // Get basic autocomplete suggestions for top terms
-  const topTerms = tfidfResults.slice(0, 5).map((r) => r.term);
-  let autocompleteKeywords: KeywordData[] = [];
-
-  for (const term of topTerms) {
-    try {
-      const suggestions = await getExpandedSuggestions(term, {});
-      autocompleteKeywords.push(...suggestions.slice(0, 5));
-    } catch {
-      // Ignore errors
-    }
-  }
-
-  // Apply volume estimates
-  autocompleteKeywords = applyEstimates(enrichKeywordsWithEstimates(autocompleteKeywords));
-
-  // Create limited results (teaser)
-  const limitedKeywords = autocompleteKeywords.slice(0, 10).map((kw) => {
-    const estimatedVolume: 'low' | 'medium' | 'high' = kw.searchVolume > 5000 ? 'high' : kw.searchVolume > 1000 ? 'medium' : 'low';
-    const estimatedDifficulty: 'low' | 'medium' | 'high' = kw.keywordDifficulty > 50 ? 'high' : kw.keywordDifficulty > 20 ? 'medium' : 'low';
-    return {
-      keyword: kw.keyword,
-      estimatedVolume,
-      estimatedDifficulty,
-      relevance: 0.5 + Math.random() * 0.5, // Simplified relevance
-    };
-  });
-
-  // Calculate opportunity score
-  const opportunityScore = Math.min(
-    100,
-    Math.round(
-      (limitedKeywords.filter((k) => k.estimatedDifficulty === 'low').length / limitedKeywords.length) * 50 +
-        (limitedKeywords.filter((k) => k.estimatedVolume !== 'low').length / limitedKeywords.length) * 50
-    )
-  );
-
-  // Detect site category
-  const siteCategory = detectSiteCategory(crawlResult, tfidfResults);
-
-  return {
-    tier: 'free',
-    keywords: limitedKeywords,
-    opportunityScore,
-    totalKeywordsFound: autocompleteKeywords.length + keyPhrases.length,
-    siteCategory,
-    teaserMessage: `We found ${autocompleteKeywords.length + keyPhrases.length} potential keywords for ${crawlResult.domain}. Upgrade to see full analysis with AI-powered insights, keyword clustering, and competitive analysis.`,
-    upgradeFeatures: [
-      '🧠 AI-powered site analysis',
-      '📊 Full keyword data (volume, difficulty, trends)',
-      '🎯 Semantic keyword clustering',
-      '🛠️ Free tool ideas with CTA suggestions',
-      '🏆 Competitive positioning analysis',
-      '📈 Prioritized action recommendations',
-      '🤖 Uncertainty wizard for better results',
-    ],
-  };
-}
-
-/**
- * PAID TIER: Full AI-powered analysis
- */
-async function runPaidKeywordResearch(
-  crawlResult: SiteCrawlResult,
-  options: AIKeywordResearchOptions
-): Promise<PaidKeywordResult> {
-  const { openaiApiKey, userContext, wizardResponses, ciMode, competitiveSearchKeys } = options;
-  const enableEnhancedToolIdeas = options.enableEnhancedToolIdeas !== false; // Default true
-
-  console.log('🚀 Running full AI analysis...');
-
-  // Step 1: AI Site Summary
-  let siteSummary = await summarizeSite(crawlResult, {
-    openaiApiKey: openaiApiKey!,
-  });
-
-  // Apply user context to improve summary
-  if (userContext) {
-    siteSummary = applyUserContext(siteSummary, userContext);
-  }
-
-  // Apply wizard responses
-  if (wizardResponses) {
-    siteSummary = applyWizardResponses(siteSummary, wizardResponses);
-  }
-
-  // Step 2: Full NLP Analysis
-  const documents = crawlResult.pages.map((p) => p.mainContent);
-  const nlpAnalysis = await runNLPAnalysis(documents, {
-    openaiApiKey,
-    numClusters: 8,
-    numTopics: 6,
-  });
-
-  // Step 3: Generate seed keywords from multiple sources
-  const seedKeywords = generateSeedKeywords(siteSummary, nlpAnalysis, crawlResult);
-
-  // Step 4: Expand keywords with autocomplete and volume data
-  console.log('🔍 Expanding keyword opportunities...');
-  let allKeywords: KeywordData[] = [];
-
-  for (const seed of seedKeywords.slice(0, 15)) {
-    try {
-      const suggestions = await getExpandedSuggestions(seed, {});
-      allKeywords.push(...suggestions.slice(0, 20));
-
-      const freeIdeas = await getAllFreeKeywordIdeas(seed, {
-        includeQuestions: true,
-        includeModifiers: true,
-      });
-      allKeywords.push(...freeIdeas);
-    } catch {
-      // Continue with other seeds
-    }
-  }
-
-  // Enrich with volume estimates
-  allKeywords = applyEstimates(enrichKeywordsWithEstimates(allKeywords));
-
-  // Deduplicate
-  const uniqueKeywords = deduplicateKeywords(allKeywords);
-
-  // Step 5: Score and prioritize
-  const opportunities = scoreKeywordOpportunities(
-    uniqueKeywords,
-    siteSummary,
-    nlpAnalysis
-  );
-
-  // Step 6: Generate free tool ideas with competitive research and LLM judge
-  let freeToolIdeas: FreeToolIdea[];
-
-  if (enableEnhancedToolIdeas && openaiApiKey) {
-    console.log('🔬 Running enhanced tool idea analysis with competitive research...');
-    freeToolIdeas = await generateEnhancedFreeToolIdeas(
-      opportunities,
-      siteSummary,
-      {
-        openaiApiKey,
-        ...competitiveSearchKeys,
-      }
-    );
-  } else {
-    // Fallback to basic tool idea generation
-    freeToolIdeas = generateBasicFreeToolIdeas(opportunities, siteSummary);
-  }
-
-  // Step 7: Create keyword clusters
-  const clusters = createKeywordClusters(opportunities, nlpAnalysis);
-
-  // Step 8: Assess uncertainty
-  const uncertainty = assessUncertainty(siteSummary, opportunities, ciMode);
-
-  // Step 9: Generate recommendations
-  const recommendations = generateRecommendations(
-    opportunities,
-    freeToolIdeas,
-    uncertainty
-  );
-
-  return {
-    tier: options.tier,
-    siteSummary,
-    opportunities: opportunities.slice(0, 100),
-    nlpAnalysis,
-    clusters,
-    freeToolIdeas: freeToolIdeas.slice(0, 20),
-    uncertainty,
-    recommendations: recommendations.slice(0, 15),
-  };
-}
-
-/**
- * Generate seed keywords from AI analysis
- */
-function generateSeedKeywords(
-  summary: SiteSummary,
-  nlp: NLPAnalysisResult,
-  crawl: SiteCrawlResult
-): string[] {
-  const seeds = new Set<string>();
-
-  // From AI summary
-  summary.suggestedSeedKeywords.forEach((k) => seeds.add(k.toLowerCase()));
-
-  // From key features
-  summary.keyFeatures.forEach((f) => {
-    const words = tokenize(f);
-    if (words.length <= 3) {
-      seeds.add(words.join(' '));
-    }
-  });
-
-  // From problems solved
-  summary.problemsSolved.forEach((p) => {
-    const words = tokenize(p);
-    if (words.length <= 4) {
-      seeds.add(words.join(' '));
-    }
-  });
-
-  // From TF-IDF
-  nlp.tfidfKeywords.slice(0, 15).forEach((k) => seeds.add(k.term));
-
-  // From bigrams
-  nlp.ngrams.bigrams.slice(0, 10).forEach((g) => seeds.add(g.phrase));
-
-  // From entity phrases
-  nlp.entityPhrases.slice(0, 10).forEach((e) => seeds.add(e.toLowerCase()));
-
-  // Product name variations
-  const productName = summary.productName.toLowerCase();
-  seeds.add(productName);
-  seeds.add(`${productName} alternative`);
-  seeds.add(`${productName} vs`);
-  seeds.add(`best ${summary.industry.toLowerCase()}`);
-
-  // Industry + modifiers
-  const industry = summary.industry.toLowerCase();
-  seeds.add(`${industry} tool`);
-  seeds.add(`${industry} software`);
-  seeds.add(`free ${industry}`);
-  seeds.add(`${industry} generator`);
-  seeds.add(`${industry} ai`);
-
-  return Array.from(seeds).filter((s) => s.length > 2 && s.length < 50);
-}
-
-/**
- * Score keyword opportunities
- */
-function scoreKeywordOpportunities(
-  keywords: KeywordData[],
-  summary: SiteSummary,
-  nlp: NLPAnalysisResult
-): KeywordOpportunity[] {
-  const opportunities: KeywordOpportunity[] = [];
-
-  // Get relevance terms from summary
-  const relevanceTerms = new Set([
-    ...summary.suggestedSeedKeywords.map((k) => k.toLowerCase()),
-    ...summary.keyFeatures.flatMap(tokenize),
-    ...summary.problemsSolved.flatMap(tokenize),
-    summary.industry.toLowerCase(),
-    summary.productName.toLowerCase(),
-  ]);
-
-  for (const kw of keywords) {
-    // Calculate relevance
-    const kwTokens = tokenize(kw.keyword);
-    const relevance = kwTokens.filter((t) => relevanceTerms.has(t)).length / Math.max(kwTokens.length, 1);
-
-    // Skip very low relevance
-    if (relevance < 0.1 && kw.searchVolume < 500) continue;
-
-    // Calculate opportunity score
-    // Higher volume + lower difficulty + higher relevance = better opportunity
-    const volumeScore = Math.min(1, Math.log10(kw.searchVolume + 1) / 4);
-    const difficultyScore = 1 - kw.keywordDifficulty / 100;
-    const opportunityScore = Math.round(
-      (volumeScore * 0.3 + difficultyScore * 0.4 + relevance * 0.3) * 100
-    );
-
-    // Determine intent
-    const intent = determineIntent(kw.keyword);
-
-    // Suggest content type
-    const suggestedContentType = suggestContentType(kw.keyword, kw.keywordDifficulty, intent);
-
-    // Generate CTA
-    const suggestedCTA = generateCTA(kw.keyword, summary);
-
-    // Generate rationale
-    const rationale = generateRationale(kw, relevance, summary);
-
-    opportunities.push({
-      keyword: kw.keyword,
-      volume: kw.searchVolume,
-      difficulty: kw.keywordDifficulty,
-      relevance: Math.round(relevance * 100),
-      opportunityScore,
-      intent,
-      suggestedContentType,
-      suggestedCTA,
-      rationale,
-    });
-  }
-
-  // Sort by opportunity score
-  return opportunities.sort((a, b) => b.opportunityScore - a.opportunityScore);
-}
-
-/**
- * Generate free tool ideas with competitive research and LLM judge (enhanced version)
- */
-async function generateEnhancedFreeToolIdeas(
-  opportunities: KeywordOpportunity[],
-  summary: SiteSummary,
-  options: {
-    openaiApiKey: string;
-    braveApiKey?: string;
-    serperApiKey?: string;
-    githubToken?: string;
-  }
-): Promise<FreeToolIdea[]> {
-  // Filter for tool-worthy keywords (low difficulty, decent volume)
-  const toolKeywords = opportunities.filter(
-    (kw) =>
-      kw.difficulty < 30 && // Slightly more lenient
-      kw.volume >= 300 && // Slightly lower threshold
-      (kw.keyword.includes('generator') ||
-        kw.keyword.includes('tool') ||
-        kw.keyword.includes('maker') ||
-        kw.keyword.includes('builder') ||
-        kw.keyword.includes('converter') ||
-        kw.keyword.includes('calculator') ||
-        kw.keyword.includes('checker') ||
-        kw.keyword.includes('validator') ||
-        kw.keyword.includes('formatter') ||
-        kw.keyword.includes('free') ||
-        kw.intent === 'informational')
-  );
-
-  // Prepare basic tool ideas for enhancement
-  const basicIdeas = toolKeywords.slice(0, 15).map((kw) => ({
-    keyword: kw.keyword,
-    toolName: generateToolName(kw.keyword),
-    toolDescription: generateToolDescription(kw.keyword, summary),
-    volume: kw.volume,
-    difficulty: kw.difficulty,
-  }));
-
-  // Create competitive search function with caching
-  const competitiveCache = new Map<string, CompetitiveSearchResult>();
-
-  const getCompetitiveData = async (keyword: string): Promise<CompetitiveSearchResult | null> => {
-    // Check cache
-    if (competitiveCache.has(keyword)) {
-      return competitiveCache.get(keyword)!;
-    }
-
-    try {
-      const result = await searchCompetitors(keyword, {
-        braveApiKey: options.braveApiKey,
-        serperApiKey: options.serperApiKey,
-        githubToken: options.githubToken,
-        maxResults: 5,
-      });
-      competitiveCache.set(keyword, result);
-      return result;
-    } catch (error) {
-      console.warn(`Competitive search failed for "${keyword}":`, error);
-      return null;
-    }
-  };
-
-  // Evaluate and enhance tool ideas using LLM judge
-  const enhancedIdeas = await evaluateAndEnhanceToolIdeas(
-    basicIdeas,
-    summary,
-    getCompetitiveData,
-    { openaiApiKey: options.openaiApiKey }
-  );
-
-  // Convert enhanced ideas to FreeToolIdea format
-  return enhancedIdeas.map((enhanced) => ({
-    keyword: enhanced.keyword,
-    toolName: enhanced.toolName,
-    toolDescription: enhanced.description,
-    estimatedEffort: complexityToEffort(enhanced.implementationHints.estimatedComplexity),
-    ctaConnection: enhanced.ctaText,
-    volume: enhanced.volume,
-    difficulty: enhanced.difficulty,
-    priority: enhanced.priority,
-    inputFormat: enhanced.inputFormat,
-    outputFormat: enhanced.outputFormat,
-    productTieIn: enhanced.productTieIn,
-    ctaText: enhanced.ctaText,
-    deepLinkPattern: enhanced.deepLinkPattern,
-    competitors: enhanced.competitors,
-    implementationHints: enhanced.implementationHints,
-    feasibilityScore: {
-      feasibility: enhanced.feasibilityScore.feasibility,
-      specificity: enhanced.feasibilityScore.specificity,
-      productFit: enhanced.feasibilityScore.productFit,
-      marketOpportunity: enhanced.feasibilityScore.marketOpportunity,
-      overallScore: enhanced.feasibilityScore.overallScore,
-      passes: enhanced.feasibilityScore.passes,
-      reasoning: enhanced.feasibilityScore.reasoning,
-    },
-  }));
-}
-
-/**
- * Convert complexity to effort level
- */
-function complexityToEffort(complexity: 'trivial' | 'simple' | 'moderate' | 'complex'): 'low' | 'medium' | 'high' {
-  switch (complexity) {
-    case 'trivial':
-    case 'simple':
-      return 'low';
-    case 'moderate':
-      return 'medium';
-    case 'complex':
-      return 'high';
-    default:
-      return 'medium';
-  }
-}
-
-/**
- * Generate free tool ideas (basic version - fallback)
- */
-function generateBasicFreeToolIdeas(
-  opportunities: KeywordOpportunity[],
-  summary: SiteSummary
-): FreeToolIdea[] {
-  const toolIdeas: FreeToolIdea[] = [];
-
-  // Filter for tool-worthy keywords (low difficulty, decent volume)
-  const toolKeywords = opportunities.filter(
-    (kw) =>
-      kw.difficulty < 20 &&
-      kw.volume >= 500 &&
-      (kw.keyword.includes('generator') ||
-        kw.keyword.includes('tool') ||
-        kw.keyword.includes('maker') ||
-        kw.keyword.includes('builder') ||
-        kw.keyword.includes('converter') ||
-        kw.keyword.includes('calculator') ||
-        kw.keyword.includes('checker') ||
-        kw.keyword.includes('free') ||
-        kw.intent === 'informational')
-  );
-
-  for (const kw of toolKeywords.slice(0, 30)) {
-    const toolName = generateToolName(kw.keyword);
-    const toolDescription = generateToolDescription(kw.keyword, summary);
-    const effort = estimateToolEffort(kw.keyword);
-    const ctaConnection = generateToolCTA(kw.keyword, summary);
-
-    // Calculate priority
-    const priority = Math.round(
-      (kw.volume / 10000) * 30 + // Volume contribution
-        ((100 - kw.difficulty) / 100) * 40 + // Difficulty contribution
-        (kw.relevance / 100) * 30 // Relevance contribution
-    );
-
-    toolIdeas.push({
-      keyword: kw.keyword,
-      toolName,
-      toolDescription,
-      estimatedEffort: effort,
-      ctaConnection,
-      volume: kw.volume,
-      difficulty: kw.difficulty,
-      priority,
-    });
-  }
-
-  return toolIdeas.sort((a, b) => b.priority - a.priority);
-}
-
-/**
- * Create keyword clusters
- */
-function createKeywordClusters(
-  opportunities: KeywordOpportunity[],
-  nlp: NLPAnalysisResult
-): Array<{
-  name: string;
-  keywords: string[];
-  totalVolume: number;
-  avgDifficulty: number;
-}> {
-  // Use NLP clusters if available
-  if (nlp.clusters.length > 0) {
-    return nlp.clusters.map((cluster) => {
-      const clusterOpps = opportunities.filter((o) =>
-        cluster.keywords.some(
-          (k) => o.keyword.includes(k) || k.includes(o.keyword.split(' ')[0])
-        )
-      );
-
-      return {
-        name: cluster.name,
-        keywords: clusterOpps.map((o) => o.keyword),
-        totalVolume: clusterOpps.reduce((sum, o) => sum + o.volume, 0),
-        avgDifficulty:
-          clusterOpps.length > 0
-            ? Math.round(clusterOpps.reduce((sum, o) => sum + o.difficulty, 0) / clusterOpps.length)
-            : 0,
-      };
-    });
-  }
-
-  // Fallback: group by common terms
-  const clusters: Map<string, KeywordOpportunity[]> = new Map();
-
-  for (const opp of opportunities) {
-    const mainTerm = opp.keyword.split(' ')[0];
-    if (!clusters.has(mainTerm)) {
-      clusters.set(mainTerm, []);
-    }
-    clusters.get(mainTerm)!.push(opp);
-  }
-
-  return Array.from(clusters.entries())
-    .filter(([_, opps]) => opps.length >= 2)
-    .map(([name, opps]) => ({
-      name,
-      keywords: opps.map((o) => o.keyword),
-      totalVolume: opps.reduce((sum, o) => sum + o.volume, 0),
-      avgDifficulty: Math.round(opps.reduce((sum, o) => sum + o.difficulty, 0) / opps.length),
-    }))
-    .sort((a, b) => b.totalVolume - a.totalVolume)
-    .slice(0, 10);
-}
-
-/**
- * Assess uncertainty for CI mode
- */
-function assessUncertainty(
-  summary: SiteSummary,
-  opportunities: KeywordOpportunity[],
-  ciMode?: boolean
-): UncertaintyAssessment {
-  const siteUnderstanding = summary.confidence;
-  const audienceClarity = summary.targetAudience.includes('Unknown') ? 0.3 : 0.8;
-  const competitiveClarity = 0.5; // Would be higher with competitor analysis
-
-  // Average relevance of top keywords
-  const topOpps = opportunities.slice(0, 20);
-  const avgRelevance = topOpps.length > 0
-    ? topOpps.reduce((sum, o) => sum + o.relevance, 0) / topOpps.length / 100
-    : 0.5;
-
-  const overallConfidence =
-    (siteUnderstanding + audienceClarity + competitiveClarity + avgRelevance) / 4;
-
-  // In CI mode, require higher confidence
-  const threshold = ciMode ? 0.7 : 0.5;
-  const canAutomate = overallConfidence >= threshold;
-
-  // Generate questions to reduce uncertainty
-  const questions = generateUncertaintyQuestions(summary);
-
-  let explanation: string;
-  if (canAutomate) {
-    explanation = `Confidence is ${Math.round(overallConfidence * 100)}%. Keyword recommendations can be automated.`;
-  } else {
-    explanation = `Confidence is only ${Math.round(overallConfidence * 100)}%. ${
-      ciMode
-        ? 'CI mode requires 70% confidence. Please run the keyword wizard to provide more context.'
-        : 'Consider running the keyword wizard for better results.'
-    }`;
-  }
-
-  return {
-    overallConfidence,
-    siteUnderstanding,
-    audienceClarity,
-    competitiveClarity,
-    canAutomate,
-    questionsToAsk: questions,
-    explanation,
-  };
-}
-
-/**
- * Generate prioritized recommendations
- */
-function generateRecommendations(
-  opportunities: KeywordOpportunity[],
-  toolIdeas: FreeToolIdea[],
-  uncertainty: UncertaintyAssessment
-): KeywordRecommendation[] {
-  const recommendations: KeywordRecommendation[] = [];
-  let priority = 1;
-
-  // If uncertainty is high, first recommendation is to run wizard
-  if (!uncertainty.canAutomate) {
-    recommendations.push({
-      priority: priority++,
-      keyword: '',
-      action: 'needs-input',
-      effort: 'low',
-      impact: 'high',
-      rationale: `Run the keyword wizard to answer ${uncertainty.questionsToAsk.length} questions and improve recommendation accuracy.`,
-    });
-  }
-
-  // Add top free tool ideas
-  for (const tool of toolIdeas.slice(0, 5)) {
-    recommendations.push({
-      priority: priority++,
-      keyword: tool.keyword,
-      action: 'build-tool',
-      effort: tool.estimatedEffort,
-      impact: tool.volume > 2000 ? 'high' : tool.volume > 500 ? 'medium' : 'low',
-      rationale: `Build "${tool.toolName}": ${tool.volume} monthly searches, KD ${tool.difficulty}. ${tool.ctaConnection}`,
-    });
-  }
-
-  // Add top content opportunities
-  const contentOpps = opportunities
-    .filter((o) => o.suggestedContentType !== 'free-tool' && o.difficulty < 30)
-    .slice(0, 5);
-
-  for (const opp of contentOpps) {
-    recommendations.push({
-      priority: priority++,
-      keyword: opp.keyword,
-      action: opp.suggestedContentType === 'landing-page' ? 'create-page' : 'write-content',
-      effort: opp.suggestedContentType === 'guide' ? 'high' : 'medium',
-      impact: opp.volume > 2000 ? 'high' : opp.volume > 500 ? 'medium' : 'low',
-      rationale: opp.rationale,
-    });
-  }
-
-  return recommendations;
-}
-
-// Helper functions
-
-function detectSiteCategory(
-  crawl: SiteCrawlResult,
-  tfidf: Array<{ term: string; tfidf: number }>
-): string {
-  const topTerms = tfidf.slice(0, 20).map((t) => t.term);
-  const text = topTerms.join(' ');
-
-  if (text.includes('saas') || text.includes('software') || text.includes('app')) {
-    return 'SaaS/Software';
-  }
-  if (text.includes('shop') || text.includes('buy') || text.includes('product')) {
-    return 'E-commerce';
-  }
-  if (text.includes('blog') || text.includes('article') || text.includes('news')) {
-    return 'Content/Media';
-  }
-  if (text.includes('agency') || text.includes('service') || text.includes('consult')) {
-    return 'Agency/Services';
-  }
-
-  return 'General';
-}
-
-function deduplicateKeywords(keywords: KeywordData[]): KeywordData[] {
-  const seen = new Map<string, KeywordData>();
-  for (const kw of keywords) {
-    const key = kw.keyword.toLowerCase().trim();
-    const existing = seen.get(key);
-    if (!existing || kw.searchVolume > existing.searchVolume) {
-      seen.set(key, kw);
-    }
-  }
-  return Array.from(seen.values());
-}
-
-function determineIntent(keyword: string): 'informational' | 'commercial' | 'transactional' | 'navigational' {
-  const kw = keyword.toLowerCase();
-  if (kw.includes('how to') || kw.includes('what is') || kw.includes('guide') || kw.includes('tutorial')) {
-    return 'informational';
-  }
-  if (kw.includes('buy') || kw.includes('price') || kw.includes('cost') || kw.includes('purchase')) {
-    return 'transactional';
-  }
-  if (kw.includes('best') || kw.includes('top') || kw.includes('review') || kw.includes('vs')) {
-    return 'commercial';
-  }
-  return 'informational';
-}
-
-function suggestContentType(
-  keyword: string,
-  difficulty: number,
-  intent: string
-): 'free-tool' | 'landing-page' | 'blog-post' | 'comparison' | 'guide' {
-  const kw = keyword.toLowerCase();
-
-  if (kw.includes('generator') || kw.includes('tool') || kw.includes('maker') || kw.includes('calculator')) {
-    return 'free-tool';
-  }
-  if (kw.includes('vs') || kw.includes('alternative') || kw.includes('comparison')) {
-    return 'comparison';
-  }
-  if (intent === 'commercial' && difficulty < 30) {
-    return 'landing-page';
-  }
-  if (kw.includes('guide') || kw.includes('how to') || kw.includes('tutorial')) {
-    return 'guide';
-  }
-  return 'blog-post';
-}
-
-function generateCTA(keyword: string, summary: SiteSummary): string {
-  const product = summary.productName;
-  const kw = keyword.toLowerCase();
-
-  if (kw.includes('generator') || kw.includes('tool')) {
-    return `Try ${product} for more advanced ${summary.industry.toLowerCase()} features →`;
-  }
-  if (kw.includes('free')) {
-    return `${product} offers a free tier with even more capabilities →`;
-  }
-  return `Discover how ${product} can help with ${kw.split(' ').slice(0, 3).join(' ')} →`;
-}
-
-function generateRationale(kw: KeywordData, relevance: number, summary: SiteSummary): string {
-  const parts: string[] = [];
-
-  if (kw.keywordDifficulty < 20) {
-    parts.push('Low competition - easy to rank');
-  } else if (kw.keywordDifficulty < 40) {
-    parts.push('Moderate competition');
-  }
-
-  if (kw.searchVolume > 5000) {
-    parts.push('High search volume');
-  } else if (kw.searchVolume > 1000) {
-    parts.push('Decent search volume');
-  }
-
-  if (relevance > 0.7) {
-    parts.push('Highly relevant to your product');
-  } else if (relevance > 0.4) {
-    parts.push('Related to your industry');
-  }
-
-  return parts.join('. ') + '.';
-}
-
-function generateToolName(keyword: string): string {
-  const kw = keyword.toLowerCase();
-  const words = kw.split(' ').map((w) => w.charAt(0).toUpperCase() + w.slice(1));
-
-  if (kw.includes('generator')) {
-    return words.join(' ');
-  }
-  if (kw.includes('checker') || kw.includes('validator')) {
-    return words.join(' ');
-  }
-
-  return `Free ${words.join(' ')} Tool`;
-}
-
-function generateToolDescription(keyword: string, summary: SiteSummary): string {
-  return `A free ${keyword} tool. Use it to ${keyword.replace('generator', 'generate').replace('checker', 'check')}. Try ${summary.productName} for more advanced features.`;
-}
-
-function estimateToolEffort(keyword: string): 'low' | 'medium' | 'high' {
-  const kw = keyword.toLowerCase();
-
-  if (kw.includes('calculator') || kw.includes('converter') || kw.includes('counter')) {
-    return 'low';
-  }
-  if (kw.includes('generator') || kw.includes('maker')) {
-    return 'medium';
-  }
-  if (kw.includes('analyzer') || kw.includes('checker') || kw.includes('builder')) {
-    return 'high';
-  }
-
-  return 'medium';
-}
-
-function generateToolCTA(keyword: string, summary: SiteSummary): string {
-  return `"Like this free tool? ${summary.productName} can do much more - ${summary.valueProposition || `try ${summary.productName} free`}"`;
-}
-
-function applyUserContext(summary: SiteSummary, context: AIKeywordResearchOptions['userContext']): SiteSummary {
-  if (!context) return summary;
-
-  return {
-    ...summary,
-    productDescription: context.productDescription || summary.productDescription,
-    targetAudience: context.targetAudience || summary.targetAudience,
-    problemsSolved: context.mainProblem
-      ? [context.mainProblem, ...summary.problemsSolved]
-      : summary.problemsSolved,
-    nicheFocus: context.differentiator || summary.nicheFocus,
-    confidence: Math.min(1, summary.confidence + 0.2), // Boost confidence with user input
-    uncertainties: summary.uncertainties.filter(
-      (u) =>
-        !(context.targetAudience && u.includes('audience')) &&
-        !(context.mainProblem && u.includes('problem'))
-    ),
-  };
-}
-
-function applyWizardResponses(
-  summary: SiteSummary,
-  responses: Array<{ id: string; answer: string }>
-): SiteSummary {
-  let updated = { ...summary };
-  let confidenceBoost = 0;
-
-  for (const response of responses) {
-    switch (response.id) {
-      case 'target_audience':
-        updated.targetAudience = response.answer;
-        confidenceBoost += 0.15;
-        break;
-      case 'industry':
-        updated.industry = response.answer;
-        confidenceBoost += 0.1;
-        break;
-      case 'main_problem':
-        updated.problemsSolved = [response.answer, ...updated.problemsSolved];
-        confidenceBoost += 0.15;
-        break;
-      case 'differentiator':
-        updated.nicheFocus = response.answer;
-        confidenceBoost += 0.1;
-        break;
-    }
-  }
-
-  updated.confidence = Math.min(1, updated.confidence + confidenceBoost);
-  updated.uncertainties = updated.uncertainties.filter(
-    (u) => !responses.some((r) => u.toLowerCase().includes(r.id.replace('_', ' ')))
-  );
-
-  return updated;
-}