@rankcli/agent-runtime 0.0.15 → 0.0.16

package/src/keywords/ai-summarizer.ts

@@ -1,387 +0,0 @@
-/**
- * AI-Powered Site Summarizer
- *
- * Uses OpenAI to understand what a site does from crawled content.
- * Extracts: product, audience, features, industry, positioning.
- */
-
-import OpenAI from 'openai';
-import type { SiteCrawlResult } from './site-crawler.js';
-
-export interface SiteSummary {
-  /** Primary product/service name */
-  productName: string;
-  /** What the product does (1-2 sentences) */
-  productDescription: string;
-  /** Target audience description */
-  targetAudience: string;
-  /** Key features/benefits (3-7 items) */
-  keyFeatures: string[];
-  /** Industry/category */
-  industry: string;
-  /** Business model */
-  businessModel: 'saas' | 'ecommerce' | 'marketplace' | 'agency' | 'media' | 'other';
-  /** Unique value proposition */
-  valueProposition: string;
-  /** Niche focus (if specific segment) */
-  nicheFocus?: string;
-  /** Pricing tier detected */
-  pricingTier?: 'free' | 'freemium' | 'paid' | 'enterprise' | 'unknown';
-  /** Geographic focus */
-  geographicFocus?: string;
-  /** Primary use cases */
-  useCases: string[];
-  /** Problems solved */
-  problemsSolved: string[];
-  /** Seed keyword suggestions */
-  suggestedSeedKeywords: string[];
-  /** Confidence score (0-1) */
-  confidence: number;
-  /** What we're uncertain about */
-  uncertainties: string[];
-}
-
-export interface SummarizerOptions {
-  openaiApiKey: string;
-  model?: string;
-  maxTokens?: number;
-}
-
-const SUMMARIZER_PROMPT = `You are an expert SEO analyst. Analyze the following website content and extract key information for keyword research.
-
-WEBSITE CONTENT:
-{content}
-
-HEADINGS FOUND:
-{headings}
-
-PAGE TYPES DETECTED:
-- Product pages: {productPages}
-- Pricing pages: {pricingPages}
-- Blog posts: {blogPages}
-- Feature pages: {featurePages}
-
-Analyze this content and respond with a JSON object containing:
-
-{
-  "productName": "The name of the product/service",
-  "productDescription": "What it does in 1-2 sentences",
-  "targetAudience": "Who this is for (be specific)",
-  "keyFeatures": ["feature1", "feature2", "..."],
-  "industry": "The industry/category (e.g., 'Marketing Automation', 'Project Management')",
-  "businessModel": "saas|ecommerce|marketplace|agency|media|other",
-  "valueProposition": "The unique value proposition",
-  "nicheFocus": "Specific niche if focused (e.g., 'small businesses', 'developers'), or null if broad",
-  "pricingTier": "free|freemium|paid|enterprise|unknown",
-  "geographicFocus": "Geographic focus if any, or null",
-  "useCases": ["use case 1", "use case 2", "..."],
-  "problemsSolved": ["problem 1", "problem 2", "..."],
-  "suggestedSeedKeywords": ["keyword1", "keyword2", "..."],
-  "confidence": 0.0 to 1.0,
-  "uncertainties": ["What we couldn't determine from the content"]
-}
-
-IMPORTANT:
-- suggestedSeedKeywords should include: product category keywords, feature keywords, problem keywords, and audience keywords
-- Be specific about the niche - don't say "businesses" if you can say "small e-commerce stores"
-- confidence should be low if the content is thin or unclear
-- List specific uncertainties that would need user input to resolve
-
-Respond ONLY with the JSON object, no other text.`;
-
-/**
- * Summarize a site using AI
- */
-export async function summarizeSite(
-  crawlResult: SiteCrawlResult,
-  options: SummarizerOptions
-): Promise<SiteSummary> {
-  const openai = new OpenAI({ apiKey: options.openaiApiKey });
-  const model = options.model || 'gpt-4o-mini';
-
-  // Prepare content (limit for token constraints)
-  let content = crawlResult.aggregatedContent;
-  if (content.length > 15000) {
-    content = content.substring(0, 15000) + '\n...[truncated]';
-  }
-
-  // Prepare headings
-  const headings = crawlResult.uniqueHeadings.slice(0, 30).join('\n');
-
-  // Build prompt
-  const prompt = SUMMARIZER_PROMPT
-    .replace('{content}', content)
-    .replace('{headings}', headings)
-    .replace('{productPages}', String(crawlResult.detectedPageTypes.product))
-    .replace('{pricingPages}', String(crawlResult.detectedPageTypes.pricing))
-    .replace('{blogPages}', String(crawlResult.detectedPageTypes.blog))
-    .replace('{featurePages}', String(crawlResult.detectedPageTypes.feature));
-
-  console.log('🤖 Analyzing site content with AI...');
-
-  try {
-    const response = await openai.chat.completions.create({
-      model,
-      messages: [
-        {
-          role: 'system',
-          content: 'You are an expert SEO analyst. Always respond with valid JSON only.',
-        },
-        {
-          role: 'user',
-          content: prompt,
-        },
-      ],
-      temperature: 0.3,
-      max_tokens: options.maxTokens || 2000,
-    });
-
-    const responseText = response.choices[0]?.message?.content || '';
-
-    // Parse JSON response
-    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
-    if (!jsonMatch) {
-      throw new Error('No JSON found in response');
-    }
-
-    const summary = JSON.parse(jsonMatch[0]) as SiteSummary;
-
-    // Validate and set defaults
-    return {
-      productName: summary.productName || 'Unknown Product',
-      productDescription: summary.productDescription || 'No description available',
-      targetAudience: summary.targetAudience || 'General audience',
-      keyFeatures: summary.keyFeatures || [],
-      industry: summary.industry || 'Unknown',
-      businessModel: summary.businessModel || 'other',
-      valueProposition: summary.valueProposition || '',
-      nicheFocus: summary.nicheFocus || undefined,
-      pricingTier: summary.pricingTier || 'unknown',
-      geographicFocus: summary.geographicFocus || undefined,
-      useCases: summary.useCases || [],
-      problemsSolved: summary.problemsSolved || [],
-      suggestedSeedKeywords: summary.suggestedSeedKeywords || [],
-      confidence: Math.min(1, Math.max(0, summary.confidence || 0.5)),
-      uncertainties: summary.uncertainties || [],
-    };
-  } catch (error) {
-    console.error('AI summarization failed:', error);
-
-    // Return fallback summary
-    return createFallbackSummary(crawlResult);
-  }
-}
-
-/**
- * Create a basic summary without AI (fallback)
- */
-export function createFallbackSummary(crawlResult: SiteCrawlResult): SiteSummary {
-  // Extract from page titles
-  const titles = crawlResult.pages.map((p) => p.title).filter(Boolean);
-  const productName = titles[0]?.split(/[|\-–—:]/)[0]?.trim() || crawlResult.domain;
-
-  // Extract key phrases from headings
-  const keyFeatures = crawlResult.uniqueHeadings
-    .filter((h) => h.length > 5 && h.length < 60)
-    .slice(0, 7);
-
-  // Extract potential seed keywords from titles and headings
-  const allText = [...titles, ...crawlResult.uniqueHeadings].join(' ').toLowerCase();
-  const words = allText.split(/\s+/).filter((w) => w.length > 4);
-  const wordFreq = new Map<string, number>();
-  for (const word of words) {
-    wordFreq.set(word, (wordFreq.get(word) || 0) + 1);
-  }
-  const suggestedSeedKeywords = Array.from(wordFreq.entries())
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 10)
-    .map(([word]) => word);
-
-  // Detect business model from page types
-  let businessModel: SiteSummary['businessModel'] = 'other';
-  if (crawlResult.detectedPageTypes.pricing > 0) {
-    businessModel = 'saas';
-  } else if (crawlResult.detectedPageTypes.product > 2) {
-    businessModel = 'ecommerce';
-  }
-
-  return {
-    productName,
-    productDescription: `Website at ${crawlResult.domain}`,
-    targetAudience: 'Unknown - requires AI analysis or user input',
-    keyFeatures,
-    industry: 'Unknown',
-    businessModel,
-    valueProposition: '',
-    useCases: [],
-    problemsSolved: [],
-    suggestedSeedKeywords,
-    confidence: 0.2, // Low confidence for fallback
-    uncertainties: [
-      'Target audience could not be determined',
-      'Industry classification is uncertain',
-      'Value proposition is unclear',
-      'Use cases need user input',
-    ],
-  };
-}
-
-/**
- * Enhance summary with competitive context
- */
-export async function enhanceSummaryWithCompetitors(
-  summary: SiteSummary,
-  competitors: string[],
-  options: SummarizerOptions
-): Promise<SiteSummary & { competitivePositioning: string }> {
-  const openai = new OpenAI({ apiKey: options.openaiApiKey });
-
-  const prompt = `Given this product summary:
-Product: ${summary.productName}
-Description: ${summary.productDescription}
-Industry: ${summary.industry}
-Key Features: ${summary.keyFeatures.join(', ')}
-
-And these competitors: ${competitors.join(', ')}
-
-Analyze the competitive positioning. If this product focuses on a niche segment while competitors are broader, identify that advantage. Respond with JSON:
-
-{
-  "competitivePositioning": "Description of how this product is positioned vs competitors",
-  "nicheAdvantages": ["advantage1", "advantage2"],
-  "competitorKeywordsToTarget": ["keyword1", "keyword2"],
-  "competitorKeywordsToAvoid": ["keyword1", "keyword2"],
-  "updatedUncertainties": ["uncertainty1", "uncertainty2"]
-}`;
-
-  try {
-    const response = await openai.chat.completions.create({
-      model: options.model || 'gpt-4o-mini',
-      messages: [
-        { role: 'system', content: 'Respond with valid JSON only.' },
-        { role: 'user', content: prompt },
-      ],
-      temperature: 0.3,
-      max_tokens: 1000,
-    });
-
-    const responseText = response.choices[0]?.message?.content || '';
-    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
-    if (jsonMatch) {
-      const analysis = JSON.parse(jsonMatch[0]);
-      return {
-        ...summary,
-        competitivePositioning: analysis.competitivePositioning || '',
-        suggestedSeedKeywords: [
-          ...summary.suggestedSeedKeywords,
-          ...(analysis.competitorKeywordsToTarget || []),
-        ],
-        uncertainties: [
-          ...summary.uncertainties.filter(
-            (u) => !analysis.updatedUncertainties?.includes(u)
-          ),
-          ...(analysis.updatedUncertainties || []),
-        ],
-      };
-    }
-  } catch (error) {
-    console.error('Competitive analysis failed:', error);
-  }
-
-  return { ...summary, competitivePositioning: '' };
-}
-
-/**
- * Generate questions to reduce uncertainty
- */
-export function generateUncertaintyQuestions(
-  summary: SiteSummary
-): Array<{
-  id: string;
-  question: string;
-  category: string;
-  impact: number;
-  options?: string[];
-}> {
-  const questions: Array<{
-    id: string;
-    question: string;
-    category: string;
-    impact: number;
-    options?: string[];
-  }> = [];
-
-  // Always ask about target audience if uncertain
-  if (summary.confidence < 0.7 || summary.targetAudience.includes('Unknown')) {
-    questions.push({
-      id: 'target_audience',
-      question: 'Who is your primary target customer?',
-      category: 'audience',
-      impact: 0.3,
-      options: [
-        'Small businesses (1-50 employees)',
-        'Mid-market companies (50-500 employees)',
-        'Enterprise (500+ employees)',
-        'Developers/Technical users',
-        'Marketing teams',
-        'Individual consumers',
-        'Freelancers/Agencies',
-      ],
-    });
-  }
-
-  // Ask about industry if uncertain
-  if (summary.industry === 'Unknown' || summary.confidence < 0.6) {
-    questions.push({
-      id: 'industry',
-      question: 'What industry or category best describes your product?',
-      category: 'product',
-      impact: 0.25,
-    });
-  }
-
-  // Ask about main problem solved
-  if (summary.problemsSolved.length === 0) {
-    questions.push({
-      id: 'main_problem',
-      question: 'What is the main problem your product solves?',
-      category: 'product',
-      impact: 0.35,
-    });
-  }
-
-  // Ask about competitors
-  questions.push({
-    id: 'competitors',
-    question: 'Who are your main competitors? (comma-separated)',
-    category: 'competition',
-    impact: 0.2,
-  });
-
-  // Ask about differentiator
-  if (!summary.nicheFocus) {
-    questions.push({
-      id: 'differentiator',
-      question: 'What makes your product different from competitors?',
-      category: 'positioning',
-      impact: 0.25,
-    });
-  }
-
-  // Ask about goals
-  questions.push({
-    id: 'seo_goals',
-    question: 'What are your SEO goals?',
-    category: 'goals',
-    impact: 0.15,
-    options: [
-        'Get more signups/trials',
-      'Generate leads',
-      'Build brand awareness',
-      'Drive organic traffic',
-      'Rank for specific keywords',
-    ],
-  });
-
-  return questions.sort((a, b) => b.impact - a.impact);
-}