@rankcli/agent-runtime 0.0.15 → 0.0.16

package/src/keywords/prioritizer.ts

@@ -1,295 +0,0 @@
-// Keyword Prioritization Algorithm
-
-import type {
-  SiteProfile,
-  KeywordData,
-  KeywordOpportunity,
-  KeywordAction,
-  KeywordResearchResult,
-} from './types.js';
-import { getMaxKdThreshold, PRIORITY_WEIGHTS } from './types.js';
-
-export function prioritizeKeywords(
-  keywords: KeywordData[],
-  siteProfile: SiteProfile,
-  existingMeta?: { title?: string; description?: string; h1?: string }
-): KeywordResearchResult {
-  const maxKd = getMaxKdThreshold(siteProfile);
-
-  // Score and categorize each keyword
-  const opportunities: KeywordOpportunity[] = keywords.map((kw) => {
-    const priorityScore = calculatePriorityScore(kw, siteProfile, maxKd);
-    const category = categorizeKeyword(kw.keywordDifficulty, maxKd);
-    const suggestedAction = suggestAction(kw, existingMeta, category);
-
-    return {
-      ...kw,
-      priorityScore,
-      category,
-      suggestedAction,
-    };
-  });
-
-  // Sort by priority score
-  opportunities.sort((a, b) => b.priorityScore - a.priorityScore);
-
-  // Categorize into buckets
-  const quickWins = opportunities.filter((kw) => kw.category === 'quick-win');
-  const mediumTerm = opportunities.filter((kw) => kw.category === 'medium-term');
-  const longTerm = opportunities.filter((kw) => kw.category === 'long-term');
-
-  // Generate recommendations
-  const recommendations = generateRecommendations(siteProfile, quickWins, mediumTerm);
-
-  return {
-    siteProfile,
-    keywords: opportunities,
-    quickWins,
-    mediumTerm,
-    longTerm,
-    recommendations,
-    maxKdThreshold: maxKd,
-  };
-}
-
-function calculatePriorityScore(
-  keyword: KeywordData,
-  profile: SiteProfile,
-  maxKd: number
-): number {
-  // Business value score (0-100)
-  const businessScore = calculateBusinessValue(keyword, profile.businessGoal);
-
-  // Difficulty score (inverse - lower KD = higher score)
-  let difficultyScore = 0;
-  if (keyword.keywordDifficulty <= maxKd) {
-    difficultyScore = 100 - (keyword.keywordDifficulty / maxKd) * 100;
-  }
-
-  // Traffic score (normalized, capped at 100)
-  const trafficScore = Math.min((keyword.searchVolume / 1000) * 100, 100);
-
-  // Weighted sum
-  const score =
-    businessScore * PRIORITY_WEIGHTS.businessValue +
-    difficultyScore * PRIORITY_WEIGHTS.difficulty +
-    trafficScore * PRIORITY_WEIGHTS.trafficPotential;
-
-  return Math.round(score);
-}
-
-function calculateBusinessValue(keyword: KeywordData, goal: SiteProfile['businessGoal']): number {
-  const intent = keyword.intent || inferIntent(keyword.keyword);
-
-  // Intent weights based on business goal
-  const intentWeights: Record<SiteProfile['businessGoal'], Record<string, number>> = {
-    signups: {
-      transactional: 100,
-      commercial: 80,
-      informational: 40,
-      navigational: 20,
-    },
-    purchases: {
-      transactional: 100,
-      commercial: 90,
-      informational: 30,
-      navigational: 20,
-    },
-    leads: {
-      commercial: 100,
-      transactional: 80,
-      informational: 60,
-      navigational: 20,
-    },
-    awareness: {
-      informational: 100,
-      commercial: 60,
-      navigational: 40,
-      transactional: 30,
-    },
-  };
-
-  return intentWeights[goal][intent] || 50;
-}
-
-function inferIntent(keyword: string): 'informational' | 'commercial' | 'transactional' | 'navigational' {
-  const kw = keyword.toLowerCase();
-
-  // Transactional indicators
-  if (/\b(buy|purchase|order|subscribe|download|get|try|free trial)\b/.test(kw)) {
-    return 'transactional';
-  }
-
-  // Commercial investigation
-  if (/\b(best|top|review|compare|vs|versus|alternative|pricing|cost)\b/.test(kw)) {
-    return 'commercial';
-  }
-
-  // Navigational
-  if (/\b(login|sign in|website|official|app)\b/.test(kw)) {
-    return 'navigational';
-  }
-
-  // Default to informational
-  return 'informational';
-}
-
-function categorizeKeyword(kd: number, maxKd: number): 'quick-win' | 'medium-term' | 'long-term' {
-  const quickWinThreshold = Math.min(maxKd * 0.5, 15);
-  const mediumThreshold = maxKd;
-
-  if (kd <= quickWinThreshold) return 'quick-win';
-  if (kd <= mediumThreshold) return 'medium-term';
-  return 'long-term';
-}
-
-function suggestAction(
-  keyword: KeywordData,
-  existingMeta: { title?: string; description?: string; h1?: string } | undefined,
-  category: 'quick-win' | 'medium-term' | 'long-term'
-): KeywordAction {
-  const kw = keyword.keyword;
-
-  // Check if keyword is already in existing meta
-  const inTitle = existingMeta?.title?.toLowerCase().includes(kw.toLowerCase());
-  const inDescription = existingMeta?.description?.toLowerCase().includes(kw.toLowerCase());
-  const inH1 = existingMeta?.h1?.toLowerCase().includes(kw.toLowerCase());
-
-  // Quick wins: optimize existing elements
-  if (category === 'quick-win') {
-    if (!inTitle && existingMeta?.title) {
-      return {
-        type: 'add-to-title',
-        description: `Add "${kw}" to your title tag`,
-        targetElement: 'title',
-        currentValue: existingMeta.title,
-        suggestedValue: generateTitleWithKeyword(existingMeta.title, kw),
-      };
-    }
-
-    if (!inH1) {
-      return {
-        type: 'add-to-h1',
-        description: `Include "${kw}" in your H1 heading`,
-        targetElement: 'h1',
-        currentValue: existingMeta?.h1,
-        suggestedValue: generateH1WithKeyword(existingMeta?.h1, kw),
-      };
-    }
-
-    if (!inDescription && existingMeta?.description) {
-      return {
-        type: 'add-to-meta',
-        description: `Add "${kw}" to your meta description`,
-        targetElement: 'meta[name="description"]',
-        currentValue: existingMeta.description,
-        suggestedValue: generateDescriptionWithKeyword(existingMeta.description, kw),
-      };
-    }
-
-    return {
-      type: 'optimize-existing',
-      description: `Optimize existing content for "${kw}"`,
-    };
-  }
-
-  // Medium-term: may need content creation
-  if (category === 'medium-term') {
-    return {
-      type: 'create-content',
-      description: `Create dedicated content targeting "${kw}"`,
-    };
-  }
-
-  // Long-term: need authority building first
-  return {
-    type: 'create-content',
-    description: `Build authority first, then target "${kw}" (high competition)`,
-  };
-}
-
-function generateTitleWithKeyword(currentTitle: string, keyword: string): string {
-  // If title is short, prepend keyword
-  if (currentTitle.length < 30) {
-    return `${capitalizeFirst(keyword)} - ${currentTitle}`;
-  }
-
-  // Otherwise, try to incorporate naturally
-  const parts = currentTitle.split(' - ');
-  if (parts.length >= 2) {
-    return `${capitalizeFirst(keyword)} | ${parts[parts.length - 1]}`;
-  }
-
-  return `${capitalizeFirst(keyword)} | ${currentTitle}`.substring(0, 60);
-}
-
-function generateH1WithKeyword(currentH1: string | undefined, keyword: string): string {
-  if (!currentH1) {
-    return capitalizeFirst(keyword);
-  }
-
-  // Try to incorporate keyword naturally
-  return `${capitalizeFirst(keyword)} - ${currentH1}`;
-}
-
-function generateDescriptionWithKeyword(currentDesc: string, keyword: string): string {
-  // If description doesn't start with keyword concept, prepend it
-  if (!currentDesc.toLowerCase().includes(keyword.toLowerCase().split(' ')[0])) {
-    const newDesc = `${capitalizeFirst(keyword)}: ${currentDesc}`;
-    return newDesc.substring(0, 160);
-  }
-
-  return currentDesc;
-}
-
-function capitalizeFirst(str: string): string {
-  return str.charAt(0).toUpperCase() + str.slice(1);
-}
-
-function generateRecommendations(
-  profile: SiteProfile,
-  quickWins: KeywordOpportunity[],
-  mediumTerm: KeywordOpportunity[]
-): string[] {
-  const recommendations: string[] = [];
-
-  // Domain age specific recommendations
-  if (profile.domainAge === 'new') {
-    recommendations.push(
-      'Focus on quick-win keywords (KD < 15) first to build initial traffic and authority.'
-    );
-    recommendations.push(
-      'Consider creating long-form, comprehensive content to establish topical authority.'
-    );
-  }
-
-  // Backlink recommendations
-  if (profile.backlinkCount === 'none' || profile.backlinkCount === 'few') {
-    recommendations.push(
-      'Prioritize building quality backlinks to increase your ranking potential for competitive keywords.'
-    );
-  }
-
-  // Content recommendations based on capacity
-  if (profile.contentCapacity === 'low' && quickWins.length > 2) {
-    recommendations.push(
-      `Focus on your top ${Math.min(2, quickWins.length)} quick-win keywords first, then expand.`
-    );
-  }
-
-  // Quick wins available
-  if (quickWins.length > 0) {
-    recommendations.push(
-      `You have ${quickWins.length} quick-win keyword opportunities that you can rank for relatively quickly.`
-    );
-  }
-
-  // Medium-term opportunities
-  if (mediumTerm.length > 0 && profile.domainAge !== 'new') {
-    recommendations.push(
-      `${mediumTerm.length} medium-difficulty keywords are within reach with focused content and some link building.`
-    );
-  }
-
-  return recommendations;
-}

package/src/keywords/site-crawler.ts

@@ -1,342 +0,0 @@
-/**
- * Site Crawler for Keyword Research
- *
- * Crawls an entire site to collect text content for AI analysis.
- * Used to understand what the site does and generate relevant keywords.
- */
-
-import * as cheerio from 'cheerio';
-import { httpGet } from '../utils/http.js';
-
-export interface CrawledPage {
-  url: string;
-  title: string;
-  description: string;
-  h1: string;
-  h2s: string[];
-  mainContent: string;
-  wordCount: number;
-  internalLinks: string[];
-  isProductPage: boolean;
-  isPricingPage: boolean;
-  isBlogPost: boolean;
-  isFeaturePage: boolean;
-}
-
-export interface SiteCrawlResult {
-  domain: string;
-  pages: CrawledPage[];
-  aggregatedContent: string;
-  totalWordCount: number;
-  uniqueHeadings: string[];
-  detectedPageTypes: {
-    product: number;
-    pricing: number;
-    blog: number;
-    feature: number;
-    other: number;
-  };
-  crawlStats: {
-    attempted: number;
-    succeeded: number;
-    failed: number;
-    duration: number;
-  };
-}
-
-const EXCLUDED_PATHS = [
-  '/cdn-cgi/',
-  '/wp-admin/',
-  '/wp-includes/',
-  '/wp-json/',
-  '/api/',
-  '/admin/',
-  '/_next/',
-  '/static/',
-  '/assets/',
-  '.xml',
-  '.json',
-  '.pdf',
-  '.zip',
-  '.png',
-  '.jpg',
-  '.jpeg',
-  '.gif',
-  '.svg',
-  '.ico',
-  '.css',
-  '.js',
-];
-
-const PAGE_TYPE_PATTERNS = {
-  product: ['/product', '/features', '/solutions', '/platform', '/tour'],
-  pricing: ['/pricing', '/plans', '/subscription'],
-  blog: ['/blog', '/news', '/articles', '/posts', '/journal'],
-  feature: ['/feature', '/capability', '/integration'],
-};
-
-/**
- * Crawl an entire site to collect text content
- */
-export async function crawlSite(
-  startUrl: string,
-  options: {
-    maxPages?: number;
-    maxDepth?: number;
-    timeout?: number;
-  } = {}
-): Promise<SiteCrawlResult> {
-  const { maxPages = 30, maxDepth = 3, timeout = 10000 } = options;
-
-  const startTime = Date.now();
-  const parsedUrl = new URL(startUrl);
-  const domain = parsedUrl.hostname;
-  const baseUrl = `${parsedUrl.protocol}//${domain}`;
-
-  const visited = new Set<string>();
-  const toVisit: Array<{ url: string; depth: number }> = [{ url: startUrl, depth: 0 }];
-  const pages: CrawledPage[] = [];
-  let failed = 0;
-
-  console.log(`🕷️  Crawling ${domain} (max ${maxPages} pages)...`);
-
-  while (toVisit.length > 0 && pages.length < maxPages) {
-    const { url, depth } = toVisit.shift()!;
-
-    // Normalize URL
-    const normalizedUrl = normalizeUrl(url, baseUrl);
-    if (!normalizedUrl || visited.has(normalizedUrl)) continue;
-
-    // Skip excluded paths
-    if (EXCLUDED_PATHS.some((p) => normalizedUrl.includes(p))) continue;
-
-    // Only crawl same domain
-    try {
-      const urlObj = new URL(normalizedUrl);
-      if (urlObj.hostname !== domain) continue;
-    } catch {
-      continue;
-    }
-
-    visited.add(normalizedUrl);
-
-    try {
-      const page = await crawlPage(normalizedUrl, timeout);
-      pages.push(page);
-
-      // Add internal links to queue (if not at max depth)
-      if (depth < maxDepth) {
-        for (const link of page.internalLinks) {
-          const normalizedLink = normalizeUrl(link, baseUrl);
-          if (normalizedLink && !visited.has(normalizedLink)) {
-            toVisit.push({ url: normalizedLink, depth: depth + 1 });
-          }
-        }
-      }
-
-      // Progress indicator
-      if (pages.length % 5 === 0) {
-        console.log(`   Crawled ${pages.length} pages...`);
-      }
-    } catch (error) {
-      failed++;
-    }
-  }
-
-  const duration = Date.now() - startTime;
-
-  // Aggregate content
-  const aggregatedContent = pages
-    .map((p) => `# ${p.title}\n${p.description}\n${p.mainContent}`)
-    .join('\n\n---\n\n');
-
-  const totalWordCount = pages.reduce((sum, p) => sum + p.wordCount, 0);
-
-  // Collect unique headings
-  const allHeadings = new Set<string>();
-  pages.forEach((p) => {
-    if (p.h1) allHeadings.add(p.h1);
-    p.h2s.forEach((h2) => allHeadings.add(h2));
-  });
-
-  // Count page types
-  const detectedPageTypes = {
-    product: pages.filter((p) => p.isProductPage).length,
-    pricing: pages.filter((p) => p.isPricingPage).length,
-    blog: pages.filter((p) => p.isBlogPost).length,
-    feature: pages.filter((p) => p.isFeaturePage).length,
-    other: pages.filter(
-      (p) => !p.isProductPage && !p.isPricingPage && !p.isBlogPost && !p.isFeaturePage
-    ).length,
-  };
-
-  console.log(`✅ Crawled ${pages.length} pages in ${(duration / 1000).toFixed(1)}s`);
-
-  return {
-    domain,
-    pages,
-    aggregatedContent,
-    totalWordCount,
-    uniqueHeadings: Array.from(allHeadings),
-    detectedPageTypes,
-    crawlStats: {
-      attempted: visited.size,
-      succeeded: pages.length,
-      failed,
-      duration,
-    },
-  };
-}
-
-/**
- * Crawl a single page
- */
-async function crawlPage(url: string, timeout: number): Promise<CrawledPage> {
-  const response = await httpGet<string>(url, {
-    timeout,
-    validateStatus: (status) => status === 200,
-  });
-
-  const html = response.data;
-  const $ = cheerio.load(html);
-
-  // Remove non-content elements
-  $('script, style, noscript, iframe, nav, footer, header, aside, [role="navigation"]').remove();
-
-  // Extract content
-  const title = $('title').text().trim();
-  const description = $('meta[name="description"]').attr('content')?.trim() || '';
-  const h1 = $('h1').first().text().trim();
-  const h2s = $('h2')
-    .map((_, el) => $(el).text().trim())
-    .get()
-    .filter((h) => h.length > 0);
-
-  // Get main content
-  const mainSelectors = ['main', 'article', '[role="main"]', '.content', '#content', '.post-content'];
-  let mainContent = '';
-
-  for (const selector of mainSelectors) {
-    const el = $(selector);
-    if (el.length > 0) {
-      mainContent = el.text().trim();
-      break;
-    }
-  }
-
-  // Fallback to body
-  if (!mainContent) {
-    mainContent = $('body').text().trim();
-  }
-
-  // Clean up whitespace
-  mainContent = mainContent
-    .replace(/\s+/g, ' ')
-    .replace(/\n+/g, '\n')
-    .trim();
-
-  // Limit content length for AI processing
-  if (mainContent.length > 5000) {
-    mainContent = mainContent.substring(0, 5000) + '...';
-  }
-
-  const wordCount = mainContent.split(/\s+/).length;
-
-  // Extract internal links
-  const internalLinks: string[] = [];
-  $('a[href]').each((_, el) => {
-    const href = $(el).attr('href');
-    if (href && !href.startsWith('mailto:') && !href.startsWith('tel:') && !href.startsWith('#')) {
-      internalLinks.push(href);
-    }
-  });
-
-  // Detect page type
-  const urlLower = url.toLowerCase();
-  const isProductPage = PAGE_TYPE_PATTERNS.product.some((p) => urlLower.includes(p));
-  const isPricingPage = PAGE_TYPE_PATTERNS.pricing.some((p) => urlLower.includes(p));
-  const isBlogPost = PAGE_TYPE_PATTERNS.blog.some((p) => urlLower.includes(p));
-  const isFeaturePage = PAGE_TYPE_PATTERNS.feature.some((p) => urlLower.includes(p));
-
-  return {
-    url,
-    title,
-    description,
-    h1,
-    h2s,
-    mainContent,
-    wordCount,
-    internalLinks,
-    isProductPage,
-    isPricingPage,
-    isBlogPost,
-    isFeaturePage,
-  };
-}
-
-/**
- * Normalize a URL
- */
-function normalizeUrl(url: string, baseUrl: string): string | null {
-  try {
-    // Handle relative URLs
-    const absoluteUrl = url.startsWith('http') ? url : new URL(url, baseUrl).href;
-
-    // Remove hash and trailing slash
-    const urlObj = new URL(absoluteUrl);
-    urlObj.hash = '';
-    let normalized = urlObj.href;
-
-    // Remove trailing slash (except for root)
-    if (normalized.endsWith('/') && normalized !== `${urlObj.origin}/`) {
-      normalized = normalized.slice(0, -1);
-    }
-
-    return normalized;
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Extract key phrases from crawled content
- */
-export function extractKeyPhrases(crawlResult: SiteCrawlResult): string[] {
-  const phrases = new Set<string>();
-
-  // From titles
-  crawlResult.pages.forEach((page) => {
-    if (page.title) {
-      // Split on common separators
-      const parts = page.title.split(/[|\-–—:]/).map((p) => p.trim());
-      parts.forEach((part) => {
-        if (part.length > 3 && part.length < 50) {
-          phrases.add(part.toLowerCase());
-        }
-      });
-    }
-  });
-
-  // From headings
-  crawlResult.uniqueHeadings.forEach((heading) => {
-    if (heading.length > 3 && heading.length < 50) {
-      phrases.add(heading.toLowerCase());
-    }
-  });
-
-  // From descriptions
-  crawlResult.pages.forEach((page) => {
-    if (page.description) {
-      // Extract noun phrases (simplified - just multi-word sequences)
-      const words = page.description.toLowerCase().split(/\s+/);
-      for (let i = 0; i < words.length - 1; i++) {
-        const bigram = `${words[i]} ${words[i + 1]}`;
-        if (bigram.length > 5 && bigram.length < 40) {
-          phrases.add(bigram);
-        }
-      }
-    }
-  });
-
-  return Array.from(phrases).slice(0, 100);
-}