@rankcli/agent-runtime 0.0.1

package/src/keywords/prioritizer.ts

@@ -0,0 +1,295 @@
+// Keyword Prioritization Algorithm
+
+import type {
+  SiteProfile,
+  KeywordData,
+  KeywordOpportunity,
+  KeywordAction,
+  KeywordResearchResult,
+} from './types.js';
+import { getMaxKdThreshold, PRIORITY_WEIGHTS } from './types.js';
+
+export function prioritizeKeywords(
+  keywords: KeywordData[],
+  siteProfile: SiteProfile,
+  existingMeta?: { title?: string; description?: string; h1?: string }
+): KeywordResearchResult {
+  const maxKd = getMaxKdThreshold(siteProfile);
+
+  // Score and categorize each keyword
+  const opportunities: KeywordOpportunity[] = keywords.map((kw) => {
+    const priorityScore = calculatePriorityScore(kw, siteProfile, maxKd);
+    const category = categorizeKeyword(kw.keywordDifficulty, maxKd);
+    const suggestedAction = suggestAction(kw, existingMeta, category);
+
+    return {
+      ...kw,
+      priorityScore,
+      category,
+      suggestedAction,
+    };
+  });
+
+  // Sort by priority score
+  opportunities.sort((a, b) => b.priorityScore - a.priorityScore);
+
+  // Categorize into buckets
+  const quickWins = opportunities.filter((kw) => kw.category === 'quick-win');
+  const mediumTerm = opportunities.filter((kw) => kw.category === 'medium-term');
+  const longTerm = opportunities.filter((kw) => kw.category === 'long-term');
+
+  // Generate recommendations
+  const recommendations = generateRecommendations(siteProfile, quickWins, mediumTerm);
+
+  return {
+    siteProfile,
+    keywords: opportunities,
+    quickWins,
+    mediumTerm,
+    longTerm,
+    recommendations,
+    maxKdThreshold: maxKd,
+  };
+}
+
+function calculatePriorityScore(
+  keyword: KeywordData,
+  profile: SiteProfile,
+  maxKd: number
+): number {
+  // Business value score (0-100)
+  const businessScore = calculateBusinessValue(keyword, profile.businessGoal);
+
+  // Difficulty score (inverse - lower KD = higher score)
+  let difficultyScore = 0;
+  if (keyword.keywordDifficulty <= maxKd) {
+    difficultyScore = 100 - (keyword.keywordDifficulty / maxKd) * 100;
+  }
+
+  // Traffic score (normalized, capped at 100)
+  const trafficScore = Math.min((keyword.searchVolume / 1000) * 100, 100);
+
+  // Weighted sum
+  const score =
+    businessScore * PRIORITY_WEIGHTS.businessValue +
+    difficultyScore * PRIORITY_WEIGHTS.difficulty +
+    trafficScore * PRIORITY_WEIGHTS.trafficPotential;
+
+  return Math.round(score);
+}
+
+function calculateBusinessValue(keyword: KeywordData, goal: SiteProfile['businessGoal']): number {
+  const intent = keyword.intent || inferIntent(keyword.keyword);
+
+  // Intent weights based on business goal
+  const intentWeights: Record<SiteProfile['businessGoal'], Record<string, number>> = {
+    signups: {
+      transactional: 100,
+      commercial: 80,
+      informational: 40,
+      navigational: 20,
+    },
+    purchases: {
+      transactional: 100,
+      commercial: 90,
+      informational: 30,
+      navigational: 20,
+    },
+    leads: {
+      commercial: 100,
+      transactional: 80,
+      informational: 60,
+      navigational: 20,
+    },
+    awareness: {
+      informational: 100,
+      commercial: 60,
+      navigational: 40,
+      transactional: 30,
+    },
+  };
+
+  return intentWeights[goal][intent] || 50;
+}
+
+function inferIntent(keyword: string): 'informational' | 'commercial' | 'transactional' | 'navigational' {
+  const kw = keyword.toLowerCase();
+
+  // Transactional indicators
+  if (/\b(buy|purchase|order|subscribe|download|get|try|free trial)\b/.test(kw)) {
+    return 'transactional';
+  }
+
+  // Commercial investigation
+  if (/\b(best|top|review|compare|vs|versus|alternative|pricing|cost)\b/.test(kw)) {
+    return 'commercial';
+  }
+
+  // Navigational
+  if (/\b(login|sign in|website|official|app)\b/.test(kw)) {
+    return 'navigational';
+  }
+
+  // Default to informational
+  return 'informational';
+}
+
+function categorizeKeyword(kd: number, maxKd: number): 'quick-win' | 'medium-term' | 'long-term' {
+  const quickWinThreshold = Math.min(maxKd * 0.5, 15);
+  const mediumThreshold = maxKd;
+
+  if (kd <= quickWinThreshold) return 'quick-win';
+  if (kd <= mediumThreshold) return 'medium-term';
+  return 'long-term';
+}
+
+function suggestAction(
+  keyword: KeywordData,
+  existingMeta: { title?: string; description?: string; h1?: string } | undefined,
+  category: 'quick-win' | 'medium-term' | 'long-term'
+): KeywordAction {
+  const kw = keyword.keyword;
+
+  // Check if keyword is already in existing meta
+  const inTitle = existingMeta?.title?.toLowerCase().includes(kw.toLowerCase());
+  const inDescription = existingMeta?.description?.toLowerCase().includes(kw.toLowerCase());
+  const inH1 = existingMeta?.h1?.toLowerCase().includes(kw.toLowerCase());
+
+  // Quick wins: optimize existing elements
+  if (category === 'quick-win') {
+    if (!inTitle && existingMeta?.title) {
+      return {
+        type: 'add-to-title',
+        description: `Add "${kw}" to your title tag`,
+        targetElement: 'title',
+        currentValue: existingMeta.title,
+        suggestedValue: generateTitleWithKeyword(existingMeta.title, kw),
+      };
+    }
+
+    if (!inH1) {
+      return {
+        type: 'add-to-h1',
+        description: `Include "${kw}" in your H1 heading`,
+        targetElement: 'h1',
+        currentValue: existingMeta?.h1,
+        suggestedValue: generateH1WithKeyword(existingMeta?.h1, kw),
+      };
+    }
+
+    if (!inDescription && existingMeta?.description) {
+      return {
+        type: 'add-to-meta',
+        description: `Add "${kw}" to your meta description`,
+        targetElement: 'meta[name="description"]',
+        currentValue: existingMeta.description,
+        suggestedValue: generateDescriptionWithKeyword(existingMeta.description, kw),
+      };
+    }
+
+    return {
+      type: 'optimize-existing',
+      description: `Optimize existing content for "${kw}"`,
+    };
+  }
+
+  // Medium-term: may need content creation
+  if (category === 'medium-term') {
+    return {
+      type: 'create-content',
+      description: `Create dedicated content targeting "${kw}"`,
+    };
+  }
+
+  // Long-term: need authority building first
+  return {
+    type: 'create-content',
+    description: `Build authority first, then target "${kw}" (high competition)`,
+  };
+}
+
+function generateTitleWithKeyword(currentTitle: string, keyword: string): string {
+  // If title is short, prepend keyword
+  if (currentTitle.length < 30) {
+    return `${capitalizeFirst(keyword)} - ${currentTitle}`;
+  }
+
+  // Otherwise, try to incorporate naturally
+  const parts = currentTitle.split(' - ');
+  if (parts.length >= 2) {
+    return `${capitalizeFirst(keyword)} | ${parts[parts.length - 1]}`;
+  }
+
+  return `${capitalizeFirst(keyword)} | ${currentTitle}`.substring(0, 60);
+}
+
+function generateH1WithKeyword(currentH1: string | undefined, keyword: string): string {
+  if (!currentH1) {
+    return capitalizeFirst(keyword);
+  }
+
+  // Try to incorporate keyword naturally
+  return `${capitalizeFirst(keyword)} - ${currentH1}`;
+}
+
+function generateDescriptionWithKeyword(currentDesc: string, keyword: string): string {
+  // If description doesn't start with keyword concept, prepend it
+  if (!currentDesc.toLowerCase().includes(keyword.toLowerCase().split(' ')[0])) {
+    const newDesc = `${capitalizeFirst(keyword)}: ${currentDesc}`;
+    return newDesc.substring(0, 160);
+  }
+
+  return currentDesc;
+}
+
+function capitalizeFirst(str: string): string {
+  return str.charAt(0).toUpperCase() + str.slice(1);
+}
+
+function generateRecommendations(
+  profile: SiteProfile,
+  quickWins: KeywordOpportunity[],
+  mediumTerm: KeywordOpportunity[]
+): string[] {
+  const recommendations: string[] = [];
+
+  // Domain age specific recommendations
+  if (profile.domainAge === 'new') {
+    recommendations.push(
+      'Focus on quick-win keywords (KD < 15) first to build initial traffic and authority.'
+    );
+    recommendations.push(
+      'Consider creating long-form, comprehensive content to establish topical authority.'
+    );
+  }
+
+  // Backlink recommendations
+  if (profile.backlinkCount === 'none' || profile.backlinkCount === 'few') {
+    recommendations.push(
+      'Prioritize building quality backlinks to increase your ranking potential for competitive keywords.'
+    );
+  }
+
+  // Content recommendations based on capacity
+  if (profile.contentCapacity === 'low' && quickWins.length > 2) {
+    recommendations.push(
+      `Focus on your top ${Math.min(2, quickWins.length)} quick-win keywords first, then expand.`
+    );
+  }
+
+  // Quick wins available
+  if (quickWins.length > 0) {
+    recommendations.push(
+      `You have ${quickWins.length} quick-win keyword opportunities that you can rank for relatively quickly.`
+    );
+  }
+
+  // Medium-term opportunities
+  if (mediumTerm.length > 0 && profile.domainAge !== 'new') {
+    recommendations.push(
+      `${mediumTerm.length} medium-difficulty keywords are within reach with focused content and some link building.`
+    );
+  }
+
+  return recommendations;
+}

package/src/keywords/site-crawler.ts

@@ -0,0 +1,342 @@
+/**
+ * Site Crawler for Keyword Research
+ *
+ * Crawls an entire site to collect text content for AI analysis.
+ * Used to understand what the site does and generate relevant keywords.
+ */
+
+import * as cheerio from 'cheerio';
+import { httpGet } from '../utils/http.js';
+
+export interface CrawledPage {
+  url: string;
+  title: string;
+  description: string;
+  h1: string;
+  h2s: string[];
+  mainContent: string;
+  wordCount: number;
+  internalLinks: string[];
+  isProductPage: boolean;
+  isPricingPage: boolean;
+  isBlogPost: boolean;
+  isFeaturePage: boolean;
+}
+
+export interface SiteCrawlResult {
+  domain: string;
+  pages: CrawledPage[];
+  aggregatedContent: string;
+  totalWordCount: number;
+  uniqueHeadings: string[];
+  detectedPageTypes: {
+    product: number;
+    pricing: number;
+    blog: number;
+    feature: number;
+    other: number;
+  };
+  crawlStats: {
+    attempted: number;
+    succeeded: number;
+    failed: number;
+    duration: number;
+  };
+}
+
+const EXCLUDED_PATHS = [
+  '/cdn-cgi/',
+  '/wp-admin/',
+  '/wp-includes/',
+  '/wp-json/',
+  '/api/',
+  '/admin/',
+  '/_next/',
+  '/static/',
+  '/assets/',
+  '.xml',
+  '.json',
+  '.pdf',
+  '.zip',
+  '.png',
+  '.jpg',
+  '.jpeg',
+  '.gif',
+  '.svg',
+  '.ico',
+  '.css',
+  '.js',
+];
+
+const PAGE_TYPE_PATTERNS = {
+  product: ['/product', '/features', '/solutions', '/platform', '/tour'],
+  pricing: ['/pricing', '/plans', '/subscription'],
+  blog: ['/blog', '/news', '/articles', '/posts', '/journal'],
+  feature: ['/feature', '/capability', '/integration'],
+};
+
+/**
+ * Crawl an entire site to collect text content
+ */
+export async function crawlSite(
+  startUrl: string,
+  options: {
+    maxPages?: number;
+    maxDepth?: number;
+    timeout?: number;
+  } = {}
+): Promise<SiteCrawlResult> {
+  const { maxPages = 30, maxDepth = 3, timeout = 10000 } = options;
+
+  const startTime = Date.now();
+  const parsedUrl = new URL(startUrl);
+  const domain = parsedUrl.hostname;
+  const baseUrl = `${parsedUrl.protocol}//${domain}`;
+
+  const visited = new Set<string>();
+  const toVisit: Array<{ url: string; depth: number }> = [{ url: startUrl, depth: 0 }];
+  const pages: CrawledPage[] = [];
+  let failed = 0;
+
+  console.log(`🕷️  Crawling ${domain} (max ${maxPages} pages)...`);
+
+  while (toVisit.length > 0 && pages.length < maxPages) {
+    const { url, depth } = toVisit.shift()!;
+
+    // Normalize URL
+    const normalizedUrl = normalizeUrl(url, baseUrl);
+    if (!normalizedUrl || visited.has(normalizedUrl)) continue;
+
+    // Skip excluded paths
+    if (EXCLUDED_PATHS.some((p) => normalizedUrl.includes(p))) continue;
+
+    // Only crawl same domain
+    try {
+      const urlObj = new URL(normalizedUrl);
+      if (urlObj.hostname !== domain) continue;
+    } catch {
+      continue;
+    }
+
+    visited.add(normalizedUrl);
+
+    try {
+      const page = await crawlPage(normalizedUrl, timeout);
+      pages.push(page);
+
+      // Add internal links to queue (if not at max depth)
+      if (depth < maxDepth) {
+        for (const link of page.internalLinks) {
+          const normalizedLink = normalizeUrl(link, baseUrl);
+          if (normalizedLink && !visited.has(normalizedLink)) {
+            toVisit.push({ url: normalizedLink, depth: depth + 1 });
+          }
+        }
+      }
+
+      // Progress indicator
+      if (pages.length % 5 === 0) {
+        console.log(`   Crawled ${pages.length} pages...`);
+      }
+    } catch (error) {
+      failed++;
+    }
+  }
+
+  const duration = Date.now() - startTime;
+
+  // Aggregate content
+  const aggregatedContent = pages
+    .map((p) => `# ${p.title}\n${p.description}\n${p.mainContent}`)
+    .join('\n\n---\n\n');
+
+  const totalWordCount = pages.reduce((sum, p) => sum + p.wordCount, 0);
+
+  // Collect unique headings
+  const allHeadings = new Set<string>();
+  pages.forEach((p) => {
+    if (p.h1) allHeadings.add(p.h1);
+    p.h2s.forEach((h2) => allHeadings.add(h2));
+  });
+
+  // Count page types
+  const detectedPageTypes = {
+    product: pages.filter((p) => p.isProductPage).length,
+    pricing: pages.filter((p) => p.isPricingPage).length,
+    blog: pages.filter((p) => p.isBlogPost).length,
+    feature: pages.filter((p) => p.isFeaturePage).length,
+    other: pages.filter(
+      (p) => !p.isProductPage && !p.isPricingPage && !p.isBlogPost && !p.isFeaturePage
+    ).length,
+  };
+
+  console.log(`✅ Crawled ${pages.length} pages in ${(duration / 1000).toFixed(1)}s`);
+
+  return {
+    domain,
+    pages,
+    aggregatedContent,
+    totalWordCount,
+    uniqueHeadings: Array.from(allHeadings),
+    detectedPageTypes,
+    crawlStats: {
+      attempted: visited.size,
+      succeeded: pages.length,
+      failed,
+      duration,
+    },
+  };
+}
+
+/**
+ * Crawl a single page
+ */
+async function crawlPage(url: string, timeout: number): Promise<CrawledPage> {
+  const response = await httpGet<string>(url, {
+    timeout,
+    validateStatus: (status) => status === 200,
+  });
+
+  const html = response.data;
+  const $ = cheerio.load(html);
+
+  // Remove non-content elements
+  $('script, style, noscript, iframe, nav, footer, header, aside, [role="navigation"]').remove();
+
+  // Extract content
+  const title = $('title').text().trim();
+  const description = $('meta[name="description"]').attr('content')?.trim() || '';
+  const h1 = $('h1').first().text().trim();
+  const h2s = $('h2')
+    .map((_, el) => $(el).text().trim())
+    .get()
+    .filter((h) => h.length > 0);
+
+  // Get main content
+  const mainSelectors = ['main', 'article', '[role="main"]', '.content', '#content', '.post-content'];
+  let mainContent = '';
+
+  for (const selector of mainSelectors) {
+    const el = $(selector);
+    if (el.length > 0) {
+      mainContent = el.text().trim();
+      break;
+    }
+  }
+
+  // Fallback to body
+  if (!mainContent) {
+    mainContent = $('body').text().trim();
+  }
+
+  // Clean up whitespace
+  mainContent = mainContent
+    .replace(/\s+/g, ' ')
+    .replace(/\n+/g, '\n')
+    .trim();
+
+  // Limit content length for AI processing
+  if (mainContent.length > 5000) {
+    mainContent = mainContent.substring(0, 5000) + '...';
+  }
+
+  const wordCount = mainContent.split(/\s+/).length;
+
+  // Extract internal links
+  const internalLinks: string[] = [];
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (href && !href.startsWith('mailto:') && !href.startsWith('tel:') && !href.startsWith('#')) {
+      internalLinks.push(href);
+    }
+  });
+
+  // Detect page type
+  const urlLower = url.toLowerCase();
+  const isProductPage = PAGE_TYPE_PATTERNS.product.some((p) => urlLower.includes(p));
+  const isPricingPage = PAGE_TYPE_PATTERNS.pricing.some((p) => urlLower.includes(p));
+  const isBlogPost = PAGE_TYPE_PATTERNS.blog.some((p) => urlLower.includes(p));
+  const isFeaturePage = PAGE_TYPE_PATTERNS.feature.some((p) => urlLower.includes(p));
+
+  return {
+    url,
+    title,
+    description,
+    h1,
+    h2s,
+    mainContent,
+    wordCount,
+    internalLinks,
+    isProductPage,
+    isPricingPage,
+    isBlogPost,
+    isFeaturePage,
+  };
+}
+
+/**
+ * Normalize a URL
+ */
+function normalizeUrl(url: string, baseUrl: string): string | null {
+  try {
+    // Handle relative URLs
+    const absoluteUrl = url.startsWith('http') ? url : new URL(url, baseUrl).href;
+
+    // Remove hash and trailing slash
+    const urlObj = new URL(absoluteUrl);
+    urlObj.hash = '';
+    let normalized = urlObj.href;
+
+    // Remove trailing slash (except for root)
+    if (normalized.endsWith('/') && normalized !== `${urlObj.origin}/`) {
+      normalized = normalized.slice(0, -1);
+    }
+
+    return normalized;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Extract key phrases from crawled content
+ */
+export function extractKeyPhrases(crawlResult: SiteCrawlResult): string[] {
+  const phrases = new Set<string>();
+
+  // From titles
+  crawlResult.pages.forEach((page) => {
+    if (page.title) {
+      // Split on common separators
+      const parts = page.title.split(/[|\-–—:]/).map((p) => p.trim());
+      parts.forEach((part) => {
+        if (part.length > 3 && part.length < 50) {
+          phrases.add(part.toLowerCase());
+        }
+      });
+    }
+  });
+
+  // From headings
+  crawlResult.uniqueHeadings.forEach((heading) => {
+    if (heading.length > 3 && heading.length < 50) {
+      phrases.add(heading.toLowerCase());
+    }
+  });
+
+  // From descriptions
+  crawlResult.pages.forEach((page) => {
+    if (page.description) {
+      // Extract noun phrases (simplified - just multi-word sequences)
+      const words = page.description.toLowerCase().split(/\s+/);
+      for (let i = 0; i < words.length - 1; i++) {
+        const bigram = `${words[i]} ${words[i + 1]}`;
+        if (bigram.length > 5 && bigram.length < 40) {
+          phrases.add(bigram);
+        }
+      }
+    }
+  });
+
+  return Array.from(phrases).slice(0, 100);
+}