@rankcli/agent-runtime 0.0.15 → 0.0.16

package/src/keywords/sources/autocomplete.ts

@@ -1,139 +0,0 @@
-// Google Autocomplete Keyword Scraper (Free)
-
-import { httpGet } from '../../utils/http.js';
-import type { KeywordData } from '../types.js';
-
-const AUTOCOMPLETE_URL = 'https://suggestqueries.google.com/complete/search';
-
-export interface AutocompleteOptions {
-  language?: string;
-  country?: string;
-}
-
-export async function getAutocompleteSuggestions(
-  seedKeyword: string,
-  options: AutocompleteOptions = {}
-): Promise<KeywordData[]> {
-  const { language = 'en', country = 'us' } = options;
-
-  try {
-    const response = await httpGet<string>(AUTOCOMPLETE_URL, {
-      params: {
-        client: 'firefox', // Returns JSON
-        q: seedKeyword,
-        hl: language,
-        gl: country,
-      },
-      timeout: 5000,
-    });
-
-    // Response format: [query, [suggestions], ...]
-    // Parse JSON if string (fetch returns text by default for this endpoint)
-    const data = typeof response.data === 'string' ? JSON.parse(response.data) : response.data;
-    const suggestions: string[] = (data as unknown[])[1] as string[] || [];
-
-    return suggestions.map((suggestion) => ({
-      keyword: suggestion,
-      searchVolume: 0, // Autocomplete doesn't provide volume
-      keywordDifficulty: 0, // Need to estimate or fetch separately
-      source: 'autocomplete' as const,
-    }));
-  } catch (error) {
-    console.error('Autocomplete fetch failed:', error);
-    return [];
-  }
-}
-
-// Get expanded suggestions using alphabet soup technique
-export async function getExpandedSuggestions(
-  seedKeyword: string,
-  options: AutocompleteOptions = {}
-): Promise<KeywordData[]> {
-  const allSuggestions: KeywordData[] = [];
-  const seen = new Set<string>();
-
-  // Base query
-  const baseSuggestions = await getAutocompleteSuggestions(seedKeyword, options);
-  for (const kw of baseSuggestions) {
-    if (!seen.has(kw.keyword.toLowerCase())) {
-      seen.add(kw.keyword.toLowerCase());
-      allSuggestions.push(kw);
-    }
-  }
-
-  // Alphabet soup: append each letter
-  const letters = 'abcdefghijklmnopqrstuvwxyz'.split('');
-
-  // Limit concurrent requests
-  const batchSize = 5;
-  for (let i = 0; i < letters.length; i += batchSize) {
-    const batch = letters.slice(i, i + batchSize);
-    const promises = batch.map((letter) =>
-      getAutocompleteSuggestions(`${seedKeyword} ${letter}`, options)
-    );
-
-    const results = await Promise.all(promises);
-    for (const suggestions of results) {
-      for (const kw of suggestions) {
-        if (!seen.has(kw.keyword.toLowerCase())) {
-          seen.add(kw.keyword.toLowerCase());
-          allSuggestions.push(kw);
-        }
-      }
-    }
-
-    // Small delay to avoid rate limiting
-    await new Promise((resolve) => setTimeout(resolve, 100));
-  }
-
-  // Question modifiers
-  const questionPrefixes = ['how to', 'what is', 'why', 'when', 'where', 'best'];
-  for (const prefix of questionPrefixes) {
-    const suggestions = await getAutocompleteSuggestions(`${prefix} ${seedKeyword}`, options);
-    for (const kw of suggestions) {
-      if (!seen.has(kw.keyword.toLowerCase())) {
-        seen.add(kw.keyword.toLowerCase());
-        allSuggestions.push(kw);
-      }
-    }
-  }
-
-  return allSuggestions;
-}
-
-// Estimate keyword difficulty based on SERP analysis
-export function estimateKeywordDifficulty(keyword: string): number {
-  // Basic heuristics (can be improved with actual SERP data)
-  const wordCount = keyword.split(' ').length;
-
-  // Longer keywords tend to be less competitive
-  if (wordCount >= 5) return 5;
-  if (wordCount >= 4) return 10;
-  if (wordCount >= 3) return 20;
-  if (wordCount >= 2) return 30;
-
-  return 50; // Single word = high competition
-}
-
-// Estimate search volume based on keyword characteristics
-export function estimateSearchVolume(keyword: string): number {
-  // This is a rough estimate - real data should come from APIs
-  const wordCount = keyword.split(' ').length;
-
-  // Longer tail = lower volume typically
-  if (wordCount >= 5) return 50;
-  if (wordCount >= 4) return 100;
-  if (wordCount >= 3) return 300;
-  if (wordCount >= 2) return 500;
-
-  return 1000; // Single word = high volume
-}
-
-// Add estimates to keywords without data
-export function enrichKeywordsWithEstimates(keywords: KeywordData[]): KeywordData[] {
-  return keywords.map((kw) => ({
-    ...kw,
-    keywordDifficulty: kw.keywordDifficulty || estimateKeywordDifficulty(kw.keyword),
-    searchVolume: kw.searchVolume || estimateSearchVolume(kw.keyword),
-  }));
-}

package/src/keywords/sources/competitive-search.ts

@@ -1,450 +0,0 @@
-/**
- * Competitive Search Module
- *
- * Uses free APIs to find existing tools, competitors, and market intelligence.
- * Priority order: Brave Search > Serper.dev > GitHub > npm/PyPI
- */
-
-export interface CompetitorTool {
-  name: string;
-  url: string;
-  description: string;
-  isPaid: boolean;
-  pricing?: string;
-  source: 'search' | 'github' | 'npm' | 'pypi' | 'hackernews';
-  stars?: number;
-  weeklyDownloads?: number;
-}
-
-export interface CompetitiveSearchResult {
-  query: string;
-  existingTools: CompetitorTool[];
-  openSourceOptions: CompetitorTool[];
-  marketInsights: {
-    hasFreeSolutions: boolean;
-    hasPaidSolutions: boolean;
-    dominantPlayer?: string;
-    marketGap?: string;
-  };
-  searchSource: string;
-}
-
-export interface CompetitiveSearchOptions {
-  braveApiKey?: string;
-  serperApiKey?: string;
-  githubToken?: string;
-  maxResults?: number;
-}
-
-/**
- * Search for existing tools and competitors
- */
-export async function searchCompetitors(
-  toolConcept: string,
-  options: CompetitiveSearchOptions = {}
-): Promise<CompetitiveSearchResult> {
-  const { maxResults = 10 } = options;
-  const existingTools: CompetitorTool[] = [];
-  const openSourceOptions: CompetitorTool[] = [];
-  let searchSource = 'none';
-
-  // Try Brave Search first (2,000 free/month)
-  if (options.braveApiKey) {
-    try {
-      const braveResults = await searchBrave(toolConcept, options.braveApiKey, maxResults);
-      existingTools.push(...braveResults);
-      searchSource = 'brave';
-    } catch (e) {
-      console.warn('Brave search failed:', e);
-    }
-  }
-
-  // Fall back to Serper.dev (2,500 free/month)
-  if (existingTools.length === 0 && options.serperApiKey) {
-    try {
-      const serperResults = await searchSerper(toolConcept, options.serperApiKey, maxResults);
-      existingTools.push(...serperResults);
-      searchSource = 'serper';
-    } catch (e) {
-      console.warn('Serper search failed:', e);
-    }
-  }
-
-  // Always search GitHub for open source (unlimited free)
-  try {
-    const githubResults = await searchGitHub(toolConcept, options.githubToken, maxResults);
-    openSourceOptions.push(...githubResults);
-    if (!searchSource || searchSource === 'none') searchSource = 'github';
-  } catch (e) {
-    console.warn('GitHub search failed:', e);
-  }
-
-  // Search npm for JavaScript tools (unlimited free)
-  try {
-    const npmResults = await searchNpm(toolConcept, maxResults);
-    openSourceOptions.push(...npmResults);
-  } catch (e) {
-    console.warn('npm search failed:', e);
-  }
-
-  // Analyze market
-  const marketInsights = analyzeMarket(existingTools, openSourceOptions);
-
-  return {
-    query: toolConcept,
-    existingTools,
-    openSourceOptions,
-    marketInsights,
-    searchSource,
-  };
-}
-
-/**
- * Brave Search API (2,000 free searches/month)
- */
-async function searchBrave(
-  query: string,
-  apiKey: string,
-  maxResults: number
-): Promise<CompetitorTool[]> {
-  const searchQuery = `${query} tool OR ${query} free online`;
-
-  const response = await fetch(
-    `https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(searchQuery)}&count=${maxResults}`,
-    {
-      headers: {
-        'Accept': 'application/json',
-        'X-Subscription-Token': apiKey,
-      },
-    }
-  );
-
-  if (!response.ok) {
-    throw new Error(`Brave API error: ${response.status}`);
-  }
-
-  const data = await response.json();
-  const results: CompetitorTool[] = [];
-
-  for (const result of data.web?.results || []) {
-    const isPaid = detectPaidTool(result.title, result.description, result.url);
-
-    results.push({
-      name: extractToolName(result.title),
-      url: result.url,
-      description: result.description || '',
-      isPaid,
-      pricing: isPaid ? extractPricing(result.description) : 'Free',
-      source: 'search',
-    });
-  }
-
-  return results;
-}
-
-/**
- * Serper.dev API (2,500 free searches/month)
- */
-async function searchSerper(
-  query: string,
-  apiKey: string,
-  maxResults: number
-): Promise<CompetitorTool[]> {
-  const searchQuery = `${query} tool OR ${query} free online`;
-
-  const response = await fetch('https://google.serper.dev/search', {
-    method: 'POST',
-    headers: {
-      'X-API-KEY': apiKey,
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify({
-      q: searchQuery,
-      num: maxResults,
-    }),
-  });
-
-  if (!response.ok) {
-    throw new Error(`Serper API error: ${response.status}`);
-  }
-
-  const data = await response.json();
-  const results: CompetitorTool[] = [];
-
-  for (const result of data.organic || []) {
-    const isPaid = detectPaidTool(result.title, result.snippet, result.link);
-
-    results.push({
-      name: extractToolName(result.title),
-      url: result.link,
-      description: result.snippet || '',
-      isPaid,
-      pricing: isPaid ? extractPricing(result.snippet) : 'Free',
-      source: 'search',
-    });
-  }
-
-  return results;
-}
-
-/**
- * GitHub Search API (60 req/hour unauthenticated, more with token)
- */
-async function searchGitHub(
-  query: string,
-  token?: string,
-  maxResults: number = 10
-): Promise<CompetitorTool[]> {
-  const searchQuery = `${query} in:name,description,readme`;
-
-  const headers: Record<string, string> = {
-    'Accept': 'application/vnd.github.v3+json',
-    'User-Agent': 'RankCLI-Keyword-Research',
-  };
-
-  if (token) {
-    headers['Authorization'] = `token ${token}`;
-  }
-
-  const response = await fetch(
-    `https://api.github.com/search/repositories?q=${encodeURIComponent(searchQuery)}&sort=stars&order=desc&per_page=${maxResults}`,
-    { headers }
-  );
-
-  if (!response.ok) {
-    throw new Error(`GitHub API error: ${response.status}`);
-  }
-
-  const data = await response.json();
-  const results: CompetitorTool[] = [];
-
-  for (const repo of data.items || []) {
-    results.push({
-      name: repo.name,
-      url: repo.html_url,
-      description: repo.description || '',
-      isPaid: false,
-      pricing: 'Open Source',
-      source: 'github',
-      stars: repo.stargazers_count,
-    });
-  }
-
-  return results;
-}
-
-/**
- * npm Registry Search (unlimited, no auth required)
- */
-async function searchNpm(
-  query: string,
-  maxResults: number = 10
-): Promise<CompetitorTool[]> {
-  const response = await fetch(
-    `https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(query)}&size=${maxResults}`,
-    {
-      headers: {
-        'User-Agent': 'RankCLI-Keyword-Research',
-      },
-    }
-  );
-
-  if (!response.ok) {
-    throw new Error(`npm API error: ${response.status}`);
-  }
-
-  const data = await response.json();
-  const results: CompetitorTool[] = [];
-
-  for (const obj of data.objects || []) {
-    const pkg = obj.package;
-    results.push({
-      name: pkg.name,
-      url: `https://www.npmjs.com/package/${pkg.name}`,
-      description: pkg.description || '',
-      isPaid: false,
-      pricing: 'Open Source (npm)',
-      source: 'npm',
-      weeklyDownloads: obj.downloads?.weekly,
-    });
-  }
-
-  return results;
-}
-
-/**
- * Detect if a tool is paid based on text signals
- */
-function detectPaidTool(title: string, description: string, url: string): boolean {
-  const text = `${title} ${description} ${url}`.toLowerCase();
-
-  // Strong paid signals
-  const paidSignals = [
-    'pricing', 'subscribe', 'premium', 'pro plan', 'enterprise',
-    'per month', '/mo', 'free trial', 'buy now', 'purchase',
-    'starting at', 'plans start', 'paid', 'upgrade',
-  ];
-
-  // Strong free signals
-  const freeSignals = [
-    'free online', '100% free', 'completely free', 'no signup',
-    'open source', 'github.com', 'github.io', 'codepen',
-  ];
-
-  const hasPaidSignal = paidSignals.some(signal => text.includes(signal));
-  const hasFreeSignal = freeSignals.some(signal => text.includes(signal));
-
-  // If has free signal and no paid signal, it's free
-  if (hasFreeSignal && !hasPaidSignal) return false;
-
-  // If has paid signal, it's paid
-  if (hasPaidSignal) return true;
-
-  // Default: assume it might be paid
-  return false;
-}
-
-/**
- * Extract tool name from search result title
- */
-function extractToolName(title: string): string {
-  // Remove common suffixes like "| Website", "- Free Online Tool", etc.
-  return title
-    .split(/[|\-–—]/)[0]
-    .replace(/free|online|tool|generator|maker/gi, '')
-    .trim() || title.trim();
-}
-
-/**
- * Extract pricing info from description
- */
-function extractPricing(text: string): string | undefined {
-  const priceMatch = text.match(/\$[\d,.]+(?:\s*\/\s*(?:mo|month|year|yr))?/i);
-  if (priceMatch) return priceMatch[0];
-
-  if (text.toLowerCase().includes('free trial')) return 'Free trial available';
-  if (text.toLowerCase().includes('freemium')) return 'Freemium';
-
-  return undefined;
-}
-
-/**
- * Analyze market based on search results
- */
-function analyzeMarket(
-  existingTools: CompetitorTool[],
-  openSourceOptions: CompetitorTool[]
-): CompetitiveSearchResult['marketInsights'] {
-  const hasFreeSolutions = existingTools.some(t => !t.isPaid) || openSourceOptions.length > 0;
-  const hasPaidSolutions = existingTools.some(t => t.isPaid);
-
-  // Find dominant player (most stars on GitHub or appears in multiple results)
-  let dominantPlayer: string | undefined;
-  if (openSourceOptions.length > 0) {
-    const topByStars = openSourceOptions
-      .filter(t => t.stars)
-      .sort((a, b) => (b.stars || 0) - (a.stars || 0))[0];
-    if (topByStars && (topByStars.stars || 0) > 1000) {
-      dominantPlayer = topByStars.name;
-    }
-  }
-
-  // Identify market gap
-  let marketGap: string | undefined;
-  if (!hasFreeSolutions && hasPaidSolutions) {
-    marketGap = 'No free alternatives exist - opportunity to capture market with free tool';
-  } else if (openSourceOptions.length > 0 && !existingTools.some(t => !t.isPaid)) {
-    marketGap = 'Open source libraries exist but no easy-to-use web tool - wrap library in simple UI';
-  }
-
-  return {
-    hasFreeSolutions,
-    hasPaidSolutions,
-    dominantPlayer,
-    marketGap,
-  };
-}
-
-/**
- * Search for specific format converters (e.g., "X to Mermaid converter")
- */
-export async function searchFormatConverters(
-  targetFormat: string,
-  options: CompetitiveSearchOptions = {}
-): Promise<{
-  sourceFormats: string[];
-  existingConverters: CompetitorTool[];
-  opportunities: string[];
-}> {
-  // Common diagram/document formats to check
-  const commonFormats = [
-    'flowchart', 'uml', 'plantuml', 'draw.io', 'lucidchart',
-    'visio', 'graphviz', 'dot', 'json', 'yaml', 'csv',
-    'excel', 'markdown', 'text', 'ascii', 'png', 'svg',
-  ];
-
-  const existingConverters: CompetitorTool[] = [];
-  const opportunities: string[] = [];
-
-  for (const format of commonFormats) {
-    const query = `${format} to ${targetFormat} converter`;
-
-    try {
-      const result = await searchCompetitors(query, options);
-
-      if (result.existingTools.length > 0 || result.openSourceOptions.length > 0) {
-        existingConverters.push(...result.existingTools.slice(0, 2));
-        existingConverters.push(...result.openSourceOptions.slice(0, 2));
-      } else {
-        // No converter exists - opportunity!
-        opportunities.push(`${format} to ${targetFormat}`);
-      }
-    } catch {
-      // Skip on error
-    }
-
-    // Rate limit: wait 100ms between searches
-    await new Promise(resolve => setTimeout(resolve, 100));
-  }
-
-  return {
-    sourceFormats: commonFormats,
-    existingConverters,
-    opportunities,
-  };
-}
-
-/**
- * Search Hacker News for tool discussions
- */
-export async function searchHackerNews(
-  query: string,
-  maxResults: number = 10
-): Promise<Array<{
-  title: string;
-  url: string;
-  points: number;
-  comments: number;
-}>> {
-  const response = await fetch(
-    `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(query)}&tags=story&hitsPerPage=${maxResults}`,
-    {
-      headers: {
-        'User-Agent': 'RankCLI-Keyword-Research',
-      },
-    }
-  );
-
-  if (!response.ok) {
-    throw new Error(`HN API error: ${response.status}`);
-  }
-
-  const data = await response.json();
-
-  return (data.hits || []).map((hit: any) => ({
-    title: hit.title,
-    url: hit.url || `https://news.ycombinator.com/item?id=${hit.objectID}`,
-    points: hit.points || 0,
-    comments: hit.num_comments || 0,
-  }));
-}