ruby_llm 1.2.0 → 1.3.0

data/lib/tasks/capability_scraper.rb

@@ -1,224 +0,0 @@
-# frozen_string_literal: true
-
-require 'ruby_llm'
-require 'uri'
-require_relative 'browser_helper'
-require_relative 'cli_helper'
-
-class CapabilityScraper # rubocop:disable Metrics/ClassLength,Style/Documentation
-  PROVIDER_DOCS = {
-    openai: {
-      url: 'https://platform.openai.com/docs/models'
-    },
-    anthropic: {
-      url: 'https://docs.anthropic.com/claude/docs/models-overview'
-    },
-    gemini: {
-      url: 'https://ai.google.dev/models/gemini',
-      pricing_url: 'https://ai.google.dev/pricing'
-    },
-    deepseek: {
-      url: 'https://api-docs.deepseek.com/quick_start/pricing'
-    }
-  }.freeze
-
-  def self.parse_providers(providers_arg) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
-    provider_input = (providers_arg || '').downcase.split(',').map(&:strip).reject(&:empty?)
-
-    target_providers = if provider_input.empty? || provider_input.include?('all')
-                         PROVIDER_DOCS.keys
-                       else
-                         provider_input.map(&:to_sym).select { |p| PROVIDER_DOCS.key?(p) }
-                       end
-
-    if target_providers.empty?
-      puts "No valid providers. Available: all, #{PROVIDER_DOCS.keys.join(', ')}"
-      exit(1)
-    end
-
-    puts "Targeting providers: #{target_providers.join(', ')}"
-    target_providers
-  end
-
-  def initialize(providers)
-    @providers = providers
-    @browser_helper = BrowserHelper.new
-  end
-
-  def run # rubocop:disable Metrics/MethodLength
-    @providers.each do |provider_sym|
-      provider_name = provider_sym.to_s
-      puts "\n======== Processing Provider: #{provider_name.upcase} =========="
-
-      docs_info = PROVIDER_DOCS[provider_sym]
-      docs_html = scrape_provider(provider_sym, docs_info)
-
-      next unless docs_html.any? { |_k, v| v && !v.strip.empty? }
-
-      summarize_html_data(docs_html)
-      yield(provider_name, docs_html) if block_given?
-    end
-  ensure
-    @browser_helper&.close
-  end
-
-  private
-
-  def scrape_provider(provider_sym, docs_info)
-    if provider_sym == :openai
-      handle_openai_scraping(docs_info[:url])
-    else
-      handle_standard_scraping(provider_sym, docs_info)
-    end
-  end
-
-  def handle_openai_scraping(main_url) # rubocop:disable Metrics/MethodLength
-    main_url_path = extract_path_from_url(main_url)
-    overview_key = generate_key_from_url(main_url, main_url_path)
-    docs_html = {}
-
-    puts "\n--- Interactive Scraping for OpenAI ---"
-    puts "  Navigating to main OpenAI models page: #{main_url}..."
-
-    begin
-      @browser_helper.goto(main_url)
-      return docs_html unless @browser_helper.wait_for_page_load
-
-      puts '  Scraping overview page HTML...'
-      overview_html = @browser_helper.get_page_content('OpenAI Overview')
-      docs_html[overview_key] = overview_html if overview_html && !overview_html.empty?
-
-      # Interactive scraping loop
-      interactive_scraping_loop(docs_html, main_url, main_url_path, overview_key)
-    rescue StandardError => e
-      puts "FATAL: Navigation failed: #{e}"
-    end
-
-    docs_html
-  end
-
-  def interactive_scraping_loop(docs_html, main_url, main_url_path, overview_key) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-    loop do # rubocop:disable Metrics/BlockLength
-      current_url = begin
-        @browser_helper.current_url
-      rescue StandardError
-        'N/A'
-      end
-      puts "\n--- OpenAI Subpage Scraping ---"
-      puts "Current URL: #{current_url}"
-      puts "Collected Keys: #{docs_html.keys.sort.join(', ')}"
-      puts '---------------------------------'
-
-      puts 'Ready to scrape current page. Options:'
-      puts 's: Scrape current page'
-      puts 'b: Back to overview'
-      puts 'q: Quit scraping'
-
-      choice = CliHelper.get_user_choice('Choose action', %w[s b q])
-
-      case choice
-      when 's'
-        scrape_current_page(docs_html, main_url_path, overview_key)
-      when 'b'
-        puts "  Navigating back to overview: #{main_url}..."
-        begin
-          @browser_helper.goto(main_url)
-          @browser_helper.wait_for_page_load
-        rescue StandardError => e
-          puts "  Error navigating back: #{e.message}. Navigate manually."
-        end
-      when 'q'
-        puts '  Finished OpenAI scraping.'
-        break
-      end
-    end
-  end
-
-  def scrape_current_page(docs_html, main_url_path, overview_key) # rubocop:disable Metrics/MethodLength
-    current_url = begin
-      @browser_helper.current_url
-    rescue StandardError
-      'N/A'
-    end
-    puts "  Scraping content from: #{current_url}"
-    page_html = @browser_helper.get_page_content(current_url)
-
-    if page_html && !page_html.empty?
-      page_key = generate_key_from_url(current_url, main_url_path)
-      if page_key != overview_key || !docs_html.key?(overview_key)
-        docs_html[page_key] = page_html
-        puts "  Stored HTML under key: #{page_key}"
-      else
-        puts '  Note: Back on overview page, skipping.'
-      end
-    end
-
-    next_choice = CliHelper.get_user_choice('Continue? (y: scrape another, n: quit)', %w[y n])
-    false if next_choice == 'n'
-  end
-
-  def handle_standard_scraping(provider, info)
-    provider_name = provider.to_s
-    docs_html = {}
-
-    puts "\n--- Standard Scraping for #{provider_name.upcase} ---"
-
-    # Main page scraping
-    scrape_url(provider_name, info[:url], "#{provider_name}_main_html", docs_html)
-
-    # Pricing page scraping (if available)
-    if info[:pricing_url] && info[:pricing_url] != info[:url]
-      scrape_url(provider_name, info[:pricing_url], "#{provider_name}_pricing_html", docs_html)
-    end
-
-    docs_html
-  end
-
-  def scrape_url(provider_name, url, key_name, docs_html)
-    puts "  Scraping #{key_name}: #{url}..."
-
-    begin
-      @browser_helper.goto(url)
-      return unless @browser_helper.wait_for_page_load
-
-      html_content = @browser_helper.get_page_content("#{provider_name} #{key_name}")
-      docs_html[key_name] = html_content if html_content && !html_content.empty?
-    rescue StandardError => e
-      puts "  Error on #{key_name}: #{e.message}"
-    end
-  end
-
-  def extract_path_from_url(url_string)
-    URI(url_string).path
-  rescue StandardError
-    '/docs/models'
-  end
-
-  def generate_key_from_url(url_string, base_url_path = '/docs/models')
-    uri = URI(url_string)
-    path = uri.path.chomp('/')
-    return 'models_overview_html' if path == base_url_path
-
-    key_part = path.split('/').reject(&:empty?).last || 'unknown'
-    "model_#{key_part.gsub(/[^a-z0-9_\-]/i, '_')}_html"
-  rescue StandardError => e
-    puts "    Warning: URL parsing failed for: #{url_string} (#{e.message})"
-    "scrape_#{Time.now.to_i}_html"
-  end
-
-  def summarize_html_data(docs_html) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
-    docs_summary = docs_html.map do |k, v|
-      desc = if v.nil? || v.strip.empty?
-               '(Empty)'
-             elsif v.include?('challenge') || v.include?('Checking')
-               '(Challenge?)'
-             else
-               '(OK)'
-             end
-      "#{k}: ~#{v&.length || 0} chars #{desc}"
-    end.join(', ')
-
-    puts "\n  HTML Summary: #{docs_summary}"
-    puts '  Warning: LLM results may be inaccurate.' if docs_summary.include?('(Challenge?)')
-  end
-end

data/lib/tasks/cli_helper.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-require 'io/console'
-
-class CliHelper # rubocop:disable Style/Documentation
-  def self.get_user_choice(prompt, valid_choices) # rubocop:disable Metrics/MethodLength
-    loop do
-      print "\n#{prompt} [#{valid_choices.join('/')}]: "
-      begin
-        choice = $stdin.getch.downcase
-        puts choice # Echo the character
-
-        return choice if valid_choices.include?(choice)
-
-        puts "  Invalid choice '#{choice}'. Valid options: #{valid_choices.join(', ')}"
-      rescue Interrupt
-        puts "\n*** Input interrupted."
-        return 'q'
-      end
-    end
-  end
-end

data/lib/tasks/code_validator.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-require 'ripper'
-
-class CodeValidator # rubocop:disable Style/Documentation
-  # Use Ruby's built-in Ripper to validate Ruby syntax instead of eval
-  def self.validate_syntax(code)
-    # Ripper.sexp returns nil if there's a syntax error
-    result = Ripper.sexp(code)
-
-    if result.nil?
-      puts 'Syntax error in generated code'
-      false
-    else
-      # Additional validation could go here
-      true
-    end
-  end
-
-  def self.extract_code_from_response(content)
-    match = content.match(/```ruby\s*(.*?)\s*```/m)
-    if match
-      match[1].strip
-    else
-      puts 'No Ruby code block found in response'
-      nil
-    end
-  end
-end

data/lib/tasks/model_updater.rb

@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-require 'ruby_llm'
-
-class ModelUpdater # rubocop:disable Style/Documentation
-  def run
-    puts 'Configuring RubyLLM...'
-    configure_from_env
-
-    refresh_models
-    display_model_stats
-  end
-
-  private
-
-  def configure_from_env
-    RubyLLM.configure do |config|
-      config.openai_api_key = ENV.fetch('OPENAI_API_KEY', nil)
-      config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', nil)
-      config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', nil)
-      config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil)
-      configure_bedrock(config)
-      config.request_timeout = 30
-    end
-  end
-
-  def configure_bedrock(config)
-    config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil)
-    config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil)
-    config.bedrock_region = ENV.fetch('AWS_REGION', nil)
-    config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil)
-  end
-
-  def refresh_models # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-    initial_count = RubyLLM.models.all.size
-    puts "Refreshing models (#{initial_count} cached)..."
-
-    models = RubyLLM.models.refresh!
-
-    if models.all.empty? && initial_count.zero?
-      puts 'Error: Failed to fetch models.'
-      exit(1)
-    elsif models.all.size == initial_count && initial_count.positive?
-      puts 'Warning: Model list unchanged.'
-    else
-      puts "Saving models.json (#{models.all.size} models)"
-      models.save_models
-    end
-
-    @models = models
-  end
-
-  def display_model_stats
-    puts "\nModel count:"
-    provider_counts = @models.all.group_by(&:provider).transform_values(&:count)
-
-    RubyLLM::Provider.providers.each_key do |sym|
-      name = sym.to_s.capitalize
-      count = provider_counts[sym.to_s] || 0
-      status = RubyLLM::Provider.providers[sym].configured? ? '(OK)' : '(SKIP)'
-      puts "  #{name}: #{count} models #{status}"
-    end
-
-    puts 'Refresh complete.'
-  end
-end

data/lib/tasks/models.rake

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'model_updater'
-require_relative 'capability_scraper'
-require_relative 'capability_generator'
-
-namespace :models do # rubocop:disable Metrics/BlockLength
-  desc 'Update available models from providers (API keys needed)'
-  task :update do
-    ModelUpdater.new.run
-  end
-
-  desc 'Update capabilities modules (GEMINI_API_KEY needed)'
-  task :update_capabilities, [:providers] do |_t, args|
-    gemini_key = ENV.fetch('GEMINI_API_KEY', nil)
-    unless gemini_key && !gemini_key.empty?
-      puts 'Error: GEMINI_API_KEY required'
-      exit(1)
-    end
-
-    RubyLLM.configure do |c|
-      c.gemini_api_key = gemini_key
-      c.request_timeout = 300
-    end
-
-    target_providers = CapabilityScraper.parse_providers(args[:providers])
-
-    begin
-      scraper = CapabilityScraper.new(target_providers)
-      scraper.run do |provider, docs_html|
-        generator = CapabilityGenerator.new(provider, docs_html)
-        generator.generate_and_save
-      end
-    rescue StandardError => e
-      puts "Error: #{e.message}"
-      puts e.backtrace.first(5).join("\n")
-    ensure
-      puts 'Update process complete. Review generated files.'
-    end
-  end
-end
-
-task default: ['models:update']