ruby_llm 1.0.1 → 1.1.0rc1

data/lib/tasks/browser_helper.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'ferrum'
+require_relative 'cli_helper'
+
+class BrowserHelper # rubocop:disable Style/Documentation
+  REALISTIC_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36' # rubocop:disable Layout/LineLength
+
+  def initialize
+    @browser = create_browser
+  end
+
+  def goto(url)
+    @browser.goto(url)
+  end
+
+  def current_url
+    @browser.page.url
+  rescue StandardError
+    'N/A'
+  end
+
+  def get_page_content(context = 'current page') # rubocop:disable Metrics/MethodLength
+    puts "    Extracting HTML for #{context}..."
+
+    begin
+      sleep(1.0) # Small delay for page stability
+      html = @browser.body
+
+      if html && !html.empty?
+        puts "      Extracted ~#{html.length} chars of HTML"
+        puts '      WARNING: Challenge page detected' if html.match?(/challenge-platform|Checking site/)
+        html
+      else
+        puts '      Warning: Empty content returned'
+        ''
+      end
+    rescue StandardError => e
+      puts "      Error getting HTML: #{e.class} - #{e.message}"
+      ''
+    end
+  end
+
+  def wait_for_page_load
+    handle_cloudflare_challenge
+  end
+
+  def close
+    puts "\nClosing browser..."
+    @browser.quit
+  rescue StandardError => e
+    puts "  Warning: Error closing browser: #{e.message}"
+  end
+
+  private
+
+  def create_browser
+    puts '  Initializing browser for manual interaction...'
+
+    Ferrum::Browser.new(
+      window_size: [1366, 768],
+      headless: false,
+      browser_options: browser_options,
+      timeout: 120,
+      process_timeout: 120,
+      pending_connection_errors: false
+    )
+  end
+
+  def browser_options
+    {
+      'user-agent' => REALISTIC_USER_AGENT,
+      'disable-gpu' => nil,
+      'no-sandbox' => nil,
+      'disable-blink-features' => 'AutomationControlled',
+      'disable-infobars' => nil,
+      'start-maximized' => nil
+    }
+  end
+
+  def handle_cloudflare_challenge # rubocop:disable Metrics/MethodLength
+    puts "\nWaiting for Cloudflare challenge resolution..."
+    puts 'c: Challenge solved'
+    puts 'q: Quit/Skip'
+
+    choice = CliHelper.get_user_choice('Confirm when ready', %w[c q])
+    return false if choice == 'q'
+
+    begin
+      @browser.page.target_id
+      true
+    rescue StandardError
+      puts 'Browser check failed after challenge'
+      false
+    end
+  end
+end

data/lib/tasks/capability_generator.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'ruby_llm'
+require 'json'
+require_relative 'code_validator'
+
+class CapabilityGenerator # rubocop:disable Style/Documentation
+  def initialize(provider_name, docs_html)
+    @provider_name = provider_name
+    @docs_html = docs_html
+    @processed_html = process_html(docs_html)
+  end
+
+  def generate_and_save # rubocop:disable Metrics/MethodLength
+    puts "  Starting code generation for #{@provider_name}..."
+
+    existing_path = File.expand_path("../../lib/ruby_llm/providers/#{@provider_name}/capabilities.rb", __dir__)
+    unless File.exist?(existing_path)
+      puts "  Skipping: No file at #{existing_path}"
+      return
+    end
+
+    existing_code = File.read(existing_path)
+    puts '  Read existing code'
+
+    generated_code = generate_capabilities(existing_code)
+
+    if generated_code
+      puts "  Writing updated code to #{existing_path}..."
+      File.write(existing_path, generated_code)
+      puts "  Updated #{@provider_name}"
+
+      verify_code_with_models_update
+    else
+      puts "  Failed to generate valid code for #{@provider_name}"
+    end
+  end
+
+  private
+
+  def process_html(docs_html) # rubocop:disable Metrics/MethodLength
+    docs_html.transform_values do |html|
+      next '' if html.nil? || html.empty?
+
+      # Extract just the main content areas, skip scripts, styles, etc
+      main_content = html.scan(%r{<main.*?>.*?</main>}m).first ||
+                     html.scan(%r{<article.*?>.*?</article>}m).first ||
+                     html.scan(%r{<div class="content.*?>.*?</div>}m).first
+
+      if main_content
+        # Further clean up the content
+        main_content.gsub(%r{<script.*?>.*?</script>}m, '')
+                    .gsub(%r{<style.*?>.*?</style>}m, '')
+                    .gsub(/<!--.*?-->/m, '')
+                    .gsub(/\s+/, ' ')
+                    .strip
+      else
+        ''
+      end
+    end
+  end
+
+  def generate_capabilities(existing_code) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    max_retries = 3
+    retries = 0
+
+    loop do # rubocop:disable Metrics/BlockLength
+      puts "  Attempt #{retries + 1}/#{max_retries}..."
+
+      begin
+        gemini = RubyLLM.chat(model: 'gemini-1.5-pro-latest')
+                        .with_temperature(0.1)
+
+        docs_json = JSON.pretty_generate(@processed_html)
+
+        prompt = <<~PROMPT
+          Update RubyLLM::Providers::#{@provider_name.capitalize}::Capabilities module.
+          Only use the provided HTML content and existing code structure.
+          Focus on updating values while preserving the module structure.
+
+          Existing code to maintain structure:
+          ```ruby
+          #{existing_code}
+          ```
+
+          HTML content to extract new values from:
+          ```json
+          #{docs_json}
+          ```
+
+          Return ONLY the complete Ruby code within ```ruby ``` tags.
+        PROMPT
+
+        response = gemini.ask(prompt)
+        generated_code = CodeValidator.extract_code_from_response(response.content)
+
+        return generated_code if generated_code && CodeValidator.validate_syntax(generated_code)
+      rescue RubyLLM::BadRequestError => e
+        puts "  Error: #{e.message}"
+        # Try with even less content if we hit token limits
+        @processed_html = @processed_html.transform_values { |html| html[0..10_000] }
+      rescue StandardError => e
+        puts "  Error: #{e.class} - #{e.message}"
+      end
+
+      retries += 1
+      break if retries >= max_retries
+    end
+
+    nil
+  end
+
+  def verify_code_with_models_update
+    puts '  Verifying with models:update...'
+    begin
+      Rake::Task['models:update'].reenable
+      Rake::Task['models:update'].invoke
+      puts '  Verification successful'
+    rescue StandardError => e
+      puts "  Verification failed: #{e.message}"
+    end
+  end
+end

data/lib/tasks/capability_scraper.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require 'ruby_llm'
+require 'uri'
+require_relative 'browser_helper'
+require_relative 'cli_helper'
+
+class CapabilityScraper # rubocop:disable Metrics/ClassLength,Style/Documentation
+  PROVIDER_DOCS = {
+    openai: {
+      url: 'https://platform.openai.com/docs/models'
+    },
+    anthropic: {
+      url: 'https://docs.anthropic.com/claude/docs/models-overview'
+    },
+    gemini: {
+      url: 'https://ai.google.dev/models/gemini',
+      pricing_url: 'https://ai.google.dev/pricing'
+    },
+    deepseek: {
+      url: 'https://api-docs.deepseek.com/quick_start/pricing'
+    }
+  }.freeze
+
+  def self.parse_providers(providers_arg) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+    provider_input = (providers_arg || '').downcase.split(',').map(&:strip).reject(&:empty?)
+
+    target_providers = if provider_input.empty? || provider_input.include?('all')
+                         PROVIDER_DOCS.keys
+                       else
+                         provider_input.map(&:to_sym).select { |p| PROVIDER_DOCS.key?(p) }
+                       end
+
+    if target_providers.empty?
+      puts "No valid providers. Available: all, #{PROVIDER_DOCS.keys.join(', ')}"
+      exit(1)
+    end
+
+    puts "Targeting providers: #{target_providers.join(', ')}"
+    target_providers
+  end
+
+  def initialize(providers)
+    @providers = providers
+    @browser_helper = BrowserHelper.new
+  end
+
+  def run # rubocop:disable Metrics/MethodLength
+    @providers.each do |provider_sym|
+      provider_name = provider_sym.to_s
+      puts "\n======== Processing Provider: #{provider_name.upcase} =========="
+
+      docs_info = PROVIDER_DOCS[provider_sym]
+      docs_html = scrape_provider(provider_sym, docs_info)
+
+      next unless docs_html.any? { |_k, v| v && !v.strip.empty? }
+
+      summarize_html_data(docs_html)
+      yield(provider_name, docs_html) if block_given?
+    end
+  ensure
+    @browser_helper&.close
+  end
+
+  private
+
+  def scrape_provider(provider_sym, docs_info)
+    if provider_sym == :openai
+      handle_openai_scraping(docs_info[:url])
+    else
+      handle_standard_scraping(provider_sym, docs_info)
+    end
+  end
+
+  def handle_openai_scraping(main_url) # rubocop:disable Metrics/MethodLength
+    main_url_path = extract_path_from_url(main_url)
+    overview_key = generate_key_from_url(main_url, main_url_path)
+    docs_html = {}
+
+    puts "\n--- Interactive Scraping for OpenAI ---"
+    puts "  Navigating to main OpenAI models page: #{main_url}..."
+
+    begin
+      @browser_helper.goto(main_url)
+      return docs_html unless @browser_helper.wait_for_page_load
+
+      puts '  Scraping overview page HTML...'
+      overview_html = @browser_helper.get_page_content('OpenAI Overview')
+      docs_html[overview_key] = overview_html if overview_html && !overview_html.empty?
+
+      # Interactive scraping loop
+      interactive_scraping_loop(docs_html, main_url, main_url_path, overview_key)
+    rescue StandardError => e
+      puts "FATAL: Navigation failed: #{e}"
+    end
+
+    docs_html
+  end
+
+  def interactive_scraping_loop(docs_html, main_url, main_url_path, overview_key) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    loop do # rubocop:disable Metrics/BlockLength
+      current_url = begin
+        @browser_helper.current_url
+      rescue StandardError
+        'N/A'
+      end
+      puts "\n--- OpenAI Subpage Scraping ---"
+      puts "Current URL: #{current_url}"
+      puts "Collected Keys: #{docs_html.keys.sort.join(', ')}"
+      puts '---------------------------------'
+
+      puts 'Ready to scrape current page. Options:'
+      puts 's: Scrape current page'
+      puts 'b: Back to overview'
+      puts 'q: Quit scraping'
+
+      choice = CliHelper.get_user_choice('Choose action', %w[s b q])
+
+      case choice
+      when 's'
+        scrape_current_page(docs_html, main_url_path, overview_key)
+      when 'b'
+        puts "  Navigating back to overview: #{main_url}..."
+        begin
+          @browser_helper.goto(main_url)
+          @browser_helper.wait_for_page_load
+        rescue StandardError => e
+          puts "  Error navigating back: #{e.message}. Navigate manually."
+        end
+      when 'q'
+        puts '  Finished OpenAI scraping.'
+        break
+      end
+    end
+  end
+
+  def scrape_current_page(docs_html, main_url_path, overview_key) # rubocop:disable Metrics/MethodLength
+    current_url = begin
+      @browser_helper.current_url
+    rescue StandardError
+      'N/A'
+    end
+    puts "  Scraping content from: #{current_url}"
+    page_html = @browser_helper.get_page_content(current_url)
+
+    if page_html && !page_html.empty?
+      page_key = generate_key_from_url(current_url, main_url_path)
+      if page_key != overview_key || !docs_html.key?(overview_key)
+        docs_html[page_key] = page_html
+        puts "  Stored HTML under key: #{page_key}"
+      else
+        puts '  Note: Back on overview page, skipping.'
+      end
+    end
+
+    next_choice = CliHelper.get_user_choice('Continue? (y: scrape another, n: quit)', %w[y n])
+    false if next_choice == 'n'
+  end
+
+  def handle_standard_scraping(provider, info)
+    provider_name = provider.to_s
+    docs_html = {}
+
+    puts "\n--- Standard Scraping for #{provider_name.upcase} ---"
+
+    # Main page scraping
+    scrape_url(provider_name, info[:url], "#{provider_name}_main_html", docs_html)
+
+    # Pricing page scraping (if available)
+    if info[:pricing_url] && info[:pricing_url] != info[:url]
+      scrape_url(provider_name, info[:pricing_url], "#{provider_name}_pricing_html", docs_html)
+    end
+
+    docs_html
+  end
+
+  def scrape_url(provider_name, url, key_name, docs_html)
+    puts "  Scraping #{key_name}: #{url}..."
+
+    begin
+      @browser_helper.goto(url)
+      return unless @browser_helper.wait_for_page_load
+
+      html_content = @browser_helper.get_page_content("#{provider_name} #{key_name}")
+      docs_html[key_name] = html_content if html_content && !html_content.empty?
+    rescue StandardError => e
+      puts "  Error on #{key_name}: #{e.message}"
+    end
+  end
+
+  def extract_path_from_url(url_string)
+    URI(url_string).path
+  rescue StandardError
+    '/docs/models'
+  end
+
+  def generate_key_from_url(url_string, base_url_path = '/docs/models')
+    uri = URI(url_string)
+    path = uri.path.chomp('/')
+    return 'models_overview_html' if path == base_url_path
+
+    key_part = path.split('/').reject(&:empty?).last || 'unknown'
+    "model_#{key_part.gsub(/[^a-z0-9_\-]/i, '_')}_html"
+  rescue StandardError => e
+    puts "    Warning: URL parsing failed for: #{url_string} (#{e.message})"
+    "scrape_#{Time.now.to_i}_html"
+  end
+
+  def summarize_html_data(docs_html) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+    docs_summary = docs_html.map do |k, v|
+      desc = if v.nil? || v.strip.empty?
+               '(Empty)'
+             elsif v.include?('challenge') || v.include?('Checking')
+               '(Challenge?)'
+             else
+               '(OK)'
+             end
+      "#{k}: ~#{v&.length || 0} chars #{desc}"
+    end.join(', ')
+
+    puts "\n  HTML Summary: #{docs_summary}"
+    puts '  Warning: LLM results may be inaccurate.' if docs_summary.include?('(Challenge?)')
+  end
+end

data/lib/tasks/cli_helper.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'io/console'
+
+class CliHelper # rubocop:disable Style/Documentation
+  def self.get_user_choice(prompt, valid_choices) # rubocop:disable Metrics/MethodLength
+    loop do
+      print "\n#{prompt} [#{valid_choices.join('/')}]: "
+      begin
+        choice = $stdin.getch.downcase
+        puts choice # Echo the character
+
+        return choice if valid_choices.include?(choice)
+
+        puts "  Invalid choice '#{choice}'. Valid options: #{valid_choices.join(', ')}"
+      rescue Interrupt
+        puts "\n*** Input interrupted."
+        return 'q'
+      end
+    end
+  end
+end

data/lib/tasks/code_validator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'ripper'
+
+class CodeValidator # rubocop:disable Style/Documentation
+  # Use Ruby's built-in Ripper to validate Ruby syntax instead of eval
+  def self.validate_syntax(code)
+    # Ripper.sexp returns nil if there's a syntax error
+    result = Ripper.sexp(code)
+
+    if result.nil?
+      puts 'Syntax error in generated code'
+      false
+    else
+      # Additional validation could go here
+      true
+    end
+  end
+
+  def self.extract_code_from_response(content)
+    match = content.match(/```ruby\s*(.*?)\s*```/m)
+    if match
+      match[1].strip
+    else
+      puts 'No Ruby code block found in response'
+      nil
+    end
+  end
+end

data/lib/tasks/model_updater.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'ruby_llm'
+
+class ModelUpdater # rubocop:disable Style/Documentation
+  def run
+    puts 'Configuring RubyLLM...'
+    configure_from_env
+
+    refresh_models
+    display_model_stats
+  end
+
+  private
+
+  def configure_from_env
+    RubyLLM.configure do |config|
+      config.openai_api_key = ENV.fetch('OPENAI_API_KEY', nil)
+      config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', nil)
+      config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', nil)
+      config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil)
+      configure_bedrock(config)
+      config.request_timeout = 30
+    end
+  end
+
+  def configure_bedrock(config)
+    config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil)
+    config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil)
+    config.bedrock_region = ENV.fetch('AWS_REGION', nil)
+    config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil)
+  end
+
+  def refresh_models # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    initial_count = RubyLLM.models.all.size
+    puts "Refreshing models (#{initial_count} cached)..."
+
+    models = RubyLLM.models.refresh!
+
+    if models.all.empty? && initial_count.zero?
+      puts 'Error: Failed to fetch models.'
+      exit(1)
+    elsif models.all.size == initial_count && initial_count.positive?
+      puts 'Warning: Model list unchanged.'
+    else
+      puts "Saving models.json (#{models.all.size} models)"
+      models.save_models
+    end
+
+    @models = models
+  end
+
+  def display_model_stats
+    puts "\nModel count:"
+    provider_counts = @models.all.group_by(&:provider).transform_values(&:count)
+
+    RubyLLM::Provider.providers.each_key do |sym|
+      name = sym.to_s.capitalize
+      count = provider_counts[sym.to_s] || 0
+      status = RubyLLM::Provider.providers[sym].configured? ? '(OK)' : '(SKIP)'
+      puts "  #{name}: #{count} models #{status}"
+    end
+
+    puts 'Refresh complete.'
+  end
+end