llms 0.1.0

data/lib/llms/conversation_tool_result.rb

@@ -0,0 +1,15 @@
+module LLMs
+  class ConversationToolResult
+    attr_reader :index, :tool_call_id, :tool_call_type, :name, :results, :is_error
+
+    def initialize(index, tool_call_id, tool_call_type, name, results, is_error)
+      raise "index is nil" if index.nil?
+      @index = index
+      @tool_call_id = tool_call_id
+      @tool_call_type = tool_call_type
+      @name = name
+      @results = results
+      @is_error = is_error
+    end
+  end
+end 

data/lib/llms/exceptions.rb

@@ -0,0 +1,33 @@
+module LLMs
+  # Base exception class for all LLMs errors
+  class Error < StandardError; end
+
+  # Configuration errors
+  class ConfigurationError < Error; end
+  class MissingAPIKeyError < ConfigurationError; end
+  class InvalidModelError < ConfigurationError; end
+  class UnsupportedFeatureError < ConfigurationError; end
+  class ModelNotFoundError < ConfigurationError; end
+  class ProviderNotFoundError < ConfigurationError; end
+
+  # API communication errors
+  class APIError < Error; end
+  class RateLimitError < APIError; end
+  class TimeoutError < APIError; end
+  class NetworkError < APIError; end
+  class AuthenticationError < APIError; end
+
+  # Usage and cost calculation errors
+  class UsageError < Error; end
+  class CostCalculationError < UsageError; end
+
+  # Tool-related errors
+  class ToolError < Error; end
+  class ToolExecutionError < ToolError; end
+  class ToolValidationError < ToolError; end
+
+  # Conversation and message errors
+  class ConversationError < Error; end
+  class MessageError < ConversationError; end
+  class InvalidMessageRoleError < MessageError; end
+end 

data/lib/llms/executors/anthropic_executor.rb

@@ -0,0 +1,247 @@
+require_relative './base_executor'
+require_relative '../adapters/anthropic_message_adapter'
+require_relative '../parsers/anthropic_chat_response_stream_parser'
+require_relative '../stream/event_emitter'
+require 'anthropic'
+
+module LLMs
+  module Executors
+    class AnthropicExecutor < BaseExecutor
+
+      def execute_conversation(conversation, &block)
+        if block_given?
+          stream_conversation(conversation) do |handler|
+            handler.on(:text_delta) do |event|
+              yield event.text
+            end
+            ## TODO configure whether to yield thinking deltas
+            handler.on(:thinking_delta) do |event|
+              yield event.thinking
+            end
+          end
+        else
+          send_conversation(conversation)
+        end
+      end
+
+      def stream_conversation(conversation)
+        init_new_request(conversation)
+
+        emitter = Stream::EventEmitter.new
+        yield emitter if block_given?
+
+        start_time = Time.now
+        begin
+          http_response, stream_parsed_response = stream_client_request(emitter)
+        rescue Faraday::BadRequestError => e
+          @last_error = e.response[:body]
+          return nil
+        end
+        execution_time = Time.now - start_time
+
+        api_response = stream_parsed_response || http_response
+
+        if api_response['error']
+          @last_error = api_response['error']
+          return nil
+        end
+
+        @last_received_message_id = LLMs::Adapters::AnthropicMessageAdapter.find_message_id(api_response)
+        @last_received_message = LLMs::Adapters::AnthropicMessageAdapter.message_from_api_format(api_response)
+        @last_usage_data = calculate_usage(api_response, execution_time)
+
+        @last_received_message
+      end
+
+      def send_conversation(conversation)
+        init_new_request(conversation)
+
+        start_time = Time.now
+        begin        
+          http_response = client_request
+        rescue Faraday::BadRequestError => e        
+          @last_error = e.response[:body]
+          return nil
+        end
+        execution_time = Time.now - start_time
+
+        if http_response['error']
+          @last_error = http_response['error']
+          return nil
+        end
+
+        @last_received_message_id = LLMs::Adapters::AnthropicMessageAdapter.find_message_id(http_response)
+        @last_received_message = LLMs::Adapters::AnthropicMessageAdapter.message_from_api_format(http_response)
+        @last_usage_data = calculate_usage(http_response, execution_time)
+
+        @last_received_message
+      end
+
+      private
+
+      def init_new_request(conversation)
+        @last_sent_message = conversation.last_message
+        @last_received_message_id = nil
+        @last_received_message = nil
+        @last_usage_data = nil
+        @last_error = nil
+
+        @system_prompt = conversation.system_message
+        @available_tools = conversation.available_tools
+        @formatted_messages = conversation.messages.map.with_index do |message, index|
+          is_last_message = index == conversation.messages.size - 1
+          LLMs::Adapters::AnthropicMessageAdapter.to_api_format(message, caching_enabled? && is_last_message)
+        end
+
+        # Figure out where to put the cache control param if no messages are provided
+        # In reality there should always be a message, but we'll check
+        if caching_enabled? && @formatted_messages.empty?
+          if @available_tools && @available_tools.any?
+            @available_tools.last[:cache_control] = {type: "ephemeral"}
+          elsif @system_prompt && (@system_prompt.is_a?(String) || !@system_prompt[:cache_control])
+            @system_prompt = {type: "text", text: @system_prompt, cache_control: {type: "ephemeral"}}
+          end
+        end
+      end
+
+      def request_params
+        {
+          messages: @formatted_messages,
+          model: @model_name,
+          temperature: @temperature,
+        }.tap do |params|
+          if @system_prompt
+            params[:system] = system_param
+          end
+          if @max_tokens
+            params[:max_tokens] = @max_tokens
+          end
+          ## Will override max_tokens if both are provided
+          if @max_completion_tokens
+            params[:max_tokens] = @max_completion_tokens
+          end
+          if @available_tools && @available_tools.any?
+            params[:tools] = tool_schemas
+          end
+          if @thinking_mode
+            params[:thinking] = { type: 'enabled' }.tap do |thinking_params|
+              if @max_thinking_tokens
+                thinking_params[:budget_tokens] = @max_thinking_tokens
+              else
+                # This is the minimum budget for thinking, and is required if thinking is enabled
+                thinking_params[:budget_tokens] = 1024
+              end
+            end
+          end
+        end
+      end
+
+      def client_request
+        @client.messages(parameters: request_params)
+      end
+
+      def stream_client_request(emitter)
+        parser = Parsers::AnthropicChatResponseStreamParser.new(emitter)
+
+        params = request_params.merge({
+          stream: Proc.new do |data|
+            parser.handle_json(data)
+          end
+        })
+
+        [@client.messages(parameters: params), parser.full_response]
+      end
+
+      def initialize_client
+        @client = Anthropic::Client.new(access_token: fetch_api_key)
+      end
+
+      ## TODO move to adapter
+      def tool_schemas
+        @available_tools.map do |tool|
+          {
+            name: tool.tool_schema[:name],
+            description: tool.tool_schema[:description],
+            input_schema: {
+              type: 'object',
+              properties: tool.tool_schema[:parameters][:properties],
+              required: tool.tool_schema[:parameters][:required]
+            }
+          }
+        end
+      end
+
+      def system_param
+        @system_prompt
+      end
+
+      def calculate_usage(api_response, execution_time)
+        input_tokens = nil
+        output_tokens = nil
+        token_counts = {}
+        cache_was_written = nil
+        cache_was_read = nil
+
+        if usage = api_response['usage']
+          input_tokens = 0
+          output_tokens = 0
+          cache_was_written = false
+          cache_was_read = false
+
+          if it = usage['input_tokens']
+            input_tokens += it
+            token_counts[:input] = it
+          end
+          
+          if ccit = usage['cache_creation_input_tokens']
+            input_tokens += ccit
+            if ccit > 0
+              cache_was_written = true
+            end
+          end
+          
+          if crit = usage['cache_read_input_tokens']
+            input_tokens += crit
+            token_counts[:cache_read] = crit
+            if crit > 0
+              cache_was_read = true
+            end
+          end
+          
+          if cache_details = usage['cache_creation']
+            if it1h = cache_details['ephemeral_1h_input_tokens']
+              token_counts[:cache_write_1hr] = it1h
+              if it1h > 0
+                cache_was_written = true
+              end
+            end
+            if it5min = cache_details['ephemeral_5min_input_tokens']
+              token_counts[:cache_write_5min] = it5min
+              if it5min > 0
+                cache_was_written = true
+              end
+            end
+          elsif ccit = usage['cache_creation_input_tokens']
+            # if no details, all caching is 5min
+            token_counts[:cache_write_5min] = ccit
+          end
+          
+          if ot = usage['output_tokens']
+            output_tokens += ot
+            token_counts[:output] = ot
+          end
+        end
+
+        {
+          input_tokens: input_tokens,
+          output_tokens: output_tokens,
+          cache_was_written: cache_was_written,
+          cache_was_read: cache_was_read,
+          token_details: token_counts,
+          execution_time: execution_time,
+          estimated_cost: calculate_cost(token_counts)
+        }
+      end
+    end
+  end
+end

data/lib/llms/executors/base_executor.rb

@@ -0,0 +1,144 @@
+require_relative '../conversation'
+require_relative '../models'
+require_relative '../exceptions'
+
+module LLMs
+  module Executors
+    class BaseExecutor
+
+      DEFAULT_TEMPERATURE = 0.0
+
+      attr_reader :client, :model_name, :base_url,
+              :system_prompt, :available_tools, :temperature,
+              :max_tokens, :max_completion_tokens, :max_thinking_tokens,
+              :thinking_mode, :thinking_effort,
+              :last_sent_message, :last_received_message_id, :last_received_message, :last_usage_data, :last_error
+
+      def initialize(**params)
+        raise LLMs::ConfigurationError, "model_name is required" if params[:model_name].nil?
+
+        @model_name = params[:model_name]
+        
+        # Connection Info
+        @base_url = params[:base_url]
+        @api_key = params[:api_key] ## Will take precedence over an env var if present
+        @api_key_env_var = params[:api_key_env_var]
+        @pricing = params[:pricing]
+        @exclude_params = params[:exclude_params]
+
+        # Execution Info
+        @system_prompt = params[:system_prompt]
+        @temperature = validate_temperature(params[:temperature] || DEFAULT_TEMPERATURE)
+        @available_tools = params[:tools]
+        
+        @cache_prompt = params[:cache_prompt] ## TODO caching is automatic for most models now
+        
+        @max_tokens = validate_positive_integer_or_nil(params[:max_tokens], "max_tokens")
+        @max_completion_tokens = validate_positive_integer_or_nil(params[:max_completion_tokens], "max_completion_tokens")
+        @max_thinking_tokens = validate_positive_integer_or_nil(params[:max_thinking_tokens], "max_thinking_tokens")
+
+        @thinking_mode = params.key?(:thinking) ? params[:thinking] : false
+        @thinking_effort = validate_thinking_effort(params[:thinking_effort])
+
+        ##TODO warn if max_tokens is used instead of max_completion_tokens and model is a thinking model (or thinking is enabled)
+
+        @last_sent_message = nil
+        @last_received_message = nil
+        @last_usage_data = nil
+        @last_error = nil
+
+        initialize_client
+      end
+
+      def execute_prompt(prompt, system_prompt: nil, &block)
+        conversation = LLMs::Conversation.new
+        if sp = system_prompt || @system_prompt
+          conversation.set_system_message(sp)
+        end
+        conversation.add_user_message(prompt)
+        response_message = self.execute_conversation(conversation, &block)
+        response_message&.text
+      end
+
+      def execute_conversation(conversation, &block)
+        raise NotImplementedError, "Subclasses must implement execute_conversation"
+      end
+
+      private
+
+      def fetch_api_key
+        if @api_key
+          @api_key
+        elsif @api_key_env_var
+          ENV[@api_key_env_var] || raise("#{@api_key_env_var} not set")
+        else
+          raise LLMs::ConfigurationError, "No API key provided"
+        end
+      end
+
+      def initialize_client
+        raise NotImplementedError, "Subclasses must implement initialize_client"
+      end
+
+      def tool_schemas
+        raise NotImplementedError, "Subclasses must implement tool_schemas"
+      end
+
+      def caching_enabled?
+        @cache_prompt == true ##TODO caching is automatic by default now for non-Anthropic models that support it
+      end
+
+      def calculate_usage(response)
+        raise NotImplementedError, "Subclasses must implement calculate_usage"
+      end
+
+      def validate_thinking_effort(effort)
+        return if effort.nil?
+
+        if effort.to_s.in?(%w[low medium high])
+          effort.to_s
+        else
+          raise LLMs::ConfigurationError, "Thinking effort must be a string 'low', 'medium', or 'high', got: #{effort}"
+        end
+      end
+
+      def validate_temperature(temp)
+        unless temp.is_a?(Numeric) && temp >= 0.0 && temp <= 2.0
+          raise LLMs::ConfigurationError, "Temperature must be a number between 0.0 and 2.0, got: #{temp}"
+        end
+        temp
+      end
+
+      def validate_positive_integer_or_nil(tokens, name)
+        unless tokens.nil? || (tokens.is_a?(Integer) && tokens > 0)
+          raise LLMs::ConfigurationError, "#{name} must be a positive integer, got: #{tokens}"
+        end
+        tokens
+      end
+
+      def calculate_cost(token_counts)
+        return nil unless @pricing
+        return nil if token_counts.nil? || token_counts.empty?
+
+        token_keys = token_counts.keys.map(&:to_s)
+        pricing_keys = @pricing.keys.map(&:to_s)
+
+        missing_keys = token_keys - pricing_keys
+        ## TODO remove this special case. Is it safe to skip all missing keys if the token count is zero for them?
+        unless missing_keys.empty? || (missing_keys.include?('cached_input') && token_counts[:cached_input] == 0)
+          raise LLMs::CostCalculationError, "Pricing missing key: #{missing_keys.join(', ')}"
+        end
+
+        token_keys.reduce(0.0) do |sum, k|
+          key = k.to_sym
+          if token_counts[key] && token_counts[key] > 0 && @pricing[key]
+            sum + (token_counts[key].to_f / 1_000_000.0) * @pricing[key]
+          else
+            sum
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/llms/executors/google_gemini_executor.rb

@@ -0,0 +1,212 @@
+require_relative 'base_executor'
+require_relative '../apis/google_gemini_api'
+require_relative '../parsers/google_gemini_chat_response_stream_parser'
+require_relative '../adapters/google_gemini_message_adapter'
+
+module LLMs
+  module Executors
+    class GoogleGeminiExecutor < BaseExecutor
+
+      def execute_conversation(conversation, &block)
+        if block_given?
+          stream_conversation(conversation) do |handler|
+            handler.on(:text_delta) do |event|
+              yield event.text
+            end
+          end
+        else
+          send_conversation(conversation)
+        end
+      end
+
+      def stream_conversation(conversation)
+        init_new_request(conversation)
+
+        emitter = Stream::EventEmitter.new
+        yield emitter if block_given?
+
+        start_time = Time.now
+        begin
+          http_response, stream_parsed_response = stream_client_request(emitter)
+        rescue StandardError => e
+          @last_error = {'error' => e.message, 'backtrace' => e.backtrace}
+          return nil
+        end
+        execution_time = Time.now - start_time
+
+        if http_response && http_response['error']
+          @last_error = http_response
+          return nil
+        end
+
+        response_data = stream_parsed_response || http_response
+
+        @last_received_message = LLMs::Adapters::GoogleGeminiMessageAdapter.message_from_api_format(response_data)
+        @last_usage_data = calculate_usage(response_data, execution_time)
+
+        @last_received_message
+      end
+
+      def send_conversation(conversation)
+        init_new_request(conversation)
+
+        start_time = Time.now
+        begin
+          http_response = client_request
+        rescue StandardError => e
+          @last_error = {'error' => e.message, 'backtrace' => e.backtrace}
+          return nil
+        end
+        execution_time = Time.now - start_time
+
+        if http_response && http_response['error']
+          @last_error = http_response
+          return nil
+        end
+
+        @last_received_message = LLMs::Adapters::GoogleGeminiMessageAdapter.message_from_api_format(http_response)
+        if @last_received_message.nil?
+          @last_error = {'error' => 'No message found in the response. Can happen with thinking models if max_tokens is too low.'}
+          return nil
+        end
+
+        @last_received_message_id = "gemini-#{Time.now.to_i}" ## no message id in the response
+        @last_usage_data = calculate_usage(http_response, execution_time)
+
+
+        @last_received_message
+      end
+
+      private
+
+      def init_new_request(conversation)
+        @last_sent_message = conversation.last_message
+        @last_received_message = nil
+        @last_usage_data = nil
+        @last_error = nil
+
+        @formatted_messages = conversation.messages.map do |message|
+          LLMs::Adapters::GoogleGeminiMessageAdapter.to_api_format(message)
+        end
+
+        @available_tools = conversation.available_tools
+      end
+
+      ## TODO we are not inserting fake message_ids nor fake tool_call_ids for the response data, only in stremed events
+
+      def client_request
+        @client.generate_content(@model_name, @formatted_messages, request_params)
+      end
+
+      def stream_client_request(emitter)
+        parser = Parsers::GoogleGeminiChatResponseStreamParser.new(emitter)
+
+        params = request_params.merge(stream: Proc.new { |chunk| parser.add_data(chunk) })
+        http_response = @client.generate_content(@model_name, @formatted_messages, params)
+
+        @last_received_message_id = parser.current_message_id ##no message id in the response
+
+        [http_response, parser.full_response]
+      end
+
+      def request_params
+        generation_config = { temperature: @temperature }.tap do |config|
+          if @max_tokens
+            config[:maxOutputTokens] = @max_tokens
+          end
+          # Will override max_tokens if both are provided
+          if @max_completion_tokens
+            config[:maxOutputTokens] = @max_completion_tokens
+          end
+          if @thinking_mode
+            config[:thinkingConfig] = { includeThoughts: true }.tap do |thinking_config|
+              if @max_thinking_tokens
+                thinking_config[:thinkingBudget] = @max_thinking_tokens
+              end
+            end
+          end
+        end
+
+        { generationConfig: generation_config }.tap do |params|
+          if @system_prompt
+            params[:system_instruction] = { parts: [{text: @system_prompt}] }
+          end
+          if @available_tools && @available_tools.any?
+            params[:tools] = tool_schemas
+          end
+        end
+      end
+
+      def tool_schemas
+        [
+          {
+            function_declarations: @available_tools.map do |tool|
+              {
+                name: tool.tool_schema[:name],
+                description: tool.tool_schema[:description],
+                parameters: tool.tool_schema[:parameters]
+              }
+            end
+          }
+        ]
+      end
+
+      def calculate_usage(response, execution_time)
+        input_tokens = nil
+        output_tokens = nil
+        cache_was_written = nil
+        cache_was_read = nil
+        token_counts = {}    
+
+        ## TODO cache write is never reported in usageMetadata so we can't calculate it's cost
+        ## Maybe there is no cost for implicit caching?
+        ## TODO support explicit caching
+
+        if usage_metadata = response['usageMetadata']
+          input_tokens = 0
+          output_tokens = 0
+          cache_was_read = false
+
+          if ptc = usage_metadata['promptTokenCount']
+            input_tokens += ptc
+            token_counts[:input] = ptc
+          end
+
+          if cctc = usage_metadata['cachedContentTokenCount']
+            cache_was_read = true
+            token_counts[:cache_read] = cctc
+            if token_counts[:input]
+              token_counts[:input] -= cctc
+            end
+          end
+          
+          if otc = usage_metadata['thoughtsTokenCount']
+            output_tokens += otc
+            token_counts[:output] = otc
+          end
+          
+          if ctc = usage_metadata['candidatesTokenCount']
+            output_tokens += ctc
+            token_counts[:output] ||= 0
+            token_counts[:output] += ctc
+          end
+        end
+
+        {
+          input_tokens: input_tokens,
+          output_tokens: output_tokens,
+          cache_was_written: cache_was_written,
+          cache_was_read: cache_was_read,
+          token_details: token_counts,
+          execution_time: execution_time,
+          estimated_cost: calculate_cost(token_counts)
+        }
+      end
+
+      def initialize_client
+        @client = LLMs::APIs::GoogleGeminiAPI.new(fetch_api_key)
+      end
+
+    end
+  end
+end

data/lib/llms/executors/hugging_face_executor.rb

@@ -0,0 +1,17 @@
+require_relative './open_ai_compatible_executor'
+
+module LLMs
+  module Executors
+    class HuggingFaceExecutor < OpenAICompatibleExecutor
+
+      private
+
+      ## TODO remove need for this class (by supporting e.g. a base_url_template param?)
+      def initialize_client
+        @base_url = "https://api-inference.huggingface.co/models/#{@model_name}/v1"
+        super
+      end
+
+    end
+  end
+end