raif 1.0.0 → 1.1.0

data/app/models/raif/llms/bedrock_claude.rb

@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
 class Raif::Llms::BedrockClaude < Raif::Llm
+  include Raif::Concerns::Llms::BedrockClaude::MessageFormatting
 
   def perform_model_completion!(model_completion)
     if Raif.config.aws_bedrock_model_name_prefix.present?
       model_completion.model_api_name = "#{Raif.config.aws_bedrock_model_name_prefix}.#{model_completion.model_api_name}"
     end
 
-    params = build_api_parameters(model_completion)
+    params = build_request_parameters(model_completion)
     resp = bedrock_client.converse(params)
 
     model_completion.raw_response = if model_completion.response_format_json?
@@ -30,25 +31,41 @@ protected
     @bedrock_client ||= Aws::BedrockRuntime::Client.new(region: Raif.config.aws_bedrock_region)
   end
 
-  def format_messages(messages)
-    messages.map(&:symbolize_keys).map do |message|
-      {
-        role: message[:role],
-        content: [{ text: message[:content] }]
-      }
-    end
-  end
+  def build_request_parameters(model_completion)
+    # The AWS Bedrock SDK requires symbols for keys
+    messages_param = model_completion.messages.map(&:deep_symbolize_keys)
+    replace_tmp_base64_data_with_bytes(messages_param)
 
-  def build_api_parameters(model_completion)
     params = {
       model_id: model_completion.model_api_name,
       inference_config: { max_tokens: model_completion.max_completion_tokens || 8192 },
-      messages: format_messages(model_completion.messages)
+      messages: messages_param
     }
 
     params[:system] = [{ text: model_completion.system_prompt }] if model_completion.system_prompt.present?
 
-    # Prepare tools configuration if needed
+    tool_config = build_tool_parameters(model_completion)
+    params[:tool_config] = tool_config if tool_config.present?
+
+    params
+  end
+
+  def replace_tmp_base64_data_with_bytes(messages)
+    # The AWS Bedrock SDK requires data sent as bytes (and doesn't support base64 like everyone else)
+    # The ModelCompletion stores the messages as JSON though, so it can't be raw bytes.
+    # We store the image data as base64, so we need to convert that to bytes before sending to AWS.
+    messages.each do |message|
+      message[:content].each do |content|
+        next unless content[:image] || content[:document]
+
+        type_key = content[:image] ? :image : :document
+        base64_data = content[type_key][:source].delete(:tmp_base64_data)
+        content[type_key][:source][:bytes] = Base64.strict_decode64(base64_data)
+      end
+    end
+  end
+
+  def build_tool_parameters(model_completion)
     tools = []
 
     # If we're looking for a JSON response, add a tool to the request that the model can use to provide a JSON response
@@ -71,14 +88,11 @@ protected
       end
     end
 
-    # Add tool configuration if any tools are available
-    if tools.any?
-      params[:tool_config] = {
-        tools: tools.map { |tool| { tool_spec: tool } }
-      }
-    end
+    return if tools.blank?
 
-    params
+    {
+      tools: tools.map{|tool| { tool_spec: tool } }
+    }
   end
 
   def extract_text_response(resp)

data/app/models/raif/llms/open_ai.rb

@@ -1,29 +1,24 @@
 # frozen_string_literal: true
 
 class Raif::Llms::OpenAi < Raif::Llm
+  include Raif::Concerns::Llms::OpenAi::MessageFormatting
 
   def perform_model_completion!(model_completion)
     model_completion.temperature ||= default_temperature
-    parameters = build_chat_parameters(model_completion)
+    parameters = build_request_parameters(model_completion)
 
     response = connection.post("chat/completions") do |req|
-      req.body = parameters.to_json
+      req.body = parameters
     end
 
-    resp = JSON.parse(response.body)
-
-    # Handle API errors
-    unless response.success?
-      error_message = resp["error"]&.dig("message") || "OpenAI API error: #{response.status}"
-      raise Raif::Errors::OpenAi::ApiError, error_message
-    end
+    response_json = response.body
 
     model_completion.update!(
-      response_tool_calls: extract_response_tool_calls(resp),
-      raw_response: resp.dig("choices", 0, "message", "content"),
-      completion_tokens: resp["usage"]["completion_tokens"],
-      prompt_tokens: resp["usage"]["prompt_tokens"],
-      total_tokens: resp["usage"]["total_tokens"],
+      response_tool_calls: extract_response_tool_calls(response_json),
+      raw_response: response_json.dig("choices", 0, "message", "content"),
+      completion_tokens: response_json.dig("usage", "completion_tokens"),
+      prompt_tokens: response_json.dig("usage", "prompt_tokens"),
+      total_tokens: response_json.dig("usage", "total_tokens"),
       response_format_parameter: parameters.dig(:response_format, :type)
     )
 
@@ -32,8 +27,10 @@ class Raif::Llms::OpenAi < Raif::Llm
 
   def connection
     @connection ||= Faraday.new(url: "https://api.openai.com/v1") do |f|
-      f.headers["Content-Type"] = "application/json"
       f.headers["Authorization"] = "Bearer #{Raif.config.open_ai_api_key}"
+      f.request :json
+      f.response :json
+      f.response :raise_error
     end
   end
 
@@ -88,7 +85,7 @@ private
     end
   end
 
-  def build_chat_parameters(model_completion)
+  def build_request_parameters(model_completion)
     formatted_system_prompt = model_completion.system_prompt.to_s.strip
 
     # If the response format is JSON, we need to include "as json" in the system prompt.
@@ -162,7 +159,7 @@ private
   def supports_structured_outputs?
     # Not all OpenAI models support structured outputs:
     # https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#supported-models
-    provider_settings[:supports_structured_outputs]
+    provider_settings.key?(:supports_structured_outputs) ? provider_settings[:supports_structured_outputs] : true
   end
 
   def validate_object_properties(schema, errors)

data/app/models/raif/llms/open_router.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+class Raif::Llms::OpenRouter < Raif::Llm
+  include Raif::Concerns::Llms::OpenAi::MessageFormatting
+
+  def perform_model_completion!(model_completion)
+    model_completion.temperature ||= default_temperature
+    parameters = build_request_parameters(model_completion)
+    response = connection.post("chat/completions") do |req|
+      req.body = parameters
+    end
+
+    response_json = response.body
+
+    model_completion.update!(
+      response_tool_calls: extract_response_tool_calls(response_json),
+      raw_response: response_json.dig("choices", 0, "message", "content"),
+      completion_tokens: response_json.dig("usage", "completion_tokens"),
+      prompt_tokens: response_json.dig("usage", "prompt_tokens"),
+      total_tokens: response_json.dig("usage", "total_tokens")
+    )
+
+    model_completion
+  end
+
+  def connection
+    @connection ||= Faraday.new(url: "https://openrouter.ai/api/v1") do |f|
+      f.headers["Authorization"] = "Bearer #{Raif.config.open_router_api_key}"
+      f.headers["HTTP-Referer"] = Raif.config.open_router_site_url if Raif.config.open_router_site_url.present?
+      f.headers["X-Title"] = Raif.config.open_router_app_name if Raif.config.open_router_app_name.present?
+      f.request :json
+      f.response :json
+      f.response :raise_error
+    end
+  end
+
+protected
+
+  def build_request_parameters(model_completion)
+    params = {
+      model: model_completion.model_api_name,
+      messages: model_completion.messages,
+      temperature: model_completion.temperature.to_f,
+      max_tokens: model_completion.max_completion_tokens || default_max_completion_tokens,
+      stream: false
+    }
+
+    # Add system message to the messages array if present
+    if model_completion.system_prompt.present?
+      params[:messages].unshift({ "role" => "system", "content" => model_completion.system_prompt })
+    end
+
+    if model_completion.available_model_tools.any?
+      tools = []
+
+      model_completion.available_model_tools_map.each do |_tool_name, tool|
+        tools << {
+          type: "function",
+          function: {
+            name: tool.tool_name,
+            description: tool.tool_description,
+            parameters: tool.tool_arguments_schema
+          }
+        }
+      end
+
+      params[:tools] = tools
+    end
+
+    params
+  end
+
+  def extract_response_tool_calls(response_json)
+    tool_calls = response_json.dig("choices", 0, "message", "tool_calls")
+    return [] unless tool_calls.is_a?(Array)
+
+    tool_calls.map do |tool_call|
+      next unless tool_call["type"] == "function"
+
+      function = tool_call["function"]
+      next unless function.is_a?(Hash)
+
+      {
+        "id" => tool_call["id"],
+        "type" => "function",
+        "function" => {
+          "name" => function["name"],
+          "arguments" => function["arguments"]
+        }
+      }
+    end.compact
+  end
+end

data/app/models/raif/model_completion.rb

@@ -12,6 +12,7 @@ class Raif::ModelCompletion < Raif::ApplicationRecord
   delegate :json_response_schema, to: :source, allow_nil: true
 
   before_save :set_total_tokens
+  before_save :calculate_costs
 
   after_initialize -> { self.messages ||= [] }
   after_initialize -> { self.available_model_tools ||= [] }
@@ -20,9 +21,27 @@ class Raif::ModelCompletion < Raif::ApplicationRecord
     source.json_response_schema if source&.respond_to?(:json_response_schema)
   end
 
-protected
-
   def set_total_tokens
     self.total_tokens ||= completion_tokens.present? && prompt_tokens.present? ? completion_tokens + prompt_tokens : nil
   end
+
+  def calculate_costs
+    if prompt_tokens.present? && llm_config[:input_token_cost].present?
+      self.prompt_token_cost = llm_config[:input_token_cost] * prompt_tokens
+    end
+
+    if completion_tokens.present? && llm_config[:output_token_cost].present?
+      self.output_token_cost = llm_config[:output_token_cost] * completion_tokens
+    end
+
+    if prompt_token_cost.present? || output_token_cost.present?
+      self.total_cost = (prompt_token_cost || 0) + (output_token_cost || 0)
+    end
+  end
+
+private
+
+  def llm_config
+    @llm_config ||= Raif.llm_config(llm_model_key.to_sym)
+  end
 end

data/app/models/raif/model_file_input.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+class Raif::ModelFileInput
+  include ActiveModel::Model
+
+  attr_accessor :input, :url, :base64_data, :filename, :content_type, :source_type
+
+  validates :url, format: { with: URI::DEFAULT_PARSER.make_regexp(%w[http https]), message: "is not a valid URL" }, allow_nil: true
+  validates :base64_data, presence: { message: "could not be read from input" }, if: -> { input.present? }
+  validates :content_type, presence: { message: "could not be determined" }, if: -> { input.present? }
+
+  def initialize(input: nil, url: nil)
+    raise ArgumentError, "You must provide either an input or a URL" if input.blank? && url.blank?
+    raise ArgumentError, "Provide either input or URL, not both" if input.present? && url.present?
+
+    super(input: input, url: url)
+
+    if url.present?
+      @source_type = :url
+    elsif input.present?
+      @source_type = :file_content
+      process_input!
+    end
+
+    unless valid?
+      raise Raif::Errors::InvalidModelFileInputError, errors.full_messages.join(", ")
+    end
+  end
+
+  def inspect
+    "#<#{self.class.name} input=#{input.inspect} url=#{url.inspect} base64_data=#{base64_data&.truncate(20)} filename=#{filename.inspect} content_type=#{content_type.inspect} source_type=#{source_type.inspect}>" # rubocop:disable Layout/LineLength
+  end
+
+  def file_bytes
+    Base64.strict_decode64(base64_data)
+  end
+
+private
+
+  def process_input!
+    if input_is_active_storage_blob?
+      process_active_storage_blob(input)
+      return
+    elsif input_has_active_storage_blob?
+      process_active_storage_blob(input.blob)
+      return
+    end
+
+    case input
+    when String
+      process_string_input
+    when Pathname
+      process_from_path
+    when File, Tempfile, IO, StringIO
+      read_from_io
+    else
+      raise Raif::Errors::InvalidModelFileInputError, "Invalid input type for #{self.class.name} (got #{input.class})"
+    end
+  end
+
+  def process_string_input
+    if File.exist?(input)
+      @input = Pathname.new(input)
+      process_from_path
+    else
+      raise Raif::Errors::InvalidModelFileInputError, "File does not exist: #{input}"
+    end
+  end
+
+  def process_active_storage_blob(blob)
+    @filename = blob.filename.to_s
+    @content_type = blob.content_type
+    @base64_data = Base64.strict_encode64(blob.download)
+  end
+
+  def process_from_path
+    @filename = input.basename.to_s
+    @content_type = Marcel::MimeType.for(input)
+    @base64_data = Base64.strict_encode64(input.binread)
+  end
+
+  def read_from_io
+    @filename = input.respond_to?(:path) ? File.basename(input.path) : nil
+    @content_type = Marcel::MimeType.for(input)
+    try_rewind
+    @base64_data = Base64.strict_encode64(input.read)
+  end
+
+  def try_rewind
+    input.rewind if input.respond_to?(:rewind)
+  rescue IOError => e
+    logger.error "Failed to rewind IO: #{e.message}"
+    logger.error e.backtrace.join("\n")
+  end
+
+  def input_looks_like_base64?
+    input.match?(%r{\A[a-zA-Z0-9+/\n\r]+={0,2}\z})
+  end
+
+  def input_has_active_storage_blob?
+    return false unless defined?(ActiveStorage)
+    return false unless input.respond_to?(:blob)
+
+    input.blob.is_a?(ActiveStorage::Blob)
+  end
+
+  def input_is_active_storage_blob?
+    return false unless defined?(ActiveStorage)
+
+    input.is_a?(ActiveStorage::Blob)
+  end
+
+end

data/app/models/raif/model_image_input.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class Raif::ModelImageInput < Raif::ModelFileInput
+end

data/app/models/raif/model_tool.rb

@@ -1,69 +1,95 @@
 # frozen_string_literal: true
 
 class Raif::ModelTool
+  include Raif::Concerns::JsonSchemaDefinition
 
-  delegate :tool_name, :tool_description, :tool_arguments_schema, to: :class
+  delegate :tool_name, :tool_description, :tool_arguments_schema, :example_model_invocation, to: :class
 
-  # The description of the tool that will be provided to the model
-  # when giving it a list of available tools.
-  def self.description_for_llm
-    <<~DESCRIPTION
-      Name: #{tool_name}
-      Description: #{tool_description}
-      Arguments Schema:
-      #{JSON.pretty_generate(tool_arguments_schema)}
-      Example Usage:
-      #{JSON.pretty_generate(example_model_invocation)}
-    DESCRIPTION
-  end
-
-  # The name of the tool as it will be provided to the model & used in the model invocation.
-  # Default for something like Raif::ModelTools::WikipediaSearch would be "wikipedia_search"
-  def self.tool_name
-    name.split("::").last.underscore
-  end
+  class << self
+    # The description of the tool that will be provided to the model
+    # when giving it a list of available tools.
+    def description_for_llm
+      <<~DESCRIPTION
+        Name: #{tool_name}
+        Description: #{tool_description}
+        Arguments Schema:
+        #{JSON.pretty_generate(tool_arguments_schema)}
+        Example Usage:
+        #{JSON.pretty_generate(example_model_invocation)}
+      DESCRIPTION
+    end
 
-  def self.tool_description
-    raise NotImplementedError, "#{self.class.name}#tool_description is not implemented"
-  end
+    # The name of the tool as it will be provided to the model & used in the model invocation.
+    # Default for something like Raif::ModelTools::WikipediaSearch would be "wikipedia_search"
+    def tool_name
+      name.split("::").last.underscore
+    end
 
-  def self.example_model_invocation
-    raise NotImplementedError, "#{self.class.name}#example_model_invocation is not implemented"
-  end
+    def tool_description(&block)
+      if block_given?
+        @tool_description = block.call
+      elsif @tool_description.present?
+        @tool_description
+      else
+        raise NotImplementedError, "#{name}#tool_description is not implemented"
+      end
+    end
 
-  def self.process_invocation(invocation)
-    raise NotImplementedError, "#{self.class.name}#process_invocation is not implemented"
-  end
+    def example_model_invocation(&block)
+      if block_given?
+        @example_model_invocation = block.call
+      elsif @example_model_invocation.present?
+        @example_model_invocation
+      else
+        raise NotImplementedError, "#{name}#example_model_invocation is not implemented"
+      end
+    end
 
-  def self.invocation_partial_name
-    name.gsub("Raif::ModelTools::", "").underscore
-  end
+    def process_invocation(invocation)
+      raise NotImplementedError, "#{name}#process_invocation is not implemented"
+    end
 
-  def self.tool_arguments_schema
-    raise NotImplementedError, "#{self.class.name}#tool_arguments_schema is not implemented"
-  end
+    def invocation_partial_name
+      name.gsub("Raif::ModelTools::", "").underscore
+    end
 
-  def self.renderable?
-    true
-  end
+    def tool_arguments_schema(&block)
+      if block_given?
+        json_schema_definition(:tool_arguments, &block)
+      elsif schema_defined?(:tool_arguments)
+        schema_for(:tool_arguments)
+      else
+        raise NotImplementedError,
+          "#{name} must define tool arguments schema via tool_arguments_schema or override #{name}.tool_arguments_schema"
+      end
+    end
 
-  def self.invoke_tool(tool_arguments:, source:)
-    tool_invocation = Raif::ModelToolInvocation.new(
-      source: source,
-      tool_type: name,
-      tool_arguments: tool_arguments
-    )
+    def renderable?
+      true
+    end
 
-    ActiveRecord::Base.transaction do
-      tool_invocation.save!
-      process_invocation(tool_invocation)
-      tool_invocation.completed!
+    def triggers_observation_to_model?
+      false
     end
 
-    tool_invocation
-  rescue StandardError => e
-    tool_invocation.failed!
-    raise e
+    def invoke_tool(tool_arguments:, source:)
+      tool_invocation = Raif::ModelToolInvocation.new(
+        source: source,
+        tool_type: name,
+        tool_arguments: tool_arguments
+      )
+
+      ActiveRecord::Base.transaction do
+        tool_invocation.save!
+        process_invocation(tool_invocation)
+        tool_invocation.completed!
+      end
+
+      tool_invocation
+    rescue StandardError => e
+      tool_invocation.failed!
+      raise e
+    end
   end
 
 end

data/app/models/raif/model_tool_invocation.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "json-schema"
-
 class Raif::ModelToolInvocation < Raif::ApplicationRecord
   belongs_to :source, polymorphic: true
 
@@ -11,7 +9,11 @@ class Raif::ModelToolInvocation < Raif::ApplicationRecord
   validates :tool_type, presence: true
   validate :ensure_valid_tool_argument_schema, if: -> { tool_type.present? && tool_arguments_schema.present? }
 
-  delegate :tool_arguments_schema, :renderable?, :tool_name, to: :tool
+  delegate :tool_arguments_schema,
+    :renderable?,
+    :tool_name,
+    :triggers_observation_to_model?,
+    to: :tool
 
   boolean_timestamp :completed_at
   boolean_timestamp :failed_at
@@ -25,9 +27,9 @@ class Raif::ModelToolInvocation < Raif::ApplicationRecord
   end
 
   def result_llm_message
-    if result.present?
-      "Result from #{tool_name}: #{result.to_json}"
-    end
+    return unless tool.respond_to?(:observation_for_invocation)
+
+    tool.observation_for_invocation(self)
   end
 
   def to_partial_path

data/app/models/raif/model_tools/agent_final_answer.rb

@@ -1,46 +1,37 @@
 # frozen_string_literal: true
 
 class Raif::ModelTools::AgentFinalAnswer < Raif::ModelTool
+  tool_arguments_schema do
+    string "final_answer", description: "Your complete and final answer to the user's question or task"
+  end
 
-  def self.example_model_invocation
+  example_model_invocation do
     {
       "name" => tool_name,
       "arguments" => { "final_answer": "The answer to the user's question or task" }
     }
   end
 
-  def self.tool_arguments_schema
-    {
-      type: "object",
-      additionalProperties: false,
-      required: ["final_answer"],
-      properties: {
-        final_answer: {
-          type: "string",
-          description: "Your complete and final answer to the user's question or task"
-        }
-      }
-    }
-  end
-
-  def self.tool_description
+  tool_description do
     "Provide your final answer to the user's question or task"
   end
 
-  def self.observation_for_invocation(tool_invocation)
-    return "No answer provided" unless tool_invocation.result.present?
+  class << self
+    def observation_for_invocation(tool_invocation)
+      return "No answer provided" unless tool_invocation.result.present?
 
-    tool_invocation.result["final_answer"]
-  end
+      tool_invocation.result["final_answer"]
+    end
 
-  def self.process_invocation(tool_invocation)
-    tool_invocation.update!(
-      result: {
-        final_answer: tool_invocation.tool_arguments["final_answer"]
-      }
-    )
+    def process_invocation(tool_invocation)
+      tool_invocation.update!(
+        result: {
+          final_answer: tool_invocation.tool_arguments["final_answer"]
+        }
+      )
 
-    tool_invocation.result
+      tool_invocation.result
+    end
   end
 
 end