raif 1.1.0 → 1.2.0

data/app/models/raif/llms/{bedrock_claude.rb → bedrock.rb}

@@ -1,34 +1,59 @@
 # frozen_string_literal: true
 
-class Raif::Llms::BedrockClaude < Raif::Llm
-  include Raif::Concerns::Llms::BedrockClaude::MessageFormatting
+class Raif::Llms::Bedrock < Raif::Llm
+  include Raif::Concerns::Llms::Bedrock::MessageFormatting
+  include Raif::Concerns::Llms::Bedrock::ToolFormatting
 
-  def perform_model_completion!(model_completion)
+  def perform_model_completion!(model_completion, &block)
     if Raif.config.aws_bedrock_model_name_prefix.present?
       model_completion.model_api_name = "#{Raif.config.aws_bedrock_model_name_prefix}.#{model_completion.model_api_name}"
     end
 
     params = build_request_parameters(model_completion)
-    resp = bedrock_client.converse(params)
 
+    if model_completion.stream_response?
+      bedrock_client.converse_stream(params) do |stream|
+        stream.on_error_event do |event|
+          raise Raif::Errors::StreamingError.new(
+            message: event.error_message,
+            type: event.event_type,
+            code: event.error_code,
+            event: event
+          )
+        end
+
+        handler = streaming_chunk_handler(model_completion, &block)
+        stream.on_event do |event|
+          handler.call(event)
+        end
+      end
+    else
+      response = bedrock_client.converse(params)
+      update_model_completion(model_completion, response)
+    end
+
+    model_completion
+  end
+
+private
+
+  def bedrock_client
+    @bedrock_client ||= Aws::BedrockRuntime::Client.new(region: Raif.config.aws_bedrock_region)
+  end
+
+  def update_model_completion(model_completion, resp)
     model_completion.raw_response = if model_completion.response_format_json?
       extract_json_response(resp)
     else
       extract_text_response(resp)
     end
 
+    model_completion.response_array = resp.output.message.content
+    model_completion.response_tool_calls = extract_response_tool_calls(resp)
     model_completion.completion_tokens = resp.usage.output_tokens
     model_completion.prompt_tokens = resp.usage.input_tokens
     model_completion.total_tokens = resp.usage.total_tokens
     model_completion.save!
-
-    model_completion
-  end
-
-protected
-
-  def bedrock_client
-    @bedrock_client ||= Aws::BedrockRuntime::Client.new(region: Raif.config.aws_bedrock_region)
   end
 
   def build_request_parameters(model_completion)
@@ -44,8 +69,10 @@ protected
 
     params[:system] = [{ text: model_completion.system_prompt }] if model_completion.system_prompt.present?
 
-    tool_config = build_tool_parameters(model_completion)
-    params[:tool_config] = tool_config if tool_config.present?
+    if supports_native_tool_use?
+      tools = build_tools_parameter(model_completion)
+      params[:tool_config] = tools unless tools.blank?
+    end
 
     params
   end
@@ -65,38 +92,7 @@ protected
     end
   end
 
-  def build_tool_parameters(model_completion)
-    tools = []
-
-    # If we're looking for a JSON response, add a tool to the request that the model can use to provide a JSON response
-    if model_completion.response_format_json? && model_completion.json_response_schema.present?
-      tools << {
-        name: "json_response",
-        description: "Generate a structured JSON response based on the provided schema.",
-        input_schema: { json: model_completion.json_response_schema }
-      }
-    end
-
-    # If we support native tool use and have tools available, add them to the request
-    if supports_native_tool_use? && model_completion.available_model_tools.any?
-      model_completion.available_model_tools_map.each do |_tool_name, tool|
-        tools << {
-          name: tool.tool_name,
-          description: tool.tool_description,
-          input_schema: { json: tool.tool_arguments_schema }
-        }
-      end
-    end
-
-    return if tools.blank?
-
-    {
-      tools: tools.map{|tool| { tool_spec: tool } }
-    }
-  end
-
   def extract_text_response(resp)
-    # Get the message from the response object
     message = resp.output.message
 
     # Find the first text content block
@@ -145,4 +141,25 @@ protected
     end
   end
 
+  def streaming_chunk_handler(model_completion, &block)
+    return unless model_completion.stream_response?
+
+    streaming_response = Raif::StreamingResponses::Bedrock.new
+    accumulated_delta = ""
+
+    proc do |event|
+      delta, finish_reason = streaming_response.process_streaming_event(event.class, event)
+      accumulated_delta += delta if delta.present?
+
+      if accumulated_delta.length >= Raif.config.streaming_update_chunk_size_threshold || finish_reason.present?
+        update_model_completion(model_completion, streaming_response.current_response)
+
+        if accumulated_delta.present?
+          block.call(model_completion, accumulated_delta, event)
+          accumulated_delta = ""
+        end
+      end
+    end
+  end
+
 end

data/app/models/raif/llms/open_ai_base.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+class Raif::Llms::OpenAiBase < Raif::Llm
+  include Raif::Concerns::Llms::OpenAi::JsonSchemaValidation
+
+  def perform_model_completion!(model_completion, &block)
+    if supports_temperature?
+      model_completion.temperature ||= default_temperature
+    else
+      Raif.logger.warn "Temperature is not supported for #{api_name}. Ignoring temperature parameter."
+      model_completion.temperature = nil
+    end
+
+    parameters = build_request_parameters(model_completion)
+
+    response = connection.post(api_path) do |req|
+      req.body = parameters
+      req.options.on_data = streaming_chunk_handler(model_completion, &block) if model_completion.stream_response?
+    end
+
+    unless model_completion.stream_response?
+      update_model_completion(model_completion, response.body)
+    end
+
+    model_completion
+  end
+
+private
+
+  def connection
+    @connection ||= Faraday.new(url: "https://api.openai.com/v1") do |f|
+      f.headers["Authorization"] = "Bearer #{Raif.config.open_ai_api_key}"
+      f.request :json
+      f.response :json
+      f.response :raise_error
+    end
+  end
+
+  def format_system_prompt(model_completion)
+    formatted_system_prompt = model_completion.system_prompt.to_s.strip
+
+    # If the response format is JSON, we need to include "as json" in the system prompt.
+    # OpenAI requires this and will throw an error if it's not included.
+    if model_completion.response_format_json?
+      # Ensure system prompt ends with a period if not empty
+      if formatted_system_prompt.present? && !formatted_system_prompt.end_with?(".", "?", "!")
+        formatted_system_prompt += "."
+      end
+      formatted_system_prompt += " Return your response as JSON."
+      formatted_system_prompt.strip!
+    end
+
+    formatted_system_prompt
+  end
+
+  def supports_structured_outputs?
+    # Not all OpenAI models support structured outputs:
+    # https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#supported-models
+    provider_settings.key?(:supports_structured_outputs) ? provider_settings[:supports_structured_outputs] : true
+  end
+
+  def supports_temperature?
+    provider_settings.key?(:supports_temperature) ? provider_settings[:supports_temperature] : true
+  end
+
+end

data/app/models/raif/llms/open_ai_completions.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+class Raif::Llms::OpenAiCompletions < Raif::Llms::OpenAiBase
+  include Raif::Concerns::Llms::OpenAiCompletions::MessageFormatting
+  include Raif::Concerns::Llms::OpenAiCompletions::ToolFormatting
+
+private
+
+  def api_path
+    "chat/completions"
+  end
+
+  def streaming_response_type
+    Raif::StreamingResponses::OpenAiCompletions
+  end
+
+  def update_model_completion(model_completion, response_json)
+    model_completion.update!(
+      response_id: response_json["id"],
+      response_tool_calls: extract_response_tool_calls(response_json),
+      raw_response: response_json.dig("choices", 0, "message", "content"),
+      response_array: response_json["choices"],
+      completion_tokens: response_json.dig("usage", "completion_tokens"),
+      prompt_tokens: response_json.dig("usage", "prompt_tokens"),
+      total_tokens: response_json.dig("usage", "total_tokens")
+    )
+  end
+
+  def extract_response_tool_calls(resp)
+    return if resp.dig("choices", 0, "message", "tool_calls").blank?
+
+    resp.dig("choices", 0, "message", "tool_calls").map do |tool_call|
+      {
+        "name" => tool_call["function"]["name"],
+        "arguments" => JSON.parse(tool_call["function"]["arguments"])
+      }
+    end
+  end
+
+  def build_request_parameters(model_completion)
+    formatted_system_prompt = format_system_prompt(model_completion)
+
+    messages = model_completion.messages
+    messages_with_system = if formatted_system_prompt.blank?
+      messages
+    else
+      [{ "role" => "system", "content" => formatted_system_prompt }] + messages
+    end
+
+    parameters = {
+      model: api_name,
+      messages: messages_with_system
+    }
+
+    if supports_temperature?
+      parameters[:temperature] = model_completion.temperature.to_f
+    end
+
+    # If the LLM supports native tool use and there are available tools, add them to the parameters
+    if supports_native_tool_use?
+      tools = build_tools_parameter(model_completion)
+      parameters[:tools] = tools unless tools.blank?
+    end
+
+    if model_completion.stream_response?
+      parameters[:stream] = true
+      # Ask for usage stats in the last chunk
+      parameters[:stream_options] = { include_usage: true }
+    end
+
+    # Add response format if needed
+    response_format = determine_response_format(model_completion)
+    parameters[:response_format] = response_format if response_format
+    model_completion.response_format_parameter = response_format[:type] if response_format
+
+    parameters
+  end
+
+  def determine_response_format(model_completion)
+    # Only configure response format for JSON outputs
+    return unless model_completion.response_format_json?
+
+    if model_completion.json_response_schema.present? && supports_structured_outputs?
+      validate_json_schema!(model_completion.json_response_schema)
+
+      {
+        type: "json_schema",
+        json_schema: {
+          name: "json_response_schema",
+          strict: true,
+          schema: model_completion.json_response_schema
+        }
+      }
+    else
+      # Default JSON mode for OpenAI models that don't support structured outputs or no schema is provided
+      { type: "json_object" }
+    end
+  end
+
+end

data/app/models/raif/llms/open_ai_responses.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+class Raif::Llms::OpenAiResponses < Raif::Llms::OpenAiBase
+  include Raif::Concerns::Llms::OpenAiResponses::MessageFormatting
+  include Raif::Concerns::Llms::OpenAiResponses::ToolFormatting
+
+private
+
+  def api_path
+    "responses"
+  end
+
+  def streaming_response_type
+    Raif::StreamingResponses::OpenAiResponses
+  end
+
+  def update_model_completion(model_completion, response_json)
+    model_completion.update!(
+      response_id: response_json["id"],
+      response_tool_calls: extract_response_tool_calls(response_json),
+      raw_response: extract_raw_response(response_json),
+      response_array: response_json["output"],
+      citations: extract_citations(response_json),
+      completion_tokens: response_json.dig("usage", "output_tokens"),
+      prompt_tokens: response_json.dig("usage", "input_tokens"),
+      total_tokens: response_json.dig("usage", "total_tokens")
+    )
+  end
+
+  def extract_response_tool_calls(resp)
+    return if resp["output"].blank?
+
+    tool_calls = []
+    resp["output"].each do |output_item|
+      next unless output_item["type"] == "function_call"
+
+      tool_calls << {
+        "name" => output_item["name"],
+        "arguments" => JSON.parse(output_item["arguments"])
+      }
+    end
+
+    tool_calls.any? ? tool_calls : nil
+  end
+
+  def extract_raw_response(resp)
+    text_outputs = []
+
+    output_messages = resp["output"]&.select{ |output_item| output_item["type"] == "message" }
+    output_messages&.each do |output_message|
+      output_message["content"].each do |content_item|
+        text_outputs << content_item["text"] if content_item["type"] == "output_text"
+      end
+    end
+
+    text_outputs.join("\n").presence
+  end
+
+  def extract_citations(resp)
+    return [] if resp["output"].blank?
+
+    citations = []
+
+    # Look through output messages for citations in annotations
+    output_messages = resp["output"].select{|output_item| output_item["type"] == "message" }
+    output_messages.each do |output_message|
+      next unless output_message["content"].present?
+
+      output_message["content"].each do |content_item|
+        next unless content_item["type"] == "output_text" && content_item["annotations"].present?
+
+        content_item["annotations"].each do |annotation|
+          next unless annotation["type"] == "url_citation"
+
+          citations << {
+            "url" => Raif::Utils::HtmlFragmentProcessor.strip_tracking_parameters(annotation["url"]),
+            "title" => annotation["title"]
+          }
+        end
+      end
+    end
+
+    citations.uniq{|citation| citation["url"] }
+  end
+
+  def build_request_parameters(model_completion)
+    parameters = {
+      model: api_name,
+      input: model_completion.messages,
+    }
+
+    if supports_temperature?
+      parameters[:temperature] = model_completion.temperature.to_f
+    end
+
+    parameters[:stream] = true if model_completion.stream_response?
+
+    # Add instructions (system prompt) if present
+    formatted_system_prompt = format_system_prompt(model_completion)
+    if formatted_system_prompt.present?
+      parameters[:instructions] = formatted_system_prompt
+    end
+
+    # Add max_output_tokens if specified
+    if model_completion.max_completion_tokens.present?
+      parameters[:max_output_tokens] = model_completion.max_completion_tokens
+    end
+
+    # If the LLM supports native tool use and there are available tools, add them to the parameters
+    if supports_native_tool_use?
+      tools = build_tools_parameter(model_completion)
+      parameters[:tools] = tools unless tools.blank?
+    end
+
+    # Add response format if needed. Default will be { "type": "text" }
+    response_format = determine_response_format(model_completion)
+    if response_format.present?
+      parameters[:text] = { format: response_format }
+      model_completion.response_format_parameter = response_format[:type]
+    end
+
+    parameters
+  end
+
+  def determine_response_format(model_completion)
+    # Only configure response format for JSON outputs
+    return unless model_completion.response_format_json?
+
+    if model_completion.json_response_schema.present? && supports_structured_outputs?
+      validate_json_schema!(model_completion.json_response_schema)
+
+      {
+        type: "json_schema",
+        name: "json_response_schema",
+        strict: true,
+        schema: model_completion.json_response_schema
+      }
+    else
+      # Default JSON mode for OpenAI models that don't support structured outputs or no schema is provided
+      { type: "json_object" }
+    end
+  end
+
+end

data/app/models/raif/llms/open_router.rb

@@ -1,28 +1,27 @@
 # frozen_string_literal: true
 
 class Raif::Llms::OpenRouter < Raif::Llm
-  include Raif::Concerns::Llms::OpenAi::MessageFormatting
+  include Raif::Concerns::Llms::OpenAiCompletions::MessageFormatting
+  include Raif::Concerns::Llms::OpenAiCompletions::ToolFormatting
+  include Raif::Concerns::Llms::OpenAi::JsonSchemaValidation
 
-  def perform_model_completion!(model_completion)
+  def perform_model_completion!(model_completion, &block)
     model_completion.temperature ||= default_temperature
     parameters = build_request_parameters(model_completion)
     response = connection.post("chat/completions") do |req|
       req.body = parameters
+      req.options.on_data = streaming_chunk_handler(model_completion, &block) if model_completion.stream_response?
     end
 
-    response_json = response.body
-
-    model_completion.update!(
-      response_tool_calls: extract_response_tool_calls(response_json),
-      raw_response: response_json.dig("choices", 0, "message", "content"),
-      completion_tokens: response_json.dig("usage", "completion_tokens"),
-      prompt_tokens: response_json.dig("usage", "prompt_tokens"),
-      total_tokens: response_json.dig("usage", "total_tokens")
-    )
+    unless model_completion.stream_response?
+      update_model_completion(model_completion, response.body)
+    end
 
     model_completion
   end
 
+private
+
   def connection
     @connection ||= Faraday.new(url: "https://openrouter.ai/api/v1") do |f|
       f.headers["Authorization"] = "Bearer #{Raif.config.open_router_api_key}"
@@ -34,7 +33,20 @@ class Raif::Llms::OpenRouter < Raif::Llm
     end
   end
 
-protected
+  def streaming_response_type
+    Raif::StreamingResponses::OpenAiCompletions
+  end
+
+  def update_model_completion(model_completion, response_json)
+    model_completion.update!(
+      response_tool_calls: extract_response_tool_calls(response_json),
+      raw_response: response_json.dig("choices", 0, "message", "content"),
+      response_array: response_json["choices"],
+      completion_tokens: response_json.dig("usage", "completion_tokens"),
+      prompt_tokens: response_json.dig("usage", "prompt_tokens"),
+      total_tokens: response_json.dig("usage", "total_tokens")
+    )
+  end
 
   def build_request_parameters(model_completion)
     params = {
@@ -42,7 +54,6 @@ protected
       messages: model_completion.messages,
       temperature: model_completion.temperature.to_f,
       max_tokens: model_completion.max_completion_tokens || default_max_completion_tokens,
-      stream: false
     }
 
     # Add system message to the messages array if present
@@ -50,44 +61,28 @@ protected
       params[:messages].unshift({ "role" => "system", "content" => model_completion.system_prompt })
     end
 
-    if model_completion.available_model_tools.any?
-      tools = []
-
-      model_completion.available_model_tools_map.each do |_tool_name, tool|
-        tools << {
-          type: "function",
-          function: {
-            name: tool.tool_name,
-            description: tool.tool_description,
-            parameters: tool.tool_arguments_schema
-          }
-        }
-      end
+    if supports_native_tool_use?
+      tools = build_tools_parameter(model_completion)
+      params[:tools] = tools unless tools.blank?
+    end
 
-      params[:tools] = tools
+    if model_completion.stream_response?
+      # Ask for usage stats in the last chunk
+      params[:stream] = true
+      params[:stream_options] = { include_usage: true }
     end
 
     params
   end
 
-  def extract_response_tool_calls(response_json)
-    tool_calls = response_json.dig("choices", 0, "message", "tool_calls")
-    return [] unless tool_calls.is_a?(Array)
-
-    tool_calls.map do |tool_call|
-      next unless tool_call["type"] == "function"
-
-      function = tool_call["function"]
-      next unless function.is_a?(Hash)
+  def extract_response_tool_calls(resp)
+    return if resp.dig("choices", 0, "message", "tool_calls").blank?
 
+    resp.dig("choices", 0, "message", "tool_calls").map do |tool_call|
       {
-        "id" => tool_call["id"],
-        "type" => "function",
-        "function" => {
-          "name" => function["name"],
-          "arguments" => function["arguments"]
-        }
+        "name" => tool_call["function"]["name"],
+        "arguments" => JSON.parse(tool_call["function"]["arguments"])
       }
-    end.compact
+    end
   end
 end

data/app/models/raif/model_completion.rb

@@ -16,6 +16,8 @@ class Raif::ModelCompletion < Raif::ApplicationRecord
 
   after_initialize -> { self.messages ||= [] }
   after_initialize -> { self.available_model_tools ||= [] }
+  after_initialize -> { self.response_array ||= [] }
+  after_initialize -> { self.citations ||= [] }
 
   def json_response_schema
     source.json_response_schema if source&.respond_to?(:json_response_schema)

data/app/models/raif/model_tool.rb

@@ -64,6 +64,10 @@ class Raif::ModelTool
       end
     end
 
+    def provider_managed?
+      false
+    end
+
     def renderable?
       true
     end

data/app/models/raif/model_tools/provider_managed/base.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class Raif::ModelTools::ProviderManaged::Base < Raif::ModelTool
+  class << self
+    def provider_managed?
+      true
+    end
+  end
+end

data/app/models/raif/model_tools/provider_managed/code_execution.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class Raif::ModelTools::ProviderManaged::CodeExecution < Raif::ModelTools::ProviderManaged::Base
+
+end

data/app/models/raif/model_tools/provider_managed/image_generation.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class Raif::ModelTools::ProviderManaged::ImageGeneration < Raif::ModelTools::ProviderManaged::Base
+
+end

data/app/models/raif/model_tools/provider_managed/web_search.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class Raif::ModelTools::ProviderManaged::WebSearch < Raif::ModelTools::ProviderManaged::Base
+
+end