raif 1.1.0 → 1.2.0

data/app/models/raif/concerns/llms/open_ai/json_schema_validation.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Raif::Concerns::Llms::OpenAi::JsonSchemaValidation
+  extend ActiveSupport::Concern
+
+  def validate_json_schema!(schema)
+    return if schema.blank?
+
+    errors = []
+
+    # Check if schema is present
+    if schema.blank?
+      errors << "JSON schema must include a 'schema' property"
+    else
+      # Check root object type
+      if schema[:type] != "object" && !schema.key?(:properties)
+        errors << "Root schema must be of type 'object' with 'properties'"
+      end
+
+      # Check all objects in the schema recursively
+      validate_object_properties(schema, errors)
+
+      # Check properties count (max 100 total)
+      validate_properties_count(schema, errors)
+
+      # Check nesting depth (max 5 levels)
+      validate_nesting_depth(schema, errors)
+
+      # Check for unsupported anyOf at root level
+      if schema[:anyOf].present? && schema[:properties].blank?
+        errors << "Root objects cannot be of type 'anyOf'"
+      end
+    end
+
+    # Raise error if any validation issues found
+    if errors.any?
+      error_message = "Invalid JSON schema for OpenAI structured outputs: #{errors.join("; ")}\nSchema was: #{schema.inspect}"
+      raise Raif::Errors::OpenAi::JsonSchemaError, error_message
+    else
+      true
+    end
+  end
+
+private
+
+  def validate_object_properties(schema, errors)
+    return unless schema.is_a?(Hash)
+
+    # Check if the current schema is an object and validate additionalProperties and required fields
+    if schema[:type] == "object"
+      if schema[:additionalProperties] != false
+        errors << "All objects must have 'additionalProperties' set to false"
+      end
+
+      # Check that all properties are required
+      if schema[:properties].is_a?(Hash) && schema[:properties].any?
+        property_keys = schema[:properties].keys
+        required_fields = schema[:required] || []
+
+        if required_fields.sort != property_keys.map(&:to_s).sort
+          errors << "All object properties must be listed in the 'required' array"
+        end
+      end
+    end
+
+    # Check if the current schema is an object and validate additionalProperties
+    if schema[:type] == "object"
+      if schema[:additionalProperties] != false
+        errors << "All objects must have 'additionalProperties' set to false"
+      end
+
+      # Check properties of the object recursively
+      if schema[:properties].is_a?(Hash)
+        schema[:properties].each_value do |property|
+          validate_object_properties(property, errors)
+        end
+      end
+    end
+
+    # Check array items
+    if schema[:type] == "array" && schema[:items].is_a?(Hash)
+      validate_object_properties(schema[:items], errors)
+    end
+
+    # Check anyOf
+    if schema[:anyOf].is_a?(Array)
+      schema[:anyOf].each do |option|
+        validate_object_properties(option, errors)
+      end
+    end
+  end
+
+  def validate_properties_count(schema, errors, count = 0)
+    return count unless schema.is_a?(Hash)
+
+    if schema[:properties].is_a?(Hash)
+      count += schema[:properties].size
+
+      if count > 100
+        errors << "Schema exceeds maximum of 100 total object properties"
+        return count
+      end
+
+      # Check nested properties
+      schema[:properties].each_value do |property|
+        count = validate_properties_count(property, errors, count)
+      end
+    end
+
+    # Check array items
+    if schema[:type] == "array" && schema[:items].is_a?(Hash)
+      count = validate_properties_count(schema[:items], errors, count)
+    end
+
+    count
+  end
+
+  def validate_nesting_depth(schema, errors, depth = 1)
+    return unless schema.is_a?(Hash)
+
+    if depth > 5
+      errors << "Schema exceeds maximum nesting depth of 5 levels"
+      return
+    end
+
+    if schema[:properties].is_a?(Hash)
+      schema[:properties].each_value do |property|
+        validate_nesting_depth(property, errors, depth + 1)
+      end
+    end
+
+    # Check array items
+    if schema[:type] == "array" && schema[:items].is_a?(Hash)
+      validate_nesting_depth(schema[:items], errors, depth + 1)
+    end
+  end
+
+end

data/app/models/raif/concerns/llms/{open_ai → open_ai_completions}/message_formatting.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module Raif::Concerns::Llms::OpenAi::MessageFormatting
+module Raif::Concerns::Llms::OpenAiCompletions::MessageFormatting
   extend ActiveSupport::Concern
 
   def format_model_image_input_message(image_input)

data/app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Raif::Concerns::Llms::OpenAiCompletions::ToolFormatting
+  extend ActiveSupport::Concern
+
+  def build_tools_parameter(model_completion)
+    model_completion.available_model_tools_map.map do |_tool_name, tool|
+      if tool.provider_managed?
+        raise Raif::Errors::UnsupportedFeatureError,
+          "Raif doesn't yet support provider-managed tools for the OpenAI Completions API. Consider using the OpenAI Responses API instead."
+      else
+        # It's a developer-managed tool
+        validate_json_schema!(tool.tool_arguments_schema)
+
+        {
+          type: "function",
+          function: {
+            name: tool.tool_name,
+            description: tool.tool_description,
+            parameters: tool.tool_arguments_schema
+          }
+        }
+      end
+    end
+  end
+end

data/app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Raif::Concerns::Llms::OpenAiResponses::MessageFormatting
+  extend ActiveSupport::Concern
+
+  def format_string_message(content, role: nil)
+    if role == "assistant"
+      { "type" => "output_text", "text" => content }
+    else
+      { "type" => "input_text", "text" => content }
+    end
+  end
+
+  def format_model_image_input_message(image_input)
+    if image_input.source_type == :url
+      {
+        "type" => "input_image",
+        "image_url" => image_input.url
+      }
+    elsif image_input.source_type == :file_content
+      {
+        "type" => "input_image",
+        "image_url" => "data:#{image_input.content_type};base64,#{image_input.base64_data}"
+      }
+    else
+      raise Raif::Errors::InvalidModelImageInputError, "Invalid model image input source type: #{image_input.source_type}"
+    end
+  end
+
+  def format_model_file_input_message(file_input)
+    if file_input.source_type == :url
+      raise Raif::Errors::UnsupportedFeatureError, "#{self.class.name} does not support providing a file by URL"
+    elsif file_input.source_type == :file_content
+      {
+        "type" => "input_file",
+        "filename" => file_input.filename,
+        "file_data" => "data:#{file_input.content_type};base64,#{file_input.base64_data}"
+      }
+    else
+      raise Raif::Errors::InvalidModelFileInputError, "Invalid model image input source type: #{file_input.source_type}"
+    end
+  end
+end

data/app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Raif::Concerns::Llms::OpenAiResponses::ToolFormatting
+  extend ActiveSupport::Concern
+
+  def build_tools_parameter(model_completion)
+    model_completion.available_model_tools_map.map do |_tool_name, tool|
+      if tool.provider_managed?
+        format_provider_managed_tool(tool)
+      else
+        # It's a developer-managed tool
+        validate_json_schema!(tool.tool_arguments_schema)
+
+        {
+          type: "function",
+          name: tool.tool_name,
+          description: tool.tool_description,
+          parameters: tool.tool_arguments_schema
+        }
+      end
+    end
+  end
+
+  def format_provider_managed_tool(tool)
+    validate_provider_managed_tool_support!(tool)
+
+    case tool.name
+    when "Raif::ModelTools::ProviderManaged::WebSearch"
+      { type: "web_search_preview" }
+    when "Raif::ModelTools::ProviderManaged::CodeExecution"
+      {
+        type: "code_interpreter",
+        container: { "type": "auto" }
+      }
+    when "Raif::ModelTools::ProviderManaged::ImageGeneration"
+      { type: "image_generation" }
+    else
+      raise Raif::Errors::UnsupportedFeatureError,
+        "Invalid provider-managed tool: #{tool.name} for #{key}"
+    end
+  end
+end

data/app/models/raif/conversation.rb

@@ -16,10 +16,9 @@ class Raif::Conversation < Raif::ApplicationRecord
   after_initialize -> { self.available_user_tools ||= [] }
 
   before_validation ->{ self.type ||= "Raif::Conversation" }, on: :create
-  before_validation -> { self.system_prompt ||= build_system_prompt }, on: :create
 
   def build_system_prompt
-    <<~PROMPT
+    <<~PROMPT.strip
       #{system_prompt_intro}
       #{system_prompt_language_preference}
     PROMPT
@@ -35,14 +34,28 @@ class Raif::Conversation < Raif::ApplicationRecord
     I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
   end
 
-  def prompt_model_for_entry_response(entry:)
+  def prompt_model_for_entry_response(entry:, &block)
+    update(system_prompt: build_system_prompt)
+
     llm.chat(
       messages: llm_messages,
       source: entry,
       response_format: response_format.to_sym,
       system_prompt: system_prompt,
-      available_model_tools: available_model_tools
+      available_model_tools: available_model_tools,
+      &block
     )
+  rescue StandardError => e
+    Rails.logger.error("Error processing conversation entry ##{entry.id}. #{e.message}")
+    entry.failed!
+
+    if defined?(Airbrake)
+      notice = Airbrake.build_notice(e)
+      notice[:context][:component] = "raif_conversation"
+      notice[:context][:action] = "prompt_model_for_entry_response"
+
+      Airbrake.notify(notice)
+    end
   end
 
   def process_model_response_message(message:, entry:)

data/app/models/raif/conversation_entry.rb

@@ -16,7 +16,7 @@ class Raif::ConversationEntry < Raif::ApplicationRecord
   has_one :raif_model_completion, as: :source, dependent: :destroy, class_name: "Raif::ModelCompletion"
 
   delegate :available_model_tools, to: :raif_conversation
-  delegate :system_prompt, :llm_model_key, to: :raif_model_completion, allow_nil: true
+  delegate :system_prompt, :llm_model_key, :citations, to: :raif_model_completion, allow_nil: true
   delegate :json_response_schema, to: :class
 
   accepts_nested_attributes_for :raif_user_tool_invocation
@@ -46,7 +46,23 @@ class Raif::ConversationEntry < Raif::ApplicationRecord
   end
 
   def process_entry!
-    self.raif_model_completion = raif_conversation.prompt_model_for_entry_response(entry: self)
+    self.model_response_message = ""
+
+    self.raif_model_completion = raif_conversation.prompt_model_for_entry_response(entry: self) do |model_completion, _delta, _sse_event|
+      self.raw_response = model_completion.raw_response
+      self.model_response_message = raif_conversation.process_model_response_message(
+        message: model_completion.parsed_response(force_reparse: true),
+        entry: self
+      )
+
+      update_columns(
+        model_response_message: model_response_message,
+        raw_response: raw_response,
+        updated_at: Time.current
+      )
+
+      broadcast_replace_to raif_conversation
+    end
 
     if raif_model_completion.parsed_response.present? || raif_model_completion.response_tool_calls.present?
       extract_message_and_invoke_tools!

data/app/models/raif/embedding_models/{bedrock_titan.rb → bedrock.rb}

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-class Raif::EmbeddingModels::BedrockTitan < Raif::EmbeddingModel
+class Raif::EmbeddingModels::Bedrock < Raif::EmbeddingModel
 
   def generate_embedding!(input, dimensions: nil)
     unless input.is_a?(String)
-      raise ArgumentError, "Raif::EmbeddingModels::BedrockTitan#generate_embedding! input must be a string"
+      raise ArgumentError, "Raif::EmbeddingModels::Bedrock#generate_embedding! input must be a string"
     end
 
     params = build_request_parameters(input, dimensions:)

data/app/models/raif/llm.rb

@@ -12,7 +12,8 @@ module Raif
       :supports_native_tool_use,
       :provider_settings,
       :input_token_cost,
-      :output_token_cost
+      :output_token_cost,
+      :supported_provider_managed_tools
 
     validates :key, presence: true
     validates :api_name, presence: true
@@ -21,8 +22,17 @@ module Raif
 
     alias_method :supports_native_tool_use?, :supports_native_tool_use
 
-    def initialize(key:, api_name:, model_provider_settings: {}, supports_native_tool_use: true, temperature: nil, max_completion_tokens: nil,
-      input_token_cost: nil, output_token_cost: nil)
+    def initialize(
+      key:,
+      api_name:,
+      model_provider_settings: {},
+      supported_provider_managed_tools: [],
+      supports_native_tool_use: true,
+      temperature: nil,
+      max_completion_tokens: nil,
+      input_token_cost: nil,
+      output_token_cost: nil
+    )
       @key = key
       @api_name = api_name
       @provider_settings = model_provider_settings
@@ -31,6 +41,7 @@ module Raif
       @default_max_completion_tokens = max_completion_tokens
       @input_token_cost = input_token_cost
       @output_token_cost = output_token_cost
+      @supported_provider_managed_tools = supported_provider_managed_tools.map(&:to_s)
     end
 
     def name
@@ -38,7 +49,7 @@ module Raif
     end
 
     def chat(message: nil, messages: nil, response_format: :text, available_model_tools: [], source: nil, system_prompt: nil, temperature: nil,
-      max_completion_tokens: nil)
+      max_completion_tokens: nil, &block)
       unless response_format.is_a?(Symbol)
         raise ArgumentError,
           "Raif::Llm#chat - Invalid response format: #{response_format}. Must be a symbol (you passed #{response_format.class}) and be one of: #{VALID_RESPONSE_FORMATS.join(", ")}" # rubocop:disable Layout/LineLength
@@ -75,17 +86,25 @@ module Raif
         model_api_name: api_name,
         temperature: temperature,
         max_completion_tokens: max_completion_tokens,
-        available_model_tools: available_model_tools
+        available_model_tools: available_model_tools,
+        stream_response: block_given?
       )
 
       retry_with_backoff(model_completion) do
-        perform_model_completion!(model_completion)
+        perform_model_completion!(model_completion, &block)
       end
 
       model_completion
+    rescue Raif::Errors::StreamingError => e
+      Rails.logger.error("Raif streaming error -- code: #{e.code} -- type: #{e.type} -- message: #{e.message} -- event: #{e.event}")
+      raise e
+    rescue Faraday::Error => e
+      Raif.logger.error("LLM API request failed (status: #{e.response_status}): #{e.message}")
+      Raif.logger.error(e.response_body)
+      raise e
     end
 
-    def perform_model_completion!(model_completion)
+    def perform_model_completion!(model_completion, &block)
       raise NotImplementedError, "#{self.class.name} must implement #perform_model_completion!"
     end
 
@@ -93,6 +112,17 @@ module Raif
       VALID_RESPONSE_FORMATS
     end
 
+    def supports_provider_managed_tool?(tool_klass)
+      supported_provider_managed_tools&.include?(tool_klass.to_s)
+    end
+
+    def validate_provider_managed_tool_support!(tool)
+      unless supports_provider_managed_tool?(tool)
+        raise Raif::Errors::UnsupportedFeatureError,
+          "Invalid provider-managed tool: #{tool.name} for #{key}"
+      end
+    end
+
   private
 
     def retry_with_backoff(model_completion)
@@ -117,5 +147,41 @@ module Raif
         end
       end
     end
+
+    def streaming_response_type
+      raise NotImplementedError, "#{self.class.name} must implement #streaming_response_type"
+    end
+
+    def streaming_chunk_handler(model_completion, &block)
+      return unless model_completion.stream_response?
+
+      streaming_response = streaming_response_type.new
+      event_parser = EventStreamParser::Parser.new
+      accumulated_delta = ""
+
+      proc do |chunk, _size, _env|
+        event_parser.feed(chunk) do |event_type, data, _id, _reconnect_time|
+          if data.blank? || data == "[DONE]"
+            update_model_completion(model_completion, streaming_response.current_response_json)
+            next
+          end
+
+          event_data = JSON.parse(data)
+          delta, finish_reason = streaming_response.process_streaming_event(event_type, event_data)
+
+          accumulated_delta += delta if delta.present?
+
+          if accumulated_delta.length >= Raif.config.streaming_update_chunk_size_threshold || finish_reason.present?
+            update_model_completion(model_completion, streaming_response.current_response_json)
+
+            if accumulated_delta.present?
+              block.call(model_completion, accumulated_delta, event_data)
+              accumulated_delta = ""
+            end
+          end
+        end
+      end
+    end
+
   end
 end

data/app/models/raif/llms/anthropic.rb

@@ -2,29 +2,24 @@
 
 class Raif::Llms::Anthropic < Raif::Llm
   include Raif::Concerns::Llms::Anthropic::MessageFormatting
+  include Raif::Concerns::Llms::Anthropic::ToolFormatting
 
-  def perform_model_completion!(model_completion)
+  def perform_model_completion!(model_completion, &block)
     params = build_request_parameters(model_completion)
     response = connection.post("messages") do |req|
       req.body = params
+      req.options.on_data = streaming_chunk_handler(model_completion, &block) if model_completion.stream_response?
     end
 
-    response_json = response.body
-
-    model_completion.raw_response = if model_completion.response_format_json?
-      extract_json_response(response_json)
-    else
-      extract_text_response(response_json)
+    unless model_completion.stream_response?
+      update_model_completion(model_completion, response.body)
     end
 
-    model_completion.response_tool_calls = extract_response_tool_calls(response_json)
-    model_completion.completion_tokens = response_json&.dig("usage", "output_tokens")
-    model_completion.prompt_tokens = response_json&.dig("usage", "input_tokens")
-    model_completion.save!
-
     model_completion
   end
 
+private
+
   def connection
     @connection ||= Faraday.new(url: "https://api.anthropic.com/v1") do |f|
       f.headers["x-api-key"] = Raif.config.anthropic_api_key
@@ -35,7 +30,26 @@ class Raif::Llms::Anthropic < Raif::Llm
     end
   end
 
-protected
+  def streaming_response_type
+    Raif::StreamingResponses::Anthropic
+  end
+
+  def update_model_completion(model_completion, response_json)
+    model_completion.raw_response = if model_completion.response_format_json?
+      extract_json_response(response_json)
+    else
+      extract_text_response(response_json)
+    end
+
+    model_completion.response_id = response_json&.dig("id")
+    model_completion.response_array = response_json&.dig("content")
+    model_completion.response_tool_calls = extract_response_tool_calls(response_json)
+    model_completion.citations = extract_citations(response_json)
+    model_completion.completion_tokens = response_json&.dig("usage", "output_tokens")
+    model_completion.prompt_tokens = response_json&.dig("usage", "input_tokens")
+    model_completion.total_tokens = model_completion.completion_tokens.to_i + model_completion.prompt_tokens.to_i
+    model_completion.save!
+  end
 
   def build_request_parameters(model_completion)
     params = {
@@ -47,36 +61,20 @@ protected
 
     params[:system] = model_completion.system_prompt if model_completion.system_prompt.present?
 
-    # Add tools to the request if needed
-    tools = []
-
-    # If we're looking for a JSON response, add a tool to the request that the model can use to provide a JSON response
-    if model_completion.response_format_json? && model_completion.json_response_schema.present?
-      tools << {
-        name: "json_response",
-        description: "Generate a structured JSON response based on the provided schema.",
-        input_schema: model_completion.json_response_schema
-      }
-    end
-
-    # If we support native tool use and have tools available, add them to the request
-    if supports_native_tool_use? && model_completion.available_model_tools.any?
-      model_completion.available_model_tools_map.each do |_tool_name, tool|
-        tools << {
-          name: tool.tool_name,
-          description: tool.tool_description,
-          input_schema: tool.tool_arguments_schema
-        }
-      end
+    if supports_native_tool_use?
+      tools = build_tools_parameter(model_completion)
+      params[:tools] = tools unless tools.blank?
     end
 
-    params[:tools] = tools if tools.any?
+    params[:stream] = true if model_completion.stream_response?
 
     params
   end
 
   def extract_text_response(resp)
-    resp&.dig("content")&.first&.dig("text")
+    return if resp&.dig("content").blank?
+
+    resp.dig("content").select{|v| v["type"] == "text" }.map{|v| v["text"] }.join("\n")
   end
 
   def extract_json_response(resp)
@@ -112,4 +110,26 @@ protected
     end
   end
 
+  def extract_citations(resp)
+    return [] if resp&.dig("content").nil?
+
+    citations = []
+
+    # Look through content blocks for citations
+    resp.dig("content").each do |content|
+      next unless content["type"] == "text" && content["citations"].present?
+
+      content["citations"].each do |citation|
+        next unless citation["type"] == "web_search_result_location"
+
+        citations << {
+          "url" => Raif::Utils::HtmlFragmentProcessor.strip_tracking_parameters(citation["url"]),
+          "title" => citation["title"]
+        }
+      end
+    end
+
+    citations.uniq{|citation| citation["url"] }
+  end
+
 end