raif 1.2.2 → 1.4.0

data/lib/generators/raif/task/templates/task_eval_set.rb.tt

@@ -0,0 +1,54 @@
+<% raif_module_namespacing(["Evals", "Tasks"]) do -%>
+  class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
+    # Run this eval set with:
+    # bundle exec raif evals ./<%= eval_set_file_path %>
+    
+    # Setup method runs before each eval
+    setup do
+      # Common setup code
+    end
+
+    # Teardown runs after each eval
+    teardown do
+      # Cleanup code
+    end
+
+    eval "<%= class_name %> produces expected output" do
+      # task = Raif::Tasks::<%= class_name %>.run(
+      #   Add your task parameters here that produce the expected output
+      # )
+
+      # The return value of the block determines if the expectation passes or fails
+      # expect "task completes successfully" do
+      #   task.completed?
+      # end
+
+      # expect "contains the word 'hello' in the output" do
+      #   task.parsed_response.include?("hello")
+      # end
+
+      # Add more specific expectations based on your task's behavior
+    end
+
+    eval "properly handles refusals" do
+      # task = Raif::Tasks::<%= class_name %>.run(
+      #   Add your task parameters here to trigger a refusal
+      # )
+
+      # expect "returns exactly the text 'I'm sorry, I can't do that.'" do
+      #   task.parsed_response == "I'm sorry, I can't do that."
+      # end
+    end
+
+    eval "<%= class_name %> uses appropriate LLM tools" do
+      # Test that the task uses the expected tools if applicable
+      # task = Raif::Tasks::<%= class_name %>.run(
+      #   Add parameters that trigger the use of the expected tools
+      # )
+
+      # Example tool invocation expectations (if your task uses tools):
+      # expect_tool_invocation(task, "tool_name", with: { param: "value" })
+      # expect_no_tool_invocation(task, "unwanted_tool")
+    end
+  end
+<% end -%>

data/lib/raif/cli/base.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Raif
+  module CLI
+    class Base
+      attr_reader :args, :options
+
+      def initialize(args = [])
+        @args = args
+        @options = {}
+      end
+
+    protected
+
+      def find_rails_root
+        current = Dir.pwd
+
+        until File.exist?(File.join(current, "config", "environment.rb"))
+          parent = File.dirname(current)
+          if parent == current
+            puts "Error: Could not find Rails application root"
+            puts "Please run this command from within a Rails application directory"
+            exit 1
+          end
+
+          current = parent
+        end
+
+        current
+      end
+
+      def load_rails_application
+        rails_root = find_rails_root
+        Dir.chdir(rails_root)
+        require File.join(rails_root, "config", "environment")
+      end
+    end
+  end
+end

data/lib/raif/cli/evals.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "base"
+
+module Raif
+  module CLI
+    class Evals < Base
+      def run
+        # Set test environment by default for evals
+        ENV["RAILS_ENV"] ||= "test"
+        ENV["RAIF_RUNNING_EVALS"] = "true"
+
+        OptionParser.new do |opts|
+          opts.banner = "Usage: raif evals [options] [FILE_PATHS]"
+
+          opts.on("-e", "--environment ENV", "Rails environment (default: test)") do |env|
+            ENV["RAILS_ENV"] = env
+          end
+
+          opts.on("-h", "--help", "Show this help message") do
+            puts opts
+            exit
+          end
+        end.parse!(args)
+
+        # Parse file paths with optional line numbers
+        file_paths = args.map do |arg|
+          if arg.include?(":")
+            file_path, line_number = arg.split(":", 2)
+            { file_path: file_path, line_number: line_number.to_i }
+          else
+            { file_path: arg, line_number: nil }
+          end
+        end if args.any?
+
+        # Find and load Rails application
+        load_rails_application
+
+        require "raif/evals"
+
+        run = Raif::Evals::Run.new(file_paths: file_paths)
+        run.execute
+      end
+    end
+  end
+end

data/lib/raif/cli/evals_setup.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "base"
+
+module Raif
+  module CLI
+    class EvalsSetup < Base
+      def run
+        OptionParser.new do |opts|
+          opts.banner = "Usage: raif evals:setup [options]"
+          opts.on("-h", "--help", "Show this help message") do
+            puts opts
+            exit
+          end
+        end.parse!(args)
+
+        # Load Rails application to use generators
+        load_rails_application
+
+        # Invoke the Rails generator
+        require "rails/generators"
+        Rails::Generators.invoke("raif:evals:setup", args)
+      end
+    end
+  end
+end

data/lib/raif/cli.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "cli/base"
+require_relative "cli/evals"
+require_relative "cli/evals_setup"
+
+module Raif
+  module CLI
+    COMMANDS = {
+      "evals" => "Run Raif evaluation sets",
+      "evals:setup" => "Setup Raif evals directory structure",
+      "version" => "Show Raif version",
+      "help" => "Show this help message"
+    }.freeze
+
+    class Runner
+      def initialize(args)
+        @args = args
+        @command = args.shift
+      end
+
+      def run
+        case @command
+        when "evals"
+          Evals.new(@args).run
+        when "evals:setup"
+          EvalsSetup.new(@args).run
+        when "version", "--version", "-v"
+          show_version
+        when "help", "--help", "-h", nil
+          show_help
+        else
+          puts "Unknown command: #{@command}"
+          puts ""
+          show_help
+          exit 1
+        end
+      end
+
+    private
+
+      def show_version
+        require_relative "../raif/version"
+        puts "Raif #{Raif::VERSION}"
+      end
+
+      def show_help
+        puts "Usage: raif COMMAND [options]"
+        puts ""
+        puts "Commands:"
+        COMMANDS.each do |command, description|
+          puts format("  %-12s %s", command, description)
+        end
+        puts ""
+        puts "For help on a specific command:"
+        puts "  raif COMMAND --help"
+        puts ""
+        puts "Examples:"
+        puts "  raif evals:setup                  # Setup eval directory structure"
+        puts "  raif evals                        # Run all eval sets in test environment"
+        puts "  raif evals CustomerSupportEvalSet # Run specific eval set"
+        puts "  raif evals -e development         # Run evals in development environment"
+        puts "  raif version                      # Show Raif version"
+      end
+    end
+  end
+end

data/lib/raif/configuration.rb

@@ -12,24 +12,37 @@ module Raif
       :aws_bedrock_region,
       :bedrock_embedding_models_enabled,
       :conversation_entries_controller,
+      :conversation_llm_messages_max_length_default,
       :conversation_system_prompt_intro,
       :conversation_types,
       :conversations_controller,
       :current_user_method,
       :default_embedding_model_key,
       :default_llm_model_key,
+      :evals_default_llm_judge_model_key,
+      :evals_verbose_output,
+      :google_api_key,
+      :google_models_enabled,
       :llm_api_requests_enabled,
       :llm_request_max_retries,
       :llm_request_retriable_exceptions,
       :model_superclass,
       :open_ai_api_key,
+      :open_ai_api_version,
+      :open_ai_auth_header_style,
+      :open_ai_base_url,
+      :open_ai_embedding_base_url,
       :open_ai_embedding_models_enabled,
       :open_ai_models_enabled,
       :open_router_api_key,
       :open_router_models_enabled,
       :open_router_app_name,
       :open_router_site_url,
+      :request_open_timeout,
+      :request_read_timeout,
+      :request_write_timeout,
       :streaming_update_chunk_size_threshold,
+      :task_creator_optional,
       :task_system_prompt_intro,
       :user_tool_types
 
@@ -40,9 +53,8 @@ module Raif
     alias_method :aws_bedrock_titan_embedding_models_enabled=, :bedrock_embedding_models_enabled=
 
     def initialize
-      # Set default config
-      @agent_types = Set.new(["Raif::Agents::ReActAgent", "Raif::Agents::NativeToolCallingAgent"])
-      @anthropic_api_key = ENV["ANTHROPIC_API_KEY"]
+      @agent_types = Set.new(["Raif::Agents::NativeToolCallingAgent"])
+      @anthropic_api_key = default_disable_llm_api_requests? ? "placeholder-anthropic-api-key" : ENV["ANTHROPIC_API_KEY"]
       @bedrock_models_enabled = false
       @anthropic_models_enabled = ENV["ANTHROPIC_API_KEY"].present?
       @authorize_admin_controller_action = ->{ false }
@@ -52,28 +64,45 @@ module Raif
       @bedrock_embedding_models_enabled = false
       @task_system_prompt_intro = "You are a helpful assistant."
       @conversation_entries_controller = "Raif::ConversationEntriesController"
+      @conversation_llm_messages_max_length_default = 50
       @conversation_system_prompt_intro = "You are a helpful assistant who is collaborating with a teammate."
       @conversation_types = Set.new(["Raif::Conversation"])
       @conversations_controller = "Raif::ConversationsController"
       @current_user_method = :current_user
       @default_embedding_model_key = "open_ai_text_embedding_3_small"
-      @default_llm_model_key = "open_ai_gpt_4o"
-      @llm_api_requests_enabled = true
+      @default_llm_model_key = default_disable_llm_api_requests? ? :raif_test_llm : (ENV["RAIF_DEFAULT_LLM_MODEL_KEY"].presence || "open_ai_gpt_4o")
+      @evals_default_llm_judge_model_key = ENV["RAIF_EVALS_DEFAULT_LLM_JUDGE_MODEL_KEY"].presence
+      @evals_verbose_output = false
+      google_api_key = ENV["GOOGLE_AI_API_KEY"].presence || ENV["GOOGLE_API_KEY"]
+      @google_api_key = default_disable_llm_api_requests? ? "placeholder-google-api-key" : google_api_key
+      @google_models_enabled = @google_api_key.present?
+      @llm_api_requests_enabled = !default_disable_llm_api_requests?
       @llm_request_max_retries = 2
       @llm_request_retriable_exceptions = [
         Faraday::ConnectionFailed,
         Faraday::TimeoutError,
         Faraday::ServerError,
+        Net::ReadTimeout,
+        Net::OpenTimeout,
       ]
       @model_superclass = "ApplicationRecord"
-      @open_ai_api_key = ENV["OPENAI_API_KEY"]
+      @open_ai_api_key = default_disable_llm_api_requests? ? "placeholder-open-ai-api-key" : ENV["OPENAI_API_KEY"]
+      @open_ai_api_version = nil
+      @open_ai_auth_header_style = :bearer
+      @open_ai_base_url = "https://api.openai.com/v1"
+      @open_ai_embedding_base_url = "https://api.openai.com/v1"
       @open_ai_embedding_models_enabled = ENV["OPENAI_API_KEY"].present?
       @open_ai_models_enabled = ENV["OPENAI_API_KEY"].present?
-      @open_router_api_key = ENV["OPEN_ROUTER_API_KEY"].presence || ENV["OPENROUTER_API_KEY"]
+      open_router_api_key = ENV["OPEN_ROUTER_API_KEY"].presence || ENV["OPENROUTER_API_KEY"]
+      @open_router_api_key = default_disable_llm_api_requests? ? "placeholder-open-router-api-key" : open_router_api_key
       @open_router_models_enabled = @open_router_api_key.present?
       @open_router_app_name = nil
       @open_router_site_url = nil
+      @request_open_timeout = nil
+      @request_read_timeout = nil
+      @request_write_timeout = nil
       @streaming_update_chunk_size_threshold = 25
+      @task_creator_optional = true
       @user_tool_types = []
     end
 
@@ -97,7 +126,9 @@ module Raif
           "Raif.config.default_llm_model_key was set to #{default_llm_model_key}, but must be one of: #{Raif.available_llm_keys.join(", ")}"
       end
 
-      if Raif.embedding_model_registry.present? && !Raif.available_embedding_model_keys.include?(default_embedding_model_key.to_sym)
+      if default_embedding_model_key.present? &&
+          Raif.embedding_model_registry.present? &&
+          !Raif.available_embedding_model_keys.include?(default_embedding_model_key.to_sym)
         raise Raif::Errors::InvalidConfigError,
           "Raif.config.default_embedding_model_key was set to #{default_embedding_model_key}, but must be one of: #{Raif.available_embedding_model_keys.join(", ")}" # rubocop:disable Layout/LineLength
       end
@@ -121,6 +152,11 @@ module Raif
           "Raif.config.open_ai_api_key is required when Raif.config.open_ai_models_enabled is true. Set it via Raif.config.open_ai_api_key or ENV[\"OPENAI_API_KEY\"]" # rubocop:disable Layout/LineLength
       end
 
+      if open_ai_models_enabled && ![:bearer, :api_key].include?(open_ai_auth_header_style)
+        raise Raif::Errors::InvalidConfigError,
+          "Raif.config.open_ai_auth_header_style must be either :bearer or :api_key"
+      end
+
       if open_ai_embedding_models_enabled && open_ai_api_key.blank?
         raise Raif::Errors::InvalidConfigError,
           "Raif.config.open_ai_api_key is required when Raif.config.open_ai_embedding_models_enabled is true. Set it via Raif.config.open_ai_api_key or ENV[\"OPENAI_API_KEY\"]" # rubocop:disable Layout/LineLength
@@ -135,6 +171,19 @@ module Raif
         raise Raif::Errors::InvalidConfigError,
           "Raif.config.open_router_api_key is required when Raif.config.open_router_models_enabled is true. Set it via Raif.config.open_router_api_key or ENV['OPEN_ROUTER_API_KEY']" # rubocop:disable Layout/LineLength
       end
+
+      if google_models_enabled && google_api_key.blank?
+        raise Raif::Errors::InvalidConfigError,
+          "Raif.config.google_api_key is required when Raif.config.google_models_enabled is true. Set it via Raif.config.google_api_key or ENV['GOOGLE_API_KEY']" # rubocop:disable Layout/LineLength
+      end
+    end
+
+  private
+
+    # By default, evals run in the test environment, but need real API keys.
+    # In normal tests, we insert placeholders to make it hard to accidentally rack up an LLM API bill.
+    def default_disable_llm_api_requests?
+      Rails.env.test? && !Raif.running_evals?
     end
 
   end

data/lib/raif/engine.rb

@@ -72,6 +72,14 @@ module Raif
       end
     end
 
+    config.after_initialize do
+      next unless Raif.config.google_models_enabled
+
+      Raif.default_llms[Raif::Llms::Google].each do |llm_config|
+        Raif.register_llm(Raif::Llms::Google, **llm_config)
+      end
+    end
+
     config.after_initialize do
       next unless Raif.config.bedrock_embedding_models_enabled
 

data/lib/raif/errors/instance_dependent_schema_error.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Raif
+  module Errors
+    class InstanceDependentSchemaError < StandardError
+    end
+  end
+end

data/lib/raif/errors/streaming_error.rb

@@ -3,16 +3,19 @@
 module Raif
   module Errors
     class StreamingError < StandardError
-      attr_reader :message, :type, :code, :event
+      attr_reader :type, :code, :event
 
       def initialize(message:, type:, event:, code: nil)
-        super
+        super(message)
 
-        @message = message
         @type = type
         @code = code
         @event = event
       end
+
+      def to_s
+        "[#{type}] #{super} (code=#{code}, event=#{event})"
+      end
     end
   end
 end

data/lib/raif/errors.rb

@@ -9,3 +9,4 @@ require "raif/errors/invalid_model_image_input_error"
 require "raif/errors/invalid_model_file_input_error"
 require "raif/errors/unsupported_feature_error"
 require "raif/errors/streaming_error"
+require "raif/errors/instance_dependent_schema_error"

data/lib/raif/evals/eval.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    class Eval
+      attr_reader :description, :expectation_results
+
+      def initialize(description:)
+        @description = description
+        @expectation_results = []
+      end
+
+      def add_expectation_result(result)
+        @expectation_results << result
+      end
+
+      def passed?
+        expectation_results.all?(&:passed?)
+      end
+
+      def to_h
+        {
+          description: description,
+          passed: passed?,
+          expectation_results: expectation_results.map(&:to_h)
+        }
+      end
+    end
+  end
+end

data/lib/raif/evals/eval_set.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "raif/evals/eval_sets/expectations"
+require "raif/evals/eval_sets/llm_judge_expectations"
+
+module Raif
+  module Evals
+    class EvalSet
+      include Raif::Evals::EvalSets::Expectations
+      include Raif::Evals::EvalSets::LlmJudgeExpectations
+
+      attr_reader :current_eval, :output, :results
+
+      def initialize(output: $stdout)
+        @output = output
+      end
+
+      class << self
+        attr_reader :setup_block
+        attr_reader :teardown_block
+
+        def inherited(subclass)
+          subclass.instance_variable_set(:@evals, [])
+          super
+        end
+
+        def evals
+          @evals ||= []
+        end
+
+        def eval(description, &block)
+          evals << { description: description, block: block, definition_line_number: caller_locations(1, 1).first.lineno }
+        end
+
+        def setup(&block)
+          @setup_block = block
+        end
+
+        def teardown(&block)
+          @teardown_block = block
+        end
+
+        def run(output: $stdout)
+          new(output: output).run
+        end
+      end
+
+      def run
+        @results = []
+
+        self.class.evals.each do |eval_definition|
+          @results << run_eval(eval_definition)
+        end
+
+        @results
+      end
+
+      def run_eval(eval_definition)
+        @current_eval = Eval.new(description: eval_definition[:description])
+
+        output.puts "Running: #{eval_definition[:description]}"
+
+        ActiveRecord::Base.transaction do
+          instance_eval(&self.class.setup_block) if self.class.setup_block
+
+          begin
+            instance_eval(&eval_definition[:block])
+          rescue => e
+            output.puts Raif::Utils::Colors.red("  Error in eval block: #{e.message}")
+            output.puts Raif::Utils::Colors.red("  #{e.backtrace.join("\n  ")}")
+            @current_eval.add_expectation_result(
+              ExpectationResult.new(
+                description: "Eval block execution",
+                status: :error,
+                error: e
+              )
+            )
+          ensure
+            instance_eval(&self.class.teardown_block) if self.class.teardown_block
+          end
+
+          raise ActiveRecord::Rollback
+        end
+
+        @current_eval
+      end
+
+      def file(filename)
+        # Validate filename to prevent directory traversal
+        raise ArgumentError, "Invalid filename: cannot be empty" if filename.nil? || filename.empty?
+        raise ArgumentError, "Invalid filename: cannot contain '..' or absolute paths" if filename.include?("..") || filename.start_with?("/")
+
+        # Ensure we're only accessing files within the raif_evals/files directory
+        base_path = Rails.root.join("raif_evals", "files")
+        full_path = base_path.join(filename)
+
+        # Verify the resolved path is within the expected directory
+        unless full_path.to_s.start_with?(base_path.to_s)
+          raise ArgumentError, "Invalid filename: path traversal detected"
+        end
+
+        if full_path.exist?
+          full_path.read
+        else
+          raise ArgumentError, "File #{filename} does not exist in raif_evals/files/"
+        end
+      end
+
+    end
+  end
+end

data/lib/raif/evals/eval_sets/expectations.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    module EvalSets
+      module Expectations
+
+        def expect(description, result_metadata: nil, &block)
+          result = begin
+            if block.call
+              output.puts Raif::Utils::Colors.green("  ✓ #{description}")
+              output.puts Raif::Utils::Colors.green("    ⎿ #{result_metadata.inspect}") if result_metadata && Raif.config.evals_verbose_output
+              ExpectationResult.new(description: description, status: :passed, metadata: result_metadata)
+            else
+              output.puts Raif::Utils::Colors.red("  ✗ #{description}")
+              output.puts Raif::Utils::Colors.red("    ⎿ #{result_metadata.inspect}") if result_metadata && Raif.config.evals_verbose_output
+              ExpectationResult.new(description: description, status: :failed, metadata: result_metadata)
+            end
+          rescue => e
+            output.puts Raif::Utils::Colors.red("  ✗ #{description} (Error: #{e.message})")
+            ExpectationResult.new(description: description, status: :error, error: e, metadata: result_metadata)
+          end
+
+          current_eval.add_expectation_result(result)
+          result
+        end
+
+        def expect_tool_invocation(tool_invoker, tool_type, with: {})
+          invocations = tool_invoker.raif_model_tool_invocations.select { |inv| inv.tool_type == tool_type }
+          invoked_tools = tool_invoker.raif_model_tool_invocations.map{|inv| [inv.tool_type, inv.tool_arguments] }.to_h
+
+          if with.any?
+            invocations = invocations.select do |invocation|
+              with.all? { |key, value| invocation.tool_arguments[key.to_s] == value }
+            end
+          end
+
+          result_metadata = { invoked_tools: invoked_tools }
+          expect "invokes #{tool_type}#{with.any? ? " with #{with.to_json}" : ""}", result_metadata: result_metadata do
+            invocations.any?
+          end
+        end
+
+        def expect_no_tool_invocation(tool_invoker, tool_name)
+          expect "does not invoke #{tool_name}" do
+            tool_invoker.raif_model_tool_invocations.none? { |inv| inv.tool_name == tool_name }
+          end
+        end
+
+      end
+    end
+  end
+end