raif 1.2.1 → 1.3.0

data/lib/raif/configuration.rb

@@ -18,6 +18,8 @@ module Raif
       :current_user_method,
       :default_embedding_model_key,
       :default_llm_model_key,
+      :evals_default_llm_judge_model_key,
+      :evals_verbose_output,
       :llm_api_requests_enabled,
       :llm_request_max_retries,
       :llm_request_retriable_exceptions,
@@ -30,6 +32,7 @@ module Raif
       :open_router_app_name,
       :open_router_site_url,
       :streaming_update_chunk_size_threshold,
+      :task_creator_optional,
       :task_system_prompt_intro,
       :user_tool_types
 
@@ -40,9 +43,8 @@ module Raif
     alias_method :aws_bedrock_titan_embedding_models_enabled=, :bedrock_embedding_models_enabled=
 
     def initialize
-      # Set default config
       @agent_types = Set.new(["Raif::Agents::ReActAgent", "Raif::Agents::NativeToolCallingAgent"])
-      @anthropic_api_key = ENV["ANTHROPIC_API_KEY"]
+      @anthropic_api_key = default_disable_llm_api_requests? ? "placeholder-anthropic-api-key" : ENV["ANTHROPIC_API_KEY"]
       @bedrock_models_enabled = false
       @anthropic_models_enabled = ENV["ANTHROPIC_API_KEY"].present?
       @authorize_admin_controller_action = ->{ false }
@@ -57,8 +59,10 @@ module Raif
       @conversations_controller = "Raif::ConversationsController"
       @current_user_method = :current_user
       @default_embedding_model_key = "open_ai_text_embedding_3_small"
-      @default_llm_model_key = "open_ai_gpt_4o"
-      @llm_api_requests_enabled = true
+      @default_llm_model_key = default_disable_llm_api_requests? ? :raif_test_llm : (ENV["RAIF_DEFAULT_LLM_MODEL_KEY"].presence || "open_ai_gpt_4o")
+      @evals_default_llm_judge_model_key = ENV["RAIF_EVALS_DEFAULT_LLM_JUDGE_MODEL_KEY"].presence
+      @evals_verbose_output = false
+      @llm_api_requests_enabled = !default_disable_llm_api_requests?
       @llm_request_max_retries = 2
       @llm_request_retriable_exceptions = [
         Faraday::ConnectionFailed,
@@ -66,14 +70,16 @@ module Raif
         Faraday::ServerError,
       ]
       @model_superclass = "ApplicationRecord"
-      @open_ai_api_key = ENV["OPENAI_API_KEY"]
+      @open_ai_api_key = default_disable_llm_api_requests? ? "placeholder-open-ai-api-key" : ENV["OPENAI_API_KEY"]
       @open_ai_embedding_models_enabled = ENV["OPENAI_API_KEY"].present?
       @open_ai_models_enabled = ENV["OPENAI_API_KEY"].present?
-      @open_router_api_key = ENV["OPENROUTER_API_KEY"]
-      @open_router_models_enabled = ENV["OPENROUTER_API_KEY"].present?
+      open_router_api_key = ENV["OPEN_ROUTER_API_KEY"].presence || ENV["OPENROUTER_API_KEY"]
+      @open_router_api_key = default_disable_llm_api_requests? ? "placeholder-open-router-api-key" : open_router_api_key
+      @open_router_models_enabled = @open_router_api_key.present?
       @open_router_app_name = nil
       @open_router_site_url = nil
       @streaming_update_chunk_size_threshold = 25
+      @task_creator_optional = true
       @user_tool_types = []
     end
 
@@ -82,7 +88,7 @@ module Raif
         puts <<~EOS
 
           !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-          No LLMs are enabled in Raif. Make sure you have an API key configured for at least one LLM provider. You can do this by setting an API key in your environment variables or in config/initializers/raif.rb (e.g. ENV["OPENAI_API_KEY"], ENV["ANTHROPIC_API_KEY"], ENV["OPENROUTER_API_KEY"]).
+          No LLMs are enabled in Raif. Make sure you have an API key configured for at least one LLM provider. You can do this by setting an API key in your environment variables or in config/initializers/raif.rb (e.g. ENV["OPENAI_API_KEY"], ENV["ANTHROPIC_API_KEY"], ENV["OPEN_ROUTER_API_KEY"]).
 
           See the README for more information: https://github.com/CultivateLabs/raif#setup
           !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -133,9 +139,17 @@ module Raif
 
       if open_router_models_enabled && open_router_api_key.blank?
         raise Raif::Errors::InvalidConfigError,
-          "Raif.config.open_router_api_key is required when Raif.config.open_router_models_enabled is true. Set it via Raif.config.open_router_api_key or ENV['OPENROUTER_API_KEY']" # rubocop:disable Layout/LineLength
+          "Raif.config.open_router_api_key is required when Raif.config.open_router_models_enabled is true. Set it via Raif.config.open_router_api_key or ENV['OPEN_ROUTER_API_KEY']" # rubocop:disable Layout/LineLength
       end
     end
 
+  private
+
+    # By default, evals run in the test environment, but need real API keys.
+    # In normal tests, we insert placeholders to make it hard to accidentally rack up an LLM API bill.
+    def default_disable_llm_api_requests?
+      Rails.env.test? && !Raif.running_evals?
+    end
+
   end
 end

data/lib/raif/engine.rb

@@ -120,7 +120,8 @@ module Raif
         Rails.application.config.assets.precompile += [
           "raif.js",
           "raif.css",
-          "raif_admin.css"
+          "raif_admin.css",
+          "raif-logo-white.svg"
         ]
       end
     end

data/lib/raif/evals/eval.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    class Eval
+      attr_reader :description, :expectation_results
+
+      def initialize(description:)
+        @description = description
+        @expectation_results = []
+      end
+
+      def add_expectation_result(result)
+        @expectation_results << result
+      end
+
+      def passed?
+        expectation_results.all?(&:passed?)
+      end
+
+      def to_h
+        {
+          description: description,
+          passed: passed?,
+          expectation_results: expectation_results.map(&:to_h)
+        }
+      end
+    end
+  end
+end

data/lib/raif/evals/eval_set.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "raif/evals/eval_sets/expectations"
+require "raif/evals/eval_sets/llm_judge_expectations"
+
+module Raif
+  module Evals
+    class EvalSet
+      include Raif::Evals::EvalSets::Expectations
+      include Raif::Evals::EvalSets::LlmJudgeExpectations
+
+      attr_reader :current_eval, :output, :results
+
+      def initialize(output: $stdout)
+        @output = output
+      end
+
+      class << self
+        attr_reader :setup_block
+        attr_reader :teardown_block
+
+        def inherited(subclass)
+          subclass.instance_variable_set(:@evals, [])
+          super
+        end
+
+        def evals
+          @evals ||= []
+        end
+
+        def eval(description, &block)
+          evals << { description: description, block: block, definition_line_number: caller_locations(1, 1).first.lineno }
+        end
+
+        def setup(&block)
+          @setup_block = block
+        end
+
+        def teardown(&block)
+          @teardown_block = block
+        end
+
+        def run(output: $stdout)
+          new(output: output).run
+        end
+      end
+
+      def run
+        @results = []
+
+        self.class.evals.each do |eval_definition|
+          @results << run_eval(eval_definition)
+        end
+
+        @results
+      end
+
+      def run_eval(eval_definition)
+        @current_eval = Eval.new(description: eval_definition[:description])
+
+        output.puts "Running: #{eval_definition[:description]}"
+
+        ActiveRecord::Base.transaction do
+          instance_eval(&self.class.setup_block) if self.class.setup_block
+
+          begin
+            instance_eval(&eval_definition[:block])
+          rescue => e
+            output.puts Raif::Utils::Colors.red("  Error in eval block: #{e.message}")
+            output.puts Raif::Utils::Colors.red("  #{e.backtrace.join("\n  ")}")
+            @current_eval.add_expectation_result(
+              ExpectationResult.new(
+                description: "Eval block execution",
+                status: :error,
+                error: e
+              )
+            )
+          ensure
+            instance_eval(&self.class.teardown_block) if self.class.teardown_block
+          end
+
+          raise ActiveRecord::Rollback
+        end
+
+        @current_eval
+      end
+
+      def file(filename)
+        # Validate filename to prevent directory traversal
+        raise ArgumentError, "Invalid filename: cannot be empty" if filename.nil? || filename.empty?
+        raise ArgumentError, "Invalid filename: cannot contain '..' or absolute paths" if filename.include?("..") || filename.start_with?("/")
+
+        # Ensure we're only accessing files within the raif_evals/files directory
+        base_path = Rails.root.join("raif_evals", "files")
+        full_path = base_path.join(filename)
+
+        # Verify the resolved path is within the expected directory
+        unless full_path.to_s.start_with?(base_path.to_s)
+          raise ArgumentError, "Invalid filename: path traversal detected"
+        end
+
+        if full_path.exist?
+          full_path.read
+        else
+          raise ArgumentError, "File #{filename} does not exist in raif_evals/files/"
+        end
+      end
+
+    end
+  end
+end

data/lib/raif/evals/eval_sets/expectations.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    module EvalSets
+      module Expectations
+
+        def expect(description, result_metadata: nil, &block)
+          result = begin
+            if block.call
+              output.puts Raif::Utils::Colors.green("  ✓ #{description}")
+              output.puts Raif::Utils::Colors.green("    ⎿ #{result_metadata.inspect}") if result_metadata && Raif.config.evals_verbose_output
+              ExpectationResult.new(description: description, status: :passed, metadata: result_metadata)
+            else
+              output.puts Raif::Utils::Colors.red("  ✗ #{description}")
+              output.puts Raif::Utils::Colors.red("    ⎿ #{result_metadata.inspect}") if result_metadata && Raif.config.evals_verbose_output
+              ExpectationResult.new(description: description, status: :failed, metadata: result_metadata)
+            end
+          rescue => e
+            output.puts Raif::Utils::Colors.red("  ✗ #{description} (Error: #{e.message})")
+            ExpectationResult.new(description: description, status: :error, error: e, metadata: result_metadata)
+          end
+
+          current_eval.add_expectation_result(result)
+          result
+        end
+
+        def expect_tool_invocation(tool_invoker, tool_type, with: {})
+          invocations = tool_invoker.raif_model_tool_invocations.select { |inv| inv.tool_type == tool_type }
+          invoked_tools = tool_invoker.raif_model_tool_invocations.map{|inv| [inv.tool_type, inv.tool_arguments] }.to_h
+
+          if with.any?
+            invocations = invocations.select do |invocation|
+              with.all? { |key, value| invocation.tool_arguments[key.to_s] == value }
+            end
+          end
+
+          result_metadata = { invoked_tools: invoked_tools }
+          expect "invokes #{tool_type}#{with.any? ? " with #{with.to_json}" : ""}", result_metadata: result_metadata do
+            invocations.any?
+          end
+        end
+
+        def expect_no_tool_invocation(tool_invoker, tool_name)
+          expect "does not invoke #{tool_name}" do
+            tool_invoker.raif_model_tool_invocations.none? { |inv| inv.tool_name == tool_name }
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/raif/evals/eval_sets/llm_judge_expectations.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    module EvalSets
+      module LlmJudgeExpectations
+
+        # Uses an LLM judge to evaluate whether content meets specific criteria with a binary pass/fail result.
+        #
+        # This method leverages the Binary LLM judge to assess content against provided criteria,
+        # returning a pass or fail judgment with reasoning and confidence scores.
+        #
+        # @param content [String] The content to be evaluated by the LLM judge
+        # @param criteria [String] The evaluation criteria that the content must meet
+        # @param examples [Array<Hash>] Optional examples showing how to evaluate similar content.
+        #   Each example should have keys: :content, :passes (boolean), :reasoning
+        # @param strict [Boolean] Whether to apply criteria strictly (true) or with reasonable judgment (false)
+        # @param llm_judge_model_key [Symbol, nil] Optional specific LLM model to use for judging.
+        #   If nil, uses the configured default judge model or falls back to default LLM
+        # @param additional_context [String, nil] Optional additional context to be provided to the judge
+        #
+        # @return [ExpectationResult] Result object containing pass/fail status and judge metadata
+        #
+        # @example Basic usage
+        #   expect_llm_judge_passes(
+        #     task.parsed_response,
+        #     criteria: "Response is polite and professional"
+        #   )
+        #
+        # @example With examples and strict mode
+        #   expect_llm_judge_passes(
+        #     content,
+        #     criteria: "Contains a proper greeting",
+        #     examples: [
+        #       { content: "Hello, how can I help?", passes: true, reasoning: "Contains greeting" },
+        #       { content: "What do you want?", passes: false, reasoning: "No greeting, rude tone" }
+        #     ],
+        #     strict: true
+        #   )
+        #
+        # @note The judge result includes metadata accessible via expectation_result.metadata:
+        #   - :passes - Boolean result
+        #   - :reasoning - Detailed explanation
+        #   - :confidence - Confidence score (0.0-1.0)
+        def expect_llm_judge_passes(content, criteria:, examples: [], strict: false, llm_judge_model_key: nil, additional_context: nil,
+          result_metadata: {})
+          judge_task = LlmJudges::Binary.run(
+            content_to_judge: content,
+            criteria: criteria,
+            examples: examples,
+            strict_mode: strict,
+            llm_model_key: llm_judge_model_key,
+            additional_context: additional_context
+          )
+
+          if judge_task.low_confidence? && output.respond_to?(:puts)
+            output.puts Raif::Utils::Colors.yellow("    ⚠ Low confidence: #{judge_task.judgment_confidence}")
+          end
+
+          if Raif.config.evals_verbose_output && output.respond_to?(:puts)
+            output.puts "    #{judge_task.judgment_reasoning}"
+          end
+
+          judge_metadata = {
+            passes: judge_task.passes?,
+            reasoning: judge_task.judgment_reasoning,
+            confidence: judge_task.judgment_confidence,
+          }.compact
+
+          # Merge user metadata with judge metadata
+          combined_metadata = result_metadata.merge(judge_metadata)
+
+          expectation_result = expect "LLM judge: #{criteria}", result_metadata: combined_metadata do
+            judge_task.passes?
+          end
+
+          if expectation_result && judge_task.errors.any?
+            expectation_result.error_message = judge_task.errors.full_messages.join(", ")
+          end
+
+          expectation_result
+        end
+
+        # Uses an LLM judge to evaluate content with a numerical score based on a detailed rubric.
+        #
+        # This method leverages the Scored LLM judge to assess content against a scoring rubric,
+        # providing a numerical score with detailed reasoning and determining pass/fail based on
+        # the minimum passing score threshold.
+        #
+        # @param output [String] The content to be evaluated by the LLM judge
+        # @param scoring_rubric [ScoringRubric, String] The rubric to use for scoring. Can be a
+        #   ScoringRubric object with structured levels or a plain string description
+        # @param min_passing_score [Integer] Minimum score required to pass
+        # @param llm_judge_model_key [Symbol, nil] Optional specific LLM model to use for judging.
+        #   If nil, uses the configured default judge model or falls back to default LLM
+        # @param additional_context [String, nil] Optional additional context to be provided to the judge
+        #
+        # @return [ExpectationResult] Result object containing pass/fail status and judge metadata
+        #
+        # @example Using a built-in rubric
+        #   expect_llm_judge_score(
+        #     task.parsed_response,
+        #     scoring_rubric: ScoringRubric.accuracy,
+        #     min_passing_score: 8
+        #   )
+        #
+        # @example Using a custom rubric
+        #   rubric = ScoringRubric.new(
+        #     name: :technical_writing,
+        #     description: "Evaluates technical writing quality",
+        #     levels: [
+        #       { score_range: (9..10), description: "Expert-level technical content" },
+        #       { score_range: (7..8), description: "Strong technical content" },
+        #       { score_range: (5..6), description: "Adequate technical content" },
+        #       { score_range: (3..4), description: "Weak technical content" },
+        #       { score_range: (0..2), description: "Poor technical content" }
+        #     ]
+        #   )
+        #   expect_llm_judge_score(output, scoring_rubric: rubric, min_passing_score: 7)
+        #
+        # @example Using a simple string rubric
+        #   expect_llm_judge_score(
+        #     output,
+        #     scoring_rubric: "Rate clarity from 0-5 where 5 is crystal clear",
+        #     min_passing_score: 4
+        #   )
+        #
+        # @note The judge result includes metadata accessible via expectation_result.metadata:
+        #   - :score - Numerical score given
+        #   - :reasoning - Detailed explanation
+        #   - :confidence - Confidence score (0.0-1.0)
+        def expect_llm_judge_score(output, scoring_rubric:, min_passing_score:, llm_judge_model_key: nil, additional_context: nil,
+          result_metadata: {})
+          scoring_rubric_obj = scoring_rubric
+
+          judge_task = LlmJudges::Scored.run(
+            content_to_judge: output,
+            scoring_rubric: scoring_rubric_obj,
+            llm_model_key: llm_judge_model_key,
+            additional_context: additional_context
+          )
+
+          rubric_name = scoring_rubric_obj.respond_to?(:name) ? scoring_rubric_obj.name : "custom"
+          if output.respond_to?(:puts)
+            output.puts "    Score: #{judge_task.judgment_score}"
+            output.puts "    #{judge_task.judgment_reasoning}" if Raif.config.evals_verbose_output
+          end
+
+          judge_metadata = {
+            score: judge_task.judgment_score,
+            reasoning: judge_task.judgment_reasoning,
+            confidence: judge_task.judgment_confidence,
+          }.compact
+
+          # Merge user metadata with judge metadata
+          combined_metadata = result_metadata.merge(judge_metadata)
+
+          expectation_result = expect "LLM judge score (#{rubric_name}): >= #{min_passing_score}", result_metadata: combined_metadata do
+            judge_task.completed? && judge_task.judgment_score && judge_task.judgment_score >= min_passing_score
+          end
+
+          if expectation_result && judge_task.errors.any?
+            expectation_result.error_message = judge_task.errors.full_messages.join(", ")
+          end
+
+          expectation_result
+        end
+
+        # Uses an LLM judge to compare two pieces of content and determine which better meets specified criteria.
+        #
+        # This method leverages the Comparative LLM judge to perform A/B testing between two pieces
+        # of content. Content placement is randomized to avoid position bias, and the judge determines
+        # which content better satisfies the comparison criteria.
+        #
+        # @param content_to_judge [String] The primary content being evaluated (will be randomly assigned to position A or B)
+        # @param over [String] The comparison content to evaluate against (will be randomly assigned to position A or B)
+        # @param criteria [String] The comparison criteria to use for evaluation
+        # @param allow_ties [Boolean] Whether the judge can declare a tie if both contents are equal (default: true)
+        # @param llm_judge_model_key [Symbol, nil] Optional specific LLM model to use for judging.
+        #   If nil, uses the configured default judge model or falls back to default LLM
+        # @param additional_context [String, nil] Optional additional context to help the judge
+        #
+        # @return [ExpectationResult] Result object containing pass/fail status and judge metadata
+        #
+        # @example Basic A/B comparison
+        #   expect_llm_judge_prefers(
+        #     new_response,
+        #     over: baseline_response,
+        #     criteria: "More comprehensive and accurate response"
+        #   )
+        #
+        # @example Model comparison with no ties allowed
+        #   expect_llm_judge_prefers(
+        #     claude_response,
+        #     over: gpt_response,
+        #     criteria: "Better follows the specific instructions given",
+        #     allow_ties: false
+        #   )
+        #
+        # @example With additional context
+        #   expect_llm_judge_prefers(
+        #     response_a,
+        #     over: response_b,
+        #     criteria: "More helpful for a beginner audience",
+        #     additional_context: "The user identified themselves as new to programming"
+        #   )
+        #
+        # @note The expectation passes if the judge correctly identifies the expected winner.
+        #   Due to randomization, content_to_judge may be assigned to either position A or B,
+        #   and the judge's choice is validated against the expected winner.
+        #
+        # @note The judge result includes metadata accessible via expectation_result.metadata:
+        #   - :winner - Which content won ("A", "B", or "tie")
+        #   - :reasoning - Detailed explanation of the choice
+        #   - :confidence - Confidence score (0.0-1.0)
+        def expect_llm_judge_prefers(content_to_judge, over:, criteria:, allow_ties: true, llm_judge_model_key: nil, additional_context: nil,
+          result_metadata: {})
+          judge_task = LlmJudges::Comparative.run(
+            content_to_judge: content_to_judge,
+            over_content: over,
+            comparison_criteria: criteria,
+            allow_ties: allow_ties,
+            llm_model_key: llm_judge_model_key,
+            additional_context: additional_context
+          )
+
+          if output.respond_to?(:puts)
+            output.puts "    Winner: #{judge_task.winner}"
+            output.puts "    #{judge_task.judgment_reasoning}" if Raif.config.evals_verbose_output
+          end
+
+          judge_metadata = {
+            winner: judge_task.winner,
+            reasoning: judge_task.judgment_reasoning,
+            confidence: judge_task.judgment_confidence,
+          }.compact
+
+          # Merge user metadata with judge metadata
+          combined_metadata = result_metadata.merge(judge_metadata)
+
+          expectation_result = expect "LLM judge prefers A over B: #{criteria}", result_metadata: combined_metadata do
+            judge_task.completed? && judge_task.correct_expected_winner?
+          end
+
+          if expectation_result && judge_task.errors.any?
+            expectation_result.error_message = judge_task.errors.full_messages.join(", ")
+          end
+
+          expectation_result
+        end
+
+      end
+    end
+  end
+end

data/lib/raif/evals/expectation_result.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    class ExpectationResult
+      attr_reader :description, :status, :error
+      attr_accessor :metadata, :error_message
+
+      def initialize(description:, status:, error: nil, error_message: nil, metadata: nil)
+        @description = description
+        @status = status
+        @error = error
+        @error_message = error_message
+        @metadata = metadata
+      end
+
+      def passed?
+        @status == :passed
+      end
+
+      def failed?
+        @status == :failed
+      end
+
+      def error?
+        @status == :error
+      end
+
+      def to_h
+        {
+          description: description,
+          status: status,
+          error: error_message.presence || error&.message,
+          metadata: metadata
+        }.compact
+      end
+    end
+  end
+end

data/lib/raif/evals/llm_judge.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Raif
+  module Evals
+    class LlmJudge < Raif::Task
+      # Set default temperature for consistent judging
+      llm_temperature 0.0
+
+      # Default to JSON response format for structured output
+      llm_response_format :json
+
+      task_run_arg :content_to_judge # the content to judge
+      task_run_arg :additional_context # additional context to be provided to the judge
+
+      def default_llm_model_key
+        Raif.config.evals_default_llm_judge_model_key || super
+      end
+
+      def judgment_reasoning
+        parsed_response["reasoning"] if completed?
+      end
+
+      def judgment_confidence
+        parsed_response["confidence"] if completed?
+      end
+
+      def low_confidence?
+        judgment_confidence && judgment_confidence < 0.5
+      end
+    end
+  end
+end