raif 1.2.1 → 1.3.0

data/lib/generators/raif/agent/templates/agent.rb.tt

@@ -1,28 +1,24 @@
-# frozen_string_literal: true
+<% raif_module_namespacing(["Agents"]) do -%>
+  class <%= class_name.demodulize %> < Raif::ApplicationAgent
+    # If you want to always include a certain set of model tools with this agent type,
+    # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
+    # def populate_default_model_tools
+    #   self.available_model_tools = [
+    #     Raif::ModelTools::WikipediaSearch,
+    #     Raif::ModelTools::FetchUrl
+    #   ]
+    # end
 
-module Raif
-  module Agents
-    class <%= class_name %> < Raif::ApplicationAgent
-      # If you want to always include a certain set of model tools with this agent type,
-      # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
-      # def populate_default_model_tools
-      #   self.available_model_tools ||= [
-      #     Raif::ModelTools::WikipediaSearchTool,
-      #     Raif::ModelTools::FetchUrlTool
-      #   ]
-      # end
-
-      # Enter your agent's system prompt here. Alternatively, you can change your agent's superclass
-      # to an existing agent types (like Raif::Agents::ReActAgent) to utilize an existing system prompt.
-      def build_system_prompt
-        # TODO: Implement your system prompt here
-      end
+    # Enter your agent's system prompt here. Alternatively, you can change your agent's superclass
+    # to an existing agent types (like Raif::Agents::ReActAgent) to utilize an existing system prompt.
+    def build_system_prompt
+      # TODO: Implement your system prompt here
+    end
 
-      # Each iteration of the agent loop will generate a new Raif::ModelCompletion record and
-      # then call this method with it as an argument.
-      def process_iteration_model_completion(model_completion)
-        # TODO: Implement your iteration processing here
-      end
+    # Each iteration of the agent loop will generate a new Raif::ModelCompletion record and
+    # then call this method with it as an argument.
+    def process_iteration_model_completion(model_completion)
+      # TODO: Implement your iteration processing here
     end
   end
-end
+<% end -%>

data/lib/generators/raif/agent/templates/agent_eval_set.rb.tt

@@ -0,0 +1,48 @@
+<% raif_module_namespacing(["Evals", "Agents"]) do -%>
+  class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
+    # Run this eval set with:
+    # bundle exec raif evals ./<%= eval_set_file_path %>
+
+    # Setup method runs before each eval
+    setup do
+      # Common setup code
+      # @user = User.create!(email: "test@example.com")
+    end
+
+    # Teardown runs after each eval
+    teardown do
+      # Cleanup code
+    end
+
+    eval "<%= class_name %> completes task successfully" do
+      # agent = Raif::Agents::<%= class_name %>.create!(
+      #   creator: @user,
+      #   task: "Your specific task here",
+      #   available_model_tools: []  # Add your tools here if needed
+      # )
+
+      # agent.run!
+
+      # expect "agent completes successfully" do
+      #   agent.completed?
+      # end
+
+      # expect "produces expected output" do
+      #   agent.final_answer.include?("expected content")
+      # end
+    end
+
+    eval "<%= class_name %> uses tools correctly" do
+      # agent = Raif::Agents::<%= class_name %>.create!(
+      #   creator: @user,
+      #   task: "A task that requires tool usage",
+      #   available_model_tools: ["expected_tool_name"]
+      # )
+
+      # agent.run!
+
+      # expect_tool_invocation(agent, "expected_tool_name")
+    end
+
+  end
+<% end -%>

data/lib/generators/raif/agent/templates/application_agent.rb.tt

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module Raif
   class ApplicationAgent < Raif::Agent
     # Add any shared agent behavior here

data/lib/generators/raif/base_generator.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Raif
+  class BaseGenerator < Rails::Generators::NamedBase
+  private
+
+    def raif_module_namespacing(intermediate_modules = [], &block)
+      content = capture(&block).rstrip
+
+      modules_names = intermediate_modules + class_path.map(&:camelize)
+      modules_names.reverse.each do |module_name|
+        content = indent "module #{module_name}\n#{content}\nend", 2
+      end
+
+      concat("module Raif\n#{content}\nend\n")
+    end
+
+  end
+end

data/lib/generators/raif/conversation/conversation_generator.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
+require_relative "../base_generator"
+
 module Raif
   module Generators
-    class ConversationGenerator < Rails::Generators::NamedBase
+    class ConversationGenerator < BaseGenerator
       source_root File.expand_path("templates", __dir__)
 
       desc "Creates a new conversation type in the app/models/raif/conversations directory"
@@ -12,19 +14,30 @@ module Raif
         default: "text",
         desc: "Response format for the task (text, html, or json)"
 
+      class_option :skip_eval_set,
+        type: :boolean,
+        default: false,
+        desc: "Skip generating the corresponding eval set"
+
       def create_application_conversation
         template "application_conversation.rb.tt",
           "app/models/raif/application_conversation.rb" unless File.exist?("app/models/raif/application_conversation.rb")
       end
 
       def create_conversation_file
-        template "conversation.rb.tt", File.join("app/models/raif/conversations", "#{file_name}.rb")
+        template "conversation.rb.tt", File.join("app/models/raif/conversations", class_path, "#{file_name}.rb")
       end
 
       def create_directory
         empty_directory "app/models/raif/conversations" unless File.directory?("app/models/raif/conversations")
       end
 
+      def create_eval_set
+        return if options[:skip_eval_set]
+
+        template "conversation_eval_set.rb.tt", eval_set_file_path
+      end
+
       def success_message
         say_status :success, "Conversation type created successfully", :green
         say "\nYou can now implement your conversation type in:"
@@ -32,6 +45,12 @@ module Raif
         say "\nDon't forget to add it to the config.conversation_types in your Raif configuration"
         say "For example: config.conversation_types += ['Raif::Conversations::#{class_name}']\n\n"
       end
+
+    private
+
+      def eval_set_file_path
+        File.join("raif_evals", "eval_sets", "conversations", class_path, "#{file_name}_eval_set.rb")
+      end
     end
   end
 end

data/lib/generators/raif/conversation/templates/application_conversation.rb.tt

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module Raif
   class ApplicationConversation < Raif::Conversation
     # Add any shared conversation behavior here

data/lib/generators/raif/conversation/templates/conversation.rb.tt

@@ -1,39 +1,35 @@
-# frozen_string_literal: true
+<% raif_module_namespacing(["Conversations"]) do -%>
+  class <%= class_name.demodulize %> < Raif::ApplicationConversation
+    # Set the response format for the conversation. Options are :html, :text, or :json.
+    # If you set this to something other than :text, make sure to include instructions to the model in your system prompt
+    llm_response_format :<%= options[:response_format] %>
 
-module Raif
-  module Conversations
-    class <%= class_name %> < Raif::ApplicationConversation
-      # Set the response format for the task. Options are :html, :text, or :json.
-      # If you set this to something other than :text, make sure to include instructions to the model in your system prompt
-      llm_response_format :<%= options[:response_format] %>
+    # If you want to always include a certain set of model tools with this conversation type,
+    # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
+    # before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
 
-      # If you want to always include a certain set of model tools with this conversation type,
-      # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
-      # before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
+    # Override the methods below to customize the system prompt for this conversation type.
+    # def system_prompt_intro
+    #   Raif.config.conversation_system_prompt_intro
+    # end
 
-      # Override the methods below to customize the system prompt for this conversation type.
-      # def system_prompt_intro
-      #   Raif.config.conversation_system_prompt_intro
-      # end
+    # def build_system_prompt
+    #   <<~PROMPT
+    #     #{system_prompt_intro}
+    #     #{system_prompt_language_preference}
+    #   PROMPT
+    # end
 
-      # def build_system_prompt
-      #   <<~PROMPT
-      #     #{system_prompt_intro}
-      #     #{system_prompt_language_preference}
-      #   PROMPT
-      # end
+    # Override this method to set the initial message shown to the user.
+    # def initial_chat_message
+    #   I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
+    # end
 
-      # Override this method to set the initial message shown to the user.
-      # def initial_chat_message
-      #   I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
-      # end
-
-      # This method will be called when receing a model response to a Raif::ConversationEntry
-      # By default, it just passes the model response message through, but you can override
-      # for custom response message processing
-      # def process_model_response_message(message:, entry:)
-      #   message
-      # end
-    end
+    # This method will be called when receing a model response to a Raif::ConversationEntry
+    # By default, it just passes the model response message through, but you can override
+    # for custom response message processing
+    # def process_model_response_message(message:, entry:)
+    #   message
+    # end
   end
-end
+<% end -%>

data/lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt

@@ -0,0 +1,70 @@
+<% raif_module_namespacing(["Evals", "Conversations"]) do -%>
+  class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
+    # Run this eval set with:
+    # bundle exec raif evals ./<%= eval_set_file_path %>
+
+    # Setup method runs before each eval
+    setup do
+      # Common setup code
+      # @user = User.create!(email: "test@example.com")
+      # @conversation = Raif::Conversations::<%= class_name %>.create!(creator: @user)
+    end
+
+    # Teardown runs after each eval
+    teardown do
+      # Cleanup code
+    end
+
+    eval "<%= class_name %> responds appropriately to user greeting" do
+      # entry = @conversation.entries.create!(
+      #   user_message: "Hello, how are you?",
+      #   creator: @user
+      # )
+
+      # entry.process_entry!
+
+      # expect "generates a response" do
+      #   entry.model_response_message.present?
+      # end
+
+      # expect "response is friendly" do
+      #   entry.model_response_message.match?(/hello|hi|greetings/i)
+      # end
+    end
+
+    eval "<%= class_name %> maintains conversation context" do
+      # First message establishes context
+      # first_entry = @conversation.entries.create!(
+      #   user_message: "My name is Alice",
+      #   creator: @user
+      # )
+      # first_entry.process_entry!
+
+      # Second message references context
+      # second_entry = @conversation.entries.create!(
+      #   user_message: "What's my name?",
+      #   creator: @user
+      # )
+      # second_entry.process_entry!
+
+      # expect "remembers the user's name" do
+      #   second_entry.model_response_message.include?("Alice")
+      # end
+    end
+
+    eval "<%= class_name %> handles tool invocations correctly" do
+      # Test if your conversation uses tools
+      # @conversation.update!(available_model_tools: [ "Raif::ModelTools::FetchUrl" ])
+
+      # entry = @conversation.entries.create!(
+      #   user_message: "What can you tell me about the content of https://en.wikipedia.org/wiki/Moon",
+      #   creator: @user
+      # )
+
+      # entry.process_entry!
+
+      # expect_tool_invocation(entry, "Raif::ModelTools::FetchUrl", with: { url: "https://en.wikipedia.org/wiki/Moon" })
+    end
+
+  end
+<% end -%>

data/lib/generators/raif/eval_set/eval_set_generator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "../base_generator"
+
+module Raif
+  module Generators
+    class EvalSetGenerator < BaseGenerator
+      source_root File.expand_path("templates", __dir__)
+
+      def create_eval_set_file
+        template "eval_set.rb.tt", eval_set_file_path
+      end
+
+      def show_instructions
+        say "\nEval set created!"
+        say "To run this eval set: bundle exec raif evals ./#{eval_set_file_path}"
+        say "To run all eval sets: bundle exec raif evals"
+        say ""
+      end
+
+    private
+
+      def eval_set_file_path
+        File.join("raif_evals", "eval_sets", class_path, "#{file_name}_eval_set.rb")
+      end
+    end
+  end
+end

data/lib/generators/raif/eval_set/templates/eval_set.rb.tt

@@ -0,0 +1,21 @@
+<% raif_module_namespacing(["Evals"]) do -%>
+  class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
+    # Run this eval set with:
+    # bundle exec raif evals ./<%= eval_set_file_path %>
+
+    # Setup method runs before each eval
+    setup do
+      # Common setup code
+    end
+
+    # Teardown runs after each eval
+    teardown do
+      # Cleanup code
+    end
+
+    eval "description of your eval" do
+      # Your eval code here
+      # expect_tool_invocation, expect_no_tool_invocation, expect, etc.
+    end
+  end
+<% end -%>

data/lib/generators/raif/evals/setup/setup_generator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module Raif
+  module Generators
+    module Evals
+      class SetupGenerator < Rails::Generators::Base
+        source_root File.expand_path("templates", __dir__)
+
+        def create_directories
+          empty_directory "raif_evals"
+          empty_directory "raif_evals/eval_sets"
+          empty_directory "raif_evals/files"
+          empty_directory "raif_evals/results"
+        end
+
+        def create_setup_file
+          create_file "raif_evals/setup.rb", <<~EOS
+            #
+            # This file is loaded at the start of a run of your evals.
+            #
+            # Add any setup code that should run before your evals.
+            #
+          EOS
+        end
+
+        def create_gitignore
+          create_file "raif_evals/results/.gitignore", <<~EOS
+            *
+            !.gitignore
+          EOS
+        end
+
+        def show_instructions
+          say "\nRaif evals setup complete!", :green
+          say "You can create evals with: rails g raif:eval_set ExampleName"
+          say ""
+          say "Run evals with:"
+          say "  bundle exec raif evals                                           # Run all evals"
+          say "  bundle exec raif evals ./raif_evals/eval_sets/my_eval_set.rb     # Run specific eval set"
+          say ""
+        end
+      end
+    end
+  end
+end

data/lib/generators/raif/install/install_generator.rb

@@ -24,8 +24,23 @@ module Raif
       end
 
       def add_engine_route
+        routes_file = "config/routes.rb"
+
+        if File.exist?(routes_file)
+          routes_content = File.read(routes_file)
+          if routes_content.include?("mount Raif::Engine")
+            say "Raif is already mounted in #{routes_file}, skipping route", :yellow
+            return
+          end
+        end
+
         route 'mount Raif::Engine => "/raif"'
       end
+
+      def setup_evals
+        say "\nSetting up Raif evals...", :green
+        generate "raif:evals:setup"
+      end
     end
   end
 end

data/lib/generators/raif/install/templates/initializer.rb

@@ -28,11 +28,11 @@ Raif.configure do |config|
   # Whether Titan embedding models are enabled. Defaults to false
   # config.bedrock_embedding_models_enabled = false
 
-  # Your OpenRouter API key. Defaults to ENV["OPENROUTER_API_KEY"]
-  # config.open_router_api_key = ENV["OPENROUTER_API_KEY"]
+  # Your OpenRouter API key. Defaults to ENV["OPEN_ROUTER_API_KEY"]
+  # config.open_router_api_key = ENV["OPEN_ROUTER_API_KEY"]
 
   # Whether OpenRouter models are enabled.
-  # config.open_router_models_enabled = ENV["OPENROUTER_API_KEY"].present?
+  # config.open_router_models_enabled = ENV["OPEN_ROUTER_API_KEY"].present?
 
   # The app name to include in OpenRouter API requests headers. Optional.
   # config.open_router_app_name = "My App"
@@ -98,6 +98,9 @@ Raif.configure do |config|
   # Or you can use a lambda to return a dynamic system prompt intro:
   # config.task_system_prompt_intro = ->(task){ "You are a helpful assistant. Today's date is #{Date.today.strftime('%B %d, %Y')}." }
 
+  # Whether the creator association is optional for Raif::Task. Defaults to true.
+  # config.task_creator_optional = true
+
   # The system prompt intro for Raif::Conversation instances. Defaults to "You are a helpful assistant who is collaborating with a teammate."
   # config.conversation_system_prompt_intro = "You are a helpful assistant who is collaborating with a teammate."
   # Or you can use a lambda to return a dynamic system prompt intro:
@@ -134,4 +137,12 @@ Raif.configure do |config|
   # Whether LLM API requests are enabled. Defaults to true.
   # Use this to globally disable requests to LLM APIs.
   # config.llm_api_requests_enabled = true
+
+  # The default LLM model to use for LLM-as-judge evaluations.
+  # If not set, falls back to the default_llm_model_key.
+  # config.evals_default_llm_judge_model_key = ENV["RAIF_EVALS_DEFAULT_LLM_JUDGE_MODEL_KEY"].presence
+
+  # Whether to output verbose information during evaluation runs. Defaults to false.
+  # When true, provides more detailed output including individual test results.
+  # config.evals_verbose_output = false
 end

data/lib/generators/raif/model_tool/model_tool_generator.rb

@@ -1,14 +1,17 @@
 # frozen_string_literal: true
 
+require_relative "../base_generator"
+
 module Raif
   module Generators
-    class ModelToolGenerator < Rails::Generators::NamedBase
+    class ModelToolGenerator < BaseGenerator
       source_root File.expand_path("templates", __dir__)
 
       desc "Creates a new model tool for the LLM to invoke in app/models/raif/model_tools"
 
       def create_model_tool_file
-        template "model_tool.rb.tt", File.join("app/models/raif/model_tools", "#{file_name}.rb")
+        template "model_tool.rb.tt", File.join("app/models/raif/model_tools", class_path, "#{file_name}.rb")
+        template "model_tool_invocation_partial.html.erb.tt", File.join("app/views/raif/model_tool_invocations", class_path, "_#{file_name}.html.erb")
       end
 
       def success_message