raif 1.2.2 → 1.4.0

data/config/locales/en.yml

@@ -6,10 +6,7 @@ en:
         errors:
           available_model_tools:
             too_short: must have at least 1 tool in addition to the agent_final_answer tool
-      re_act_agent:
-        errors:
-          available_model_tools:
-            too_short: must have at least 1 tool
+        final_answer_warning: 'Warning: This is your final iteration. You must provide your final answer using the agent_final_answer tool.'
     common:
       send: Send
       sources: Sources
@@ -18,6 +15,14 @@ en:
       type_your_message: Type your message...
     conversation:
       initial_chat_message: Hello, how can I help you today?
+    conversations:
+      index:
+        no_conversations: No conversations found.
+        table:
+          actions: Actions
+          started: Started
+          view: View
+        title: Past Conversations
     embedding_model_names:
       bedrock_titan_embed_text_v2: AWS Bedrock Titan Text Embeddings v2
       open_ai_text_embedding_3_large: OpenAI Text Embedding 3 Large
@@ -51,6 +56,9 @@ en:
       anthropic_claude_3_5_sonnet: Anthropic Claude 3.5 Sonnet
       anthropic_claude_3_7_sonnet: Anthropic Claude 3.7 Sonnet
       anthropic_claude_3_opus: Anthropic Claude 3 Opus
+      anthropic_claude_4_1_opus: Anthropic Claude 4.1 Opus
+      anthropic_claude_4_5_haiku: Anthropic Claude 4.5 Haiku
+      anthropic_claude_4_5_sonnet: Anthropic Claude 4.5 Sonnet
       anthropic_claude_4_opus: Anthropic Claude 4 Opus
       anthropic_claude_4_sonnet: Anthropic Claude 4 Sonnet
       bedrock_amazon_nova_lite: Amazon Nova Lite (via AWS Bedrock)
@@ -60,14 +68,26 @@ en:
       bedrock_claude_3_5_sonnet: Anthropic Claude 3.5 Sonnet (via AWS Bedrock)
       bedrock_claude_3_7_sonnet: Anthropic Claude 3.7 Sonnet (via AWS Bedrock)
       bedrock_claude_3_opus: Anthropic Claude 3 Opus (via AWS Bedrock)
+      bedrock_claude_4_1_opus: Claude 4.1 Opus (via AWS Bedrock)
+      bedrock_claude_4_5_haiku: Anthropic Claude 4.5 Haiku (via AWS Bedrock)
+      bedrock_claude_4_5_sonnet: Anthropic Claude 4.5 Sonnet (via AWS Bedrock)
       bedrock_claude_4_opus: Anthropic Claude 4 Opus (via AWS Bedrock)
       bedrock_claude_4_sonnet: Anthropic Claude 4 Sonnet (via AWS Bedrock)
+      google_gemini_2_5_flash: Google Gemini 2.5 Flash
+      google_gemini_2_5_pro: Google Gemini 2.5 Pro
+      google_gemini_3_0_flash: Google Gemini 3.0 Flash
+      google_gemini_3_0_pro: Google Gemini 3.0 Pro
       open_ai_gpt_3_5_turbo: OpenAI GPT-3.5 Turbo
       open_ai_gpt_4_1: OpenAI GPT-4.1
       open_ai_gpt_4_1_mini: OpenAI GPT-4.1 Mini
       open_ai_gpt_4_1_nano: OpenAI GPT-4.1 Nano
       open_ai_gpt_4o: OpenAI GPT-4o
       open_ai_gpt_4o_mini: OpenAI GPT-4o Mini
+      open_ai_gpt_5: OpenAI GPT-5
+      open_ai_gpt_5_1: OpenAI GPT-5.1
+      open_ai_gpt_5_2: OpenAI GPT-5.2
+      open_ai_gpt_5_mini: OpenAI GPT-5 Mini
+      open_ai_gpt_5_nano: OpenAI GPT-5 Nano
       open_ai_o1: OpenAI o1
       open_ai_o1_mini: OpenAI o1 Mini
       open_ai_o3: OpenAI o3
@@ -79,6 +99,11 @@ en:
       open_ai_responses_gpt_4_1_nano: OpenAI GPT-4.1 Nano (Responses API)
       open_ai_responses_gpt_4o: OpenAI GPT-4o (Responses API)
       open_ai_responses_gpt_4o_mini: OpenAI GPT-4o Mini (Responses API)
+      open_ai_responses_gpt_5: OpenAI GPT-5 (Responses API)
+      open_ai_responses_gpt_5_1: OpenAI GPT-5.1 (Responses API)
+      open_ai_responses_gpt_5_2: OpenAI GPT-5.2 (Responses API)
+      open_ai_responses_gpt_5_mini: OpenAI GPT-5 Mini (Responses API)
+      open_ai_responses_gpt_5_nano: OpenAI GPT-5 Nano (Responses API)
       open_ai_responses_o1: OpenAI o1 (Responses API)
       open_ai_responses_o1_mini: OpenAI o1 Mini (Responses API)
       open_ai_responses_o1_pro: OpenAI o1 Pro (Responses API)
@@ -88,9 +113,21 @@ en:
       open_ai_responses_o4_mini: OpenAI o4 Mini (Responses API)
       open_router_claude_3_7_sonnet: Anthropic Claude 3.7 Sonnet (via OpenRouter)
       open_router_deepseek_chat_v3: DeepSeek Chat v3 (via OpenRouter)
+      open_router_deepseek_v3_1: DeepSeek v3.1 (via OpenRouter)
       open_router_gemini_2_0_flash: Google Gemini 2.0 Flash (via OpenRouter)
+      open_router_gemini_2_5_flash: Gemini 2.5 Flash (via OpenRouter)
+      open_router_gemini_2_5_pro: Gemini 2.5 Pro (via OpenRouter)
+      open_router_gemini_3_pro_preview: Gemini 3 Pro Preview (via OpenRouter)
+      open_router_grok_4: Grok 4 (via OpenRouter)
+      open_router_grok_4_1_fast: Grok 4.1 Fast (via OpenRouter)
+      open_router_kimi_k2_thinking: Kimi K2 Thinking (via OpenRouter)
       open_router_llama_3_1_8b_instruct: Meta Llama 3.1 8B Instruct (via OpenRouter)
       open_router_llama_3_3_70b_instruct: Meta Llama 3.3 70B Instruct (via OpenRouter)
       open_router_llama_4_maverick: Meta Llama 4 Maverick (via OpenRouter)
       open_router_llama_4_scout: Meta Llama 4 Scout (via OpenRouter)
+      open_router_minimax_m2: Minimax M2 (via OpenRouter)
+      open_router_mistral_large_3_2512: Mistral Large 3 (via OpenRouter)
+      open_router_mistral_small_3_2_24b: Mistral Small 3.2 24B (via OpenRouter)
+      open_router_open_ai_gpt_oss_120b: OpenAI GPT-OSS 120B (via OpenRouter)
+      open_router_open_ai_gpt_oss_20b: OpenAI GPT-OSS 20B (via OpenRouter)
       raif_test_llm: Raif Test LLM

data/config/routes.rb

@@ -17,6 +17,7 @@ Raif::Engine.routes.draw do
 
     namespace :stats do
       resources :tasks, only: [:index]
+      resources :model_tool_invocations, only: [:index]
     end
 
     resources :tasks, only: [:index, :show]
@@ -24,5 +25,6 @@ Raif::Engine.routes.draw do
     resources :model_completions, only: [:index, :show]
     resources :agents, only: [:index, :show]
     resources :model_tool_invocations, only: [:index, :show]
+    resource :config, only: [:show]
   end
 end

data/db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddTaskRunArgsToRaifTasks < ActiveRecord::Migration[7.1]
+  def change
+    json_column_type = if connection.adapter_name.downcase.include?("postgresql")
+      :jsonb
+    else
+      :json
+    end
+
+    add_column :raif_tasks, :task_run_args, json_column_type
+  end
+end

data/db/migrate/20250811171150_make_raif_task_creator_optional.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class MakeRaifTaskCreatorOptional < ActiveRecord::Migration[7.1]
+  def change
+    change_column_null :raif_tasks, :creator_id, true
+    change_column_null :raif_tasks, :creator_type, true
+  end
+end

data/db/migrate/20250904194456_add_generating_entry_response_to_raif_conversations.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddGeneratingEntryResponseToRaifConversations < ActiveRecord::Migration[7.1]
+  def change
+    add_column :raif_conversations, :generating_entry_response, :boolean, default: false, null: false
+  end
+end

data/db/migrate/20250911125234_add_source_to_raif_tasks.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddSourceToRaifTasks < ActiveRecord::Migration[7.1]
+  def change
+    add_reference :raif_tasks, :source, polymorphic: true, index: true
+  end
+end

data/db/migrate/20251020005853_add_source_to_raif_agents.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddSourceToRaifAgents < ActiveRecord::Migration[7.1]
+  def change
+    add_reference :raif_agents, :source, polymorphic: true, index: true
+  end
+end

data/db/migrate/20251020011346_rename_task_run_args_to_run_with.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class RenameTaskRunArgsToRunWith < ActiveRecord::Migration[7.1]
+  def change
+    rename_column :raif_tasks, :task_run_args, :run_with
+  end
+end

data/db/migrate/20251020011405_add_run_with_to_raif_agents.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddRunWithToRaifAgents < ActiveRecord::Migration[7.1]
+  def change
+    json_column_type = if connection.adapter_name.downcase.include?("postgresql")
+      :jsonb
+    else
+      :json
+    end
+
+    add_column :raif_agents, :run_with, json_column_type
+  end
+end

data/db/migrate/20251024160119_add_llm_messages_max_length_to_raif_conversations.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddLlmMessagesMaxLengthToRaifConversations < ActiveRecord::Migration[7.1]
+  def change
+    add_column :raif_conversations, :llm_messages_max_length, :integer
+
+    reversible do |dir|
+      dir.up do
+        # Set default value for existing conversations
+        execute "UPDATE raif_conversations SET llm_messages_max_length = 50 WHERE llm_messages_max_length IS NULL"
+      end
+    end
+  end
+end

data/db/migrate/20251124185033_add_provider_tool_call_id_to_raif_model_tool_invocations.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddProviderToolCallIdToRaifModelToolInvocations < ActiveRecord::Migration[7.2]
+  def change
+    add_column :raif_model_tool_invocations, :provider_tool_call_id, :string
+  end
+end

data/db/migrate/20251128202941_add_tool_choice_to_raif_model_completions.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddToolChoiceToRaifModelCompletions < ActiveRecord::Migration[7.2]
+  def change
+    add_column :raif_model_completions, :tool_choice, :string
+  end
+end

data/db/migrate/20260118144846_add_source_to_raif_conversations.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddSourceToRaifConversations < ActiveRecord::Migration[7.1]
+  def change
+    add_reference :raif_conversations, :source, polymorphic: true, index: true
+  end
+end

data/db/migrate/20260119000000_add_failure_tracking_to_raif_model_completions.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class AddFailureTrackingToRaifModelCompletions < ActiveRecord::Migration[7.1]
+  def change
+    add_column :raif_model_completions, :failed_at, :datetime
+    add_column :raif_model_completions, :failure_error, :string
+    add_column :raif_model_completions, :failure_reason, :text
+    add_index :raif_model_completions, :failed_at
+  end
+end

data/db/migrate/20260119000001_add_completed_at_to_raif_model_completions.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddCompletedAtToRaifModelCompletions < ActiveRecord::Migration[7.1]
+  def change
+    add_column :raif_model_completions, :completed_at, :datetime
+    add_index :raif_model_completions, :completed_at
+  end
+end

data/db/migrate/20260119000002_add_started_at_to_raif_model_completions.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddStartedAtToRaifModelCompletions < ActiveRecord::Migration[7.1]
+  def change
+    add_column :raif_model_completions, :started_at, :datetime
+    add_index :raif_model_completions, :started_at
+  end
+end

data/exe/raif

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../lib/raif/cli"
+
+# Run the CLI
+Raif::CLI::Runner.new(ARGV).run

data/lib/generators/raif/agent/agent_generator.rb

@@ -1,32 +1,47 @@
 # frozen_string_literal: true
 
+require_relative "../base_generator"
+
 module Raif
   module Generators
-    class AgentGenerator < Rails::Generators::NamedBase
+    class AgentGenerator < BaseGenerator
       source_root File.expand_path("templates", __dir__)
       desc "Creates a new Raif::Agent subclass in app/models/raif/agents"
 
+      class_option :skip_eval_set,
+        type: :boolean,
+        default: false,
+        desc: "Skip generating the corresponding eval set"
+
       def create_application_agent
         template "application_agent.rb.tt", "app/models/raif/application_agent.rb" unless File.exist?("app/models/raif/application_agent.rb")
       end
 
       def create_agent
-        template "agent.rb.tt", "app/models/raif/agents/#{file_name}.rb"
+        template "agent.rb.tt", File.join("app/models/raif/agents", class_path, "#{file_name}.rb")
       end
 
       def create_directory
         empty_directory "app/models/raif/agents" unless File.directory?("app/models/raif/agents")
       end
 
-    private
+      def create_eval_set
+        return if options[:skip_eval_set]
+
+        template "agent_eval_set.rb.tt", eval_set_file_path
+      end
 
-      def class_name
-        name.classify
+      def show_instructions
+        say "\nAgent created!"
+        say ""
       end
 
-      def file_name
-        name.underscore
+    private
+
+      def eval_set_file_path
+        File.join("raif_evals", "eval_sets", "agents", class_path, "#{file_name}_eval_set.rb")
       end
+
     end
   end
 end

data/lib/generators/raif/agent/templates/agent.rb.tt

@@ -1,28 +1,24 @@
-# frozen_string_literal: true
+<% raif_module_namespacing(["Agents"]) do -%>
+  class <%= class_name.demodulize %> < Raif::ApplicationAgent
+    # If you want to always include a certain set of model tools with this agent type,
+    # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
+    # def populate_default_model_tools
+    #   self.available_model_tools = [
+    #     Raif::ModelTools::WikipediaSearch,
+    #     Raif::ModelTools::FetchUrl
+    #   ]
+    # end
 
-module Raif
-  module Agents
-    class <%= class_name %> < Raif::ApplicationAgent
-      # If you want to always include a certain set of model tools with this agent type,
-      # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
-      # def populate_default_model_tools
-      #   self.available_model_tools ||= [
-      #     Raif::ModelTools::WikipediaSearchTool,
-      #     Raif::ModelTools::FetchUrlTool
-      #   ]
-      # end
-
-      # Enter your agent's system prompt here. Alternatively, you can change your agent's superclass
-      # to an existing agent types (like Raif::Agents::ReActAgent) to utilize an existing system prompt.
-      def build_system_prompt
-        # TODO: Implement your system prompt here
-      end
+    # Enter your agent's system prompt here. Alternatively, you can change your agent's superclass
+    # to an existing agent types (like Raif::Agents::NativeToolCallingAgent) to utilize an existing system prompt.
+    def build_system_prompt
+      # TODO: Implement your system prompt here
+    end
 
-      # Each iteration of the agent loop will generate a new Raif::ModelCompletion record and
-      # then call this method with it as an argument.
-      def process_iteration_model_completion(model_completion)
-        # TODO: Implement your iteration processing here
-      end
+    # Each iteration of the agent loop will generate a new Raif::ModelCompletion record and
+    # then call this method with it as an argument.
+    def process_iteration_model_completion(model_completion)
+      # TODO: Implement your iteration processing here
     end
   end
-end
+<% end -%>

data/lib/generators/raif/agent/templates/agent_eval_set.rb.tt

@@ -0,0 +1,48 @@
+<% raif_module_namespacing(["Evals", "Agents"]) do -%>
+  class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
+    # Run this eval set with:
+    # bundle exec raif evals ./<%= eval_set_file_path %>
+
+    # Setup method runs before each eval
+    setup do
+      # Common setup code
+      # @user = User.create!(email: "test@example.com")
+    end
+
+    # Teardown runs after each eval
+    teardown do
+      # Cleanup code
+    end
+
+    eval "<%= class_name %> completes task successfully" do
+      # agent = Raif::Agents::<%= class_name %>.create!(
+      #   creator: @user,
+      #   task: "Your specific task here",
+      #   available_model_tools: []  # Add your tools here if needed
+      # )
+
+      # agent.run!
+
+      # expect "agent completes successfully" do
+      #   agent.completed?
+      # end
+
+      # expect "produces expected output" do
+      #   agent.final_answer.include?("expected content")
+      # end
+    end
+
+    eval "<%= class_name %> uses tools correctly" do
+      # agent = Raif::Agents::<%= class_name %>.create!(
+      #   creator: @user,
+      #   task: "A task that requires tool usage",
+      #   available_model_tools: ["expected_tool_name"]
+      # )
+
+      # agent.run!
+
+      # expect_tool_invocation(agent, "expected_tool_name")
+    end
+
+  end
+<% end -%>

data/lib/generators/raif/agent/templates/application_agent.rb.tt

@@ -1,7 +1,5 @@
-# frozen_string_literal: true
-
 module Raif
-  class ApplicationAgent < Raif::Agent
+  class ApplicationAgent < Raif::Agents::NativeToolCallingAgent
     # Add any shared agent behavior here
   end
 end

data/lib/generators/raif/base_generator.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Raif
+  class BaseGenerator < Rails::Generators::NamedBase
+  private
+
+    def raif_module_namespacing(intermediate_modules = [], &block)
+      content = capture(&block).rstrip
+
+      modules_names = intermediate_modules + class_path.map(&:camelize)
+      modules_names.reverse.each do |module_name|
+        content = indent "module #{module_name}\n#{content}\nend", 2
+      end
+
+      concat("module Raif\n#{content}\nend\n")
+    end
+
+  end
+end

data/lib/generators/raif/conversation/conversation_generator.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
+require_relative "../base_generator"
+
 module Raif
   module Generators
-    class ConversationGenerator < Rails::Generators::NamedBase
+    class ConversationGenerator < BaseGenerator
       source_root File.expand_path("templates", __dir__)
 
       desc "Creates a new conversation type in the app/models/raif/conversations directory"
@@ -12,19 +14,30 @@ module Raif
         default: "text",
         desc: "Response format for the task (text, html, or json)"
 
+      class_option :skip_eval_set,
+        type: :boolean,
+        default: false,
+        desc: "Skip generating the corresponding eval set"
+
       def create_application_conversation
         template "application_conversation.rb.tt",
           "app/models/raif/application_conversation.rb" unless File.exist?("app/models/raif/application_conversation.rb")
       end
 
       def create_conversation_file
-        template "conversation.rb.tt", File.join("app/models/raif/conversations", "#{file_name}.rb")
+        template "conversation.rb.tt", File.join("app/models/raif/conversations", class_path, "#{file_name}.rb")
       end
 
       def create_directory
         empty_directory "app/models/raif/conversations" unless File.directory?("app/models/raif/conversations")
       end
 
+      def create_eval_set
+        return if options[:skip_eval_set]
+
+        template "conversation_eval_set.rb.tt", eval_set_file_path
+      end
+
       def success_message
         say_status :success, "Conversation type created successfully", :green
         say "\nYou can now implement your conversation type in:"
@@ -32,6 +45,12 @@ module Raif
         say "\nDon't forget to add it to the config.conversation_types in your Raif configuration"
         say "For example: config.conversation_types += ['Raif::Conversations::#{class_name}']\n\n"
       end
+
+    private
+
+      def eval_set_file_path
+        File.join("raif_evals", "eval_sets", "conversations", class_path, "#{file_name}_eval_set.rb")
+      end
     end
   end
 end

data/lib/generators/raif/conversation/templates/application_conversation.rb.tt

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module Raif
   class ApplicationConversation < Raif::Conversation
     # Add any shared conversation behavior here

data/lib/generators/raif/conversation/templates/conversation.rb.tt

@@ -1,39 +1,41 @@
-# frozen_string_literal: true
+<% raif_module_namespacing(["Conversations"]) do -%>
+  class <%= class_name.demodulize %> < Raif::ApplicationConversation
+    # Set the response format for the conversation. Options are :html, :text, or :json.
+    # If you set this to something other than :text, make sure to include instructions to the model in your system prompt
+    llm_response_format :<%= options[:response_format] %>
 
-module Raif
-  module Conversations
-    class <%= class_name %> < Raif::ApplicationConversation
-      # Set the response format for the task. Options are :html, :text, or :json.
-      # If you set this to something other than :text, make sure to include instructions to the model in your system prompt
-      llm_response_format :<%= options[:response_format] %>
+    # If you want to always include a certain set of model tools with this conversation type,
+    # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
+    # before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
 
-      # If you want to always include a certain set of model tools with this conversation type,
-      # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
-      # before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
+    # Use this callback to run custom logic before prompting the model for an entry response.
+    # The block receives the entry as an argument, and `self` is the conversation instance.
+    # before_prompt_model_for_entry_response do |entry|
+    #   # Custom logic here
+    # end
 
-      # Override the methods below to customize the system prompt for this conversation type.
-      # def system_prompt_intro
-      #   Raif.config.conversation_system_prompt_intro
-      # end
+    # Override the methods below to customize the system prompt for this conversation type.
+    # def system_prompt_intro
+    #   Raif.config.conversation_system_prompt_intro
+    # end
 
-      # def build_system_prompt
-      #   <<~PROMPT
-      #     #{system_prompt_intro}
-      #     #{system_prompt_language_preference}
-      #   PROMPT
-      # end
+    # def build_system_prompt
+    #   <<~PROMPT
+    #     #{system_prompt_intro}
+    #     #{system_prompt_language_preference}
+    #   PROMPT
+    # end
 
-      # Override this method to set the initial message shown to the user.
-      # def initial_chat_message
-      #   I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
-      # end
+    # Override this method to set the initial message shown to the user.
+    # def initial_chat_message
+    #   I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
+    # end
 
-      # This method will be called when receing a model response to a Raif::ConversationEntry
-      # By default, it just passes the model response message through, but you can override
-      # for custom response message processing
-      # def process_model_response_message(message:, entry:)
-      #   message
-      # end
-    end
+    # This method will be called when receing a model response to a Raif::ConversationEntry
+    # By default, it just passes the model response message through, but you can override
+    # for custom response message processing
+    # def process_model_response_message(message:, entry:)
+    #   message
+    # end
   end
-end
+<% end -%>

data/lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt

@@ -0,0 +1,70 @@
+<% raif_module_namespacing(["Evals", "Conversations"]) do -%>
+  class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
+    # Run this eval set with:
+    # bundle exec raif evals ./<%= eval_set_file_path %>
+
+    # Setup method runs before each eval
+    setup do
+      # Common setup code
+      # @user = User.create!(email: "test@example.com")
+      # @conversation = Raif::Conversations::<%= class_name %>.create!(creator: @user)
+    end
+
+    # Teardown runs after each eval
+    teardown do
+      # Cleanup code
+    end
+
+    eval "<%= class_name %> responds appropriately to user greeting" do
+      # entry = @conversation.entries.create!(
+      #   user_message: "Hello, how are you?",
+      #   creator: @user
+      # )
+
+      # entry.process_entry!
+
+      # expect "generates a response" do
+      #   entry.model_response_message.present?
+      # end
+
+      # expect "response is friendly" do
+      #   entry.model_response_message.match?(/hello|hi|greetings/i)
+      # end
+    end
+
+    eval "<%= class_name %> maintains conversation context" do
+      # First message establishes context
+      # first_entry = @conversation.entries.create!(
+      #   user_message: "My name is Alice",
+      #   creator: @user
+      # )
+      # first_entry.process_entry!
+
+      # Second message references context
+      # second_entry = @conversation.entries.create!(
+      #   user_message: "What's my name?",
+      #   creator: @user
+      # )
+      # second_entry.process_entry!
+
+      # expect "remembers the user's name" do
+      #   second_entry.model_response_message.include?("Alice")
+      # end
+    end
+
+    eval "<%= class_name %> handles tool invocations correctly" do
+      # Test if your conversation uses tools
+      # @conversation.update!(available_model_tools: [ "Raif::ModelTools::FetchUrl" ])
+
+      # entry = @conversation.entries.create!(
+      #   user_message: "What can you tell me about the content of https://en.wikipedia.org/wiki/Moon",
+      #   creator: @user
+      # )
+
+      # entry.process_entry!
+
+      # expect_tool_invocation(entry, "Raif::ModelTools::FetchUrl", with: { url: "https://en.wikipedia.org/wiki/Moon" })
+    end
+
+  end
+<% end -%>