raif 1.2.2 → 1.4.0

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification
 name: raif
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.4.0
 platform: ruby
 authors:
 - Ben Roesch
 - Brian Leslie
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
@@ -84,16 +84,16 @@ dependencies:
   name: pagy
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -142,7 +142,8 @@ description: Raif (Ruby AI Framework) is a Rails engine that helps you add AI-po
 email:
 - ben@cultivatelabs.com
 - brian@cultivatelabs.com
-executables: []
+executables:
+- raif
 extensions: []
 extra_rdoc_files: []
 files:
@@ -159,13 +160,16 @@ files:
 - app/assets/stylesheets/raif.scss
 - app/assets/stylesheets/raif/admin/conversation.scss
 - app/assets/stylesheets/raif/admin/stats.scss
+- app/assets/stylesheets/raif/conversations.scss
 - app/assets/stylesheets/raif/loader.scss
 - app/assets/stylesheets/raif_admin.scss
 - app/controllers/raif/admin/agents_controller.rb
 - app/controllers/raif/admin/application_controller.rb
+- app/controllers/raif/admin/configs_controller.rb
 - app/controllers/raif/admin/conversations_controller.rb
 - app/controllers/raif/admin/model_completions_controller.rb
 - app/controllers/raif/admin/model_tool_invocations_controller.rb
+- app/controllers/raif/admin/stats/model_tool_invocations_controller.rb
 - app/controllers/raif/admin/stats/tasks_controller.rb
 - app/controllers/raif/admin/stats_controller.rb
 - app/controllers/raif/admin/tasks_controller.rb
@@ -176,11 +180,11 @@ files:
 - app/helpers/raif/shared/conversations_helper.rb
 - app/jobs/raif/application_job.rb
 - app/jobs/raif/conversation_entry_job.rb
+- app/models/raif/admin/task_stat.rb
 - app/models/raif/agent.rb
 - app/models/raif/agents/native_tool_calling_agent.rb
-- app/models/raif/agents/re_act_agent.rb
-- app/models/raif/agents/re_act_step.rb
 - app/models/raif/application_record.rb
+- app/models/raif/concerns/agent_inference_stats.rb
 - app/models/raif/concerns/boolean_timestamp.rb
 - app/models/raif/concerns/has_available_model_tools.rb
 - app/models/raif/concerns/has_llm.rb
@@ -190,15 +194,23 @@ files:
 - app/models/raif/concerns/llm_response_parsing.rb
 - app/models/raif/concerns/llm_temperature.rb
 - app/models/raif/concerns/llms/anthropic/message_formatting.rb
+- app/models/raif/concerns/llms/anthropic/response_tool_calls.rb
 - app/models/raif/concerns/llms/anthropic/tool_formatting.rb
 - app/models/raif/concerns/llms/bedrock/message_formatting.rb
+- app/models/raif/concerns/llms/bedrock/response_tool_calls.rb
 - app/models/raif/concerns/llms/bedrock/tool_formatting.rb
+- app/models/raif/concerns/llms/google/message_formatting.rb
+- app/models/raif/concerns/llms/google/response_tool_calls.rb
+- app/models/raif/concerns/llms/google/tool_formatting.rb
 - app/models/raif/concerns/llms/message_formatting.rb
 - app/models/raif/concerns/llms/open_ai/json_schema_validation.rb
 - app/models/raif/concerns/llms/open_ai_completions/message_formatting.rb
+- app/models/raif/concerns/llms/open_ai_completions/response_tool_calls.rb
 - app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb
 - app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb
+- app/models/raif/concerns/llms/open_ai_responses/response_tool_calls.rb
 - app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb
+- app/models/raif/concerns/run_with.rb
 - app/models/raif/conversation.rb
 - app/models/raif/conversation_entry.rb
 - app/models/raif/embedding_model.rb
@@ -207,6 +219,7 @@ files:
 - app/models/raif/llm.rb
 - app/models/raif/llms/anthropic.rb
 - app/models/raif/llms/bedrock.rb
+- app/models/raif/llms/google.rb
 - app/models/raif/llms/open_ai_base.rb
 - app/models/raif/llms/open_ai_completions.rb
 - app/models/raif/llms/open_ai_responses.rb
@@ -225,6 +238,7 @@ files:
 - app/models/raif/model_tools/wikipedia_search.rb
 - app/models/raif/streaming_responses/anthropic.rb
 - app/models/raif/streaming_responses/bedrock.rb
+- app/models/raif/streaming_responses/google.rb
 - app/models/raif/streaming_responses/open_ai_completions.rb
 - app/models/raif/streaming_responses/open_ai_responses.rb
 - app/models/raif/task.rb
@@ -234,6 +248,7 @@ files:
 - app/views/raif/admin/agents/_conversation_message.html.erb
 - app/views/raif/admin/agents/index.html.erb
 - app/views/raif/admin/agents/show.html.erb
+- app/views/raif/admin/configs/show.html.erb
 - app/views/raif/admin/conversations/_conversation.html.erb
 - app/views/raif/admin/conversations/_conversation_entry.html.erb
 - app/views/raif/admin/conversations/index.html.erb
@@ -244,7 +259,11 @@ files:
 - app/views/raif/admin/model_tool_invocations/_model_tool_invocation.html.erb
 - app/views/raif/admin/model_tool_invocations/index.html.erb
 - app/views/raif/admin/model_tool_invocations/show.html.erb
+- app/views/raif/admin/model_tools/_list.html.erb
+- app/views/raif/admin/model_tools/_model_tool.html.erb
+- app/views/raif/admin/stats/_stats_tile.html.erb
 - app/views/raif/admin/stats/index.html.erb
+- app/views/raif/admin/stats/model_tool_invocations/index.html.erb
 - app/views/raif/admin/stats/tasks/index.html.erb
 - app/views/raif/admin/tasks/_task.html.erb
 - app/views/raif/admin/tasks/index.html.erb
@@ -260,7 +279,11 @@ files:
 - app/views/raif/conversation_entries/create.turbo_stream.erb
 - app/views/raif/conversation_entries/new.turbo_stream.erb
 - app/views/raif/conversations/_available_user_tools.html.erb
+- app/views/raif/conversations/_conversation.html.erb
+- app/views/raif/conversations/_entry_processed.turbo_stream.erb
 - app/views/raif/conversations/_full_conversation.html.erb
+- app/views/raif/conversations/_initial_chat_message.html.erb
+- app/views/raif/conversations/index.html.erb
 - app/views/raif/conversations/show.html.erb
 - config/i18n-tasks.yml
 - config/importmap.rb
@@ -277,12 +300,33 @@ files:
 - db/migrate/20250527213016_add_response_id_and_response_array_to_model_completions.rb
 - db/migrate/20250603140622_add_citations_to_raif_model_completions.rb
 - db/migrate/20250603202013_add_stream_response_to_raif_model_completions.rb
+- db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb
+- db/migrate/20250811171150_make_raif_task_creator_optional.rb
+- db/migrate/20250904194456_add_generating_entry_response_to_raif_conversations.rb
+- db/migrate/20250911125234_add_source_to_raif_tasks.rb
+- db/migrate/20251020005853_add_source_to_raif_agents.rb
+- db/migrate/20251020011346_rename_task_run_args_to_run_with.rb
+- db/migrate/20251020011405_add_run_with_to_raif_agents.rb
+- db/migrate/20251024160119_add_llm_messages_max_length_to_raif_conversations.rb
+- db/migrate/20251124185033_add_provider_tool_call_id_to_raif_model_tool_invocations.rb
+- db/migrate/20251128202941_add_tool_choice_to_raif_model_completions.rb
+- db/migrate/20260118144846_add_source_to_raif_conversations.rb
+- db/migrate/20260119000000_add_failure_tracking_to_raif_model_completions.rb
+- db/migrate/20260119000001_add_completed_at_to_raif_model_completions.rb
+- db/migrate/20260119000002_add_started_at_to_raif_model_completions.rb
+- exe/raif
 - lib/generators/raif/agent/agent_generator.rb
 - lib/generators/raif/agent/templates/agent.rb.tt
+- lib/generators/raif/agent/templates/agent_eval_set.rb.tt
 - lib/generators/raif/agent/templates/application_agent.rb.tt
+- lib/generators/raif/base_generator.rb
 - lib/generators/raif/conversation/conversation_generator.rb
 - lib/generators/raif/conversation/templates/application_conversation.rb.tt
 - lib/generators/raif/conversation/templates/conversation.rb.tt
+- lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt
+- lib/generators/raif/eval_set/eval_set_generator.rb
+- lib/generators/raif/eval_set/templates/eval_set.rb.tt
+- lib/generators/raif/evals/setup/setup_generator.rb
 - lib/generators/raif/install/install_generator.rb
 - lib/generators/raif/install/templates/initializer.rb
 - lib/generators/raif/model_tool/model_tool_generator.rb
@@ -291,13 +335,19 @@ files:
 - lib/generators/raif/task/task_generator.rb
 - lib/generators/raif/task/templates/application_task.rb.tt
 - lib/generators/raif/task/templates/task.rb.tt
+- lib/generators/raif/task/templates/task_eval_set.rb.tt
 - lib/generators/raif/views_generator.rb
 - lib/raif.rb
+- lib/raif/cli.rb
+- lib/raif/cli/base.rb
+- lib/raif/cli/evals.rb
+- lib/raif/cli/evals_setup.rb
 - lib/raif/configuration.rb
 - lib/raif/embedding_model_registry.rb
 - lib/raif/engine.rb
 - lib/raif/errors.rb
 - lib/raif/errors/action_not_authorized_error.rb
+- lib/raif/errors/instance_dependent_schema_error.rb
 - lib/raif/errors/invalid_config_error.rb
 - lib/raif/errors/invalid_conversation_type_error.rb
 - lib/raif/errors/invalid_model_file_input_error.rb
@@ -306,18 +356,35 @@ files:
 - lib/raif/errors/open_ai/json_schema_error.rb
 - lib/raif/errors/streaming_error.rb
 - lib/raif/errors/unsupported_feature_error.rb
+- lib/raif/evals.rb
+- lib/raif/evals/eval.rb
+- lib/raif/evals/eval_set.rb
+- lib/raif/evals/eval_sets/expectations.rb
+- lib/raif/evals/eval_sets/llm_judge_expectations.rb
+- lib/raif/evals/expectation_result.rb
+- lib/raif/evals/llm_judge.rb
+- lib/raif/evals/llm_judges/binary.rb
+- lib/raif/evals/llm_judges/comparative.rb
+- lib/raif/evals/llm_judges/scored.rb
+- lib/raif/evals/llm_judges/summarization.rb
+- lib/raif/evals/run.rb
+- lib/raif/evals/scoring_rubric.rb
 - lib/raif/json_schema_builder.rb
 - lib/raif/languages.rb
 - lib/raif/llm_registry.rb
+- lib/raif/messages.rb
 - lib/raif/migration_checker.rb
 - lib/raif/rspec.rb
 - lib/raif/utils.rb
+- lib/raif/utils/colors.rb
 - lib/raif/utils/html_fragment_processor.rb
 - lib/raif/utils/html_to_markdown_converter.rb
 - lib/raif/utils/readable_content_extractor.rb
 - lib/raif/version.rb
+- lib/tasks/annotate_rb.rake
 - lib/tasks/raif_tasks.rake
 - spec/support/complex_test_tool.rb
+- spec/support/current_temperature_test_tool.rb
 - spec/support/rspec_helpers.rb
 - spec/support/test_conversation.rb
 - spec/support/test_embedding_model.rb
@@ -346,7 +413,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Raif (Ruby AI Framework) is a Rails engine that helps you add AI-powered
   features to your Rails apps, such as tasks, conversations, and agents.

data/app/models/raif/agents/re_act_agent.rb

@@ -1,127 +0,0 @@
-# frozen_string_literal: true
-
-module Raif
-  module Agents
-    class ReActAgent < Raif::Agent
-      validates :available_model_tools, length: {
-        minimum: 1,
-        message: ->(_object, _data) {
-          I18n.t("raif.agents.re_act_agent.errors.available_model_tools.too_short")
-        }
-      }
-
-      def build_system_prompt
-        <<~PROMPT.strip
-          You are an intelligent assistant that follows the ReAct (Reasoning + Acting) framework to complete tasks step by step using tool calls.
-
-          # Available Tools
-          You have access to the following tools:
-          #{available_model_tools_map.values.map(&:description_for_llm).join("\n---\n")}
-          # Your Responses
-          Your responses should follow this structure & format:
-          <thought>Your step-by-step reasoning about what to do</thought>
-          <action>JSON object with "tool" and "arguments" keys</action>
-          <observation>Results from the tool, which will be provided to you</observation>
-          ... (repeat Thought/Action/Observation as needed until the task is complete)
-          <thought>Final reasoning based on all observations</thought>
-          <answer>Your final response to the user</answer>
-
-          # How to Use Tools
-          When you need to use a tool:
-          1. Identify which tool is appropriate for the task
-          2. Format your tool call using JSON with the required arguments and place it in the <action> tag
-          3. Here is an example: <action>{"tool": "tool_name", "arguments": {...}}</action>
-
-          # Guidelines
-          - Always think step by step
-          - Use tools when appropriate, but don't use tools for tasks you can handle directly
-          - Be concise in your reasoning but thorough in your analysis
-          - If a tool returns an error, try to understand why and adjust your approach
-          - If you're unsure about something, explain your uncertainty, but do not make things up
-          - After each thought, make sure to also include an <action> or <answer>
-          - Always provide a final answer that directly addresses the user's request
-
-          Remember: Your goal is to be helpful, accurate, and efficient in solving the user's request.#{system_prompt_language_preference}
-        PROMPT
-      end
-
-    private
-
-      def process_iteration_model_completion(model_completion)
-        agent_step = Raif::Agents::ReActStep.new(model_response_text: model_completion.raw_response)
-
-        # Add the thought to conversation history
-        if agent_step.thought
-          add_conversation_history_entry({ role: "assistant", content: "<thought>#{agent_step.thought}</thought>" })
-        end
-
-        # If there's an answer, we're done
-        if agent_step.answer
-          self.final_answer = agent_step.answer
-          add_conversation_history_entry({ role: "assistant", content: "<answer>#{agent_step.answer}</answer>" })
-          return
-        end
-
-        # If there's an action, execute it
-        process_action(agent_step.action) if agent_step.action
-      end
-
-      def process_action(action)
-        add_conversation_history_entry({ role: "assistant", content: "<action>#{action}</action>" })
-
-        # The action should always be a JSON object with "tool" and "arguments" keys
-        parsed_action = begin
-          JSON.parse(action)
-        rescue JSON::ParserError => e
-          add_conversation_history_entry({
-            role: "assistant",
-            content: "<observation>Error parsing action JSON: #{e.message}</observation>"
-          })
-
-          nil
-        end
-
-        return if parsed_action.blank?
-
-        unless parsed_action["tool"] && parsed_action["arguments"]
-          add_conversation_history_entry({
-            role: "assistant",
-            content: "<observation>Error: Invalid action specified. Please provide a valid action, formatted as a JSON object with 'tool' and 'arguments' keys.</observation>" # rubocop:disable Layout/LineLength
-          })
-          return
-        end
-
-        tool_name = parsed_action["tool"]
-        tool_arguments = parsed_action["arguments"]
-        tool_klass = available_model_tools_map[tool_name]
-
-        # The model tried to use a tool that doesn't exist
-        unless tool_klass
-          add_conversation_history_entry({
-            role: "assistant",
-            content: "<observation>Error: Tool '#{tool_name}' not found. Available tools: #{available_model_tools_map.keys.join(", ")}</observation>"
-          })
-          return
-        end
-
-        unless JSON::Validator.validate(tool_klass.tool_arguments_schema, tool_arguments)
-          add_conversation_history_entry({
-            role: "assistant",
-            content: "<observation>Error: Invalid tool arguments. Please provide valid arguments for the tool '#{tool_name}'. Tool arguments schema: #{tool_klass.tool_arguments_schema.to_json}</observation>" # rubocop:disable Layout/LineLength
-          })
-          return
-        end
-
-        tool_invocation = tool_klass.invoke_tool(tool_arguments: tool_arguments, source: self)
-        observation = tool_klass.observation_for_invocation(tool_invocation)
-
-        # Add the tool invocation to conversation history
-        add_conversation_history_entry({
-          role: "assistant",
-          content: "<observation>#{observation}</observation>"
-        })
-      end
-
-    end
-  end
-end

data/app/models/raif/agents/re_act_step.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Raif
-  module Agents
-    class ReActStep
-      attr_reader :model_response_text
-
-      def initialize(model_response_text:)
-        @model_response_text = model_response_text
-      end
-
-      def thought
-        @thought ||= extract_tag_content("thought")
-      end
-
-      def answer
-        @answer ||= extract_tag_content("answer")
-      end
-
-      def action
-        @action ||= extract_tag_content("action")
-      end
-
-    private
-
-      def extract_tag_content(tag_name)
-        match = model_response_text.match(%r{<#{tag_name}>(.*?)</#{tag_name}>}m)
-        match ? match[1].strip : nil
-      end
-
-    end
-  end
-end