raif 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/jobs/raif/conversation_entry_job.rb +1 -1
- data/app/models/raif/agents/re_act_step.rb +1 -2
- data/app/models/raif/concerns/has_llm.rb +1 -1
- data/app/models/raif/concerns/task_run_args.rb +62 -0
- data/app/models/raif/conversation.rb +5 -1
- data/app/models/raif/conversation_entry.rb +6 -8
- data/app/models/raif/llm.rb +1 -1
- data/app/models/raif/llms/open_router.rb +3 -1
- data/app/models/raif/task.rb +22 -9
- data/app/views/raif/conversation_entries/_form.html.erb +1 -1
- data/app/views/raif/conversations/_full_conversation.html.erb +3 -6
- data/app/views/raif/conversations/_initial_chat_message.html.erb +5 -0
- data/config/locales/en.yml +8 -0
- data/db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb +13 -0
- data/db/migrate/20250811171150_make_raif_task_creator_optional.rb +8 -0
- data/exe/raif +7 -0
- data/lib/generators/raif/agent/agent_generator.rb +22 -7
- data/lib/generators/raif/agent/templates/agent.rb.tt +20 -24
- data/lib/generators/raif/agent/templates/agent_eval_set.rb.tt +48 -0
- data/lib/generators/raif/agent/templates/application_agent.rb.tt +0 -2
- data/lib/generators/raif/base_generator.rb +19 -0
- data/lib/generators/raif/conversation/conversation_generator.rb +21 -2
- data/lib/generators/raif/conversation/templates/application_conversation.rb.tt +0 -2
- data/lib/generators/raif/conversation/templates/conversation.rb.tt +29 -33
- data/lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt +70 -0
- data/lib/generators/raif/eval_set/eval_set_generator.rb +28 -0
- data/lib/generators/raif/eval_set/templates/eval_set.rb.tt +21 -0
- data/lib/generators/raif/evals/setup/setup_generator.rb +47 -0
- data/lib/generators/raif/install/install_generator.rb +15 -0
- data/lib/generators/raif/install/templates/initializer.rb +11 -0
- data/lib/generators/raif/model_tool/model_tool_generator.rb +5 -5
- data/lib/generators/raif/model_tool/templates/model_tool.rb.tt +78 -78
- data/lib/generators/raif/model_tool/templates/model_tool_invocation_partial.html.erb.tt +1 -1
- data/lib/generators/raif/task/task_generator.rb +22 -3
- data/lib/generators/raif/task/templates/application_task.rb.tt +0 -2
- data/lib/generators/raif/task/templates/task.rb.tt +55 -59
- data/lib/generators/raif/task/templates/task_eval_set.rb.tt +54 -0
- data/lib/raif/cli/base.rb +39 -0
- data/lib/raif/cli/evals.rb +47 -0
- data/lib/raif/cli/evals_setup.rb +27 -0
- data/lib/raif/cli.rb +67 -0
- data/lib/raif/configuration.rb +20 -6
- data/lib/raif/evals/eval.rb +30 -0
- data/lib/raif/evals/eval_set.rb +111 -0
- data/lib/raif/evals/eval_sets/expectations.rb +53 -0
- data/lib/raif/evals/eval_sets/llm_judge_expectations.rb +255 -0
- data/lib/raif/evals/expectation_result.rb +39 -0
- data/lib/raif/evals/llm_judge.rb +32 -0
- data/lib/raif/evals/llm_judges/binary.rb +94 -0
- data/lib/raif/evals/llm_judges/comparative.rb +89 -0
- data/lib/raif/evals/llm_judges/scored.rb +63 -0
- data/lib/raif/evals/llm_judges/summarization.rb +166 -0
- data/lib/raif/evals/run.rb +201 -0
- data/lib/raif/evals/scoring_rubric.rb +174 -0
- data/lib/raif/evals.rb +26 -0
- data/lib/raif/llm_registry.rb +33 -0
- data/lib/raif/migration_checker.rb +3 -3
- data/lib/raif/utils/colors.rb +23 -0
- data/lib/raif/utils.rb +1 -0
- data/lib/raif/version.rb +1 -1
- data/lib/raif.rb +4 -0
- data/spec/support/current_temperature_test_tool.rb +34 -0
- data/spec/support/test_conversation.rb +1 -1
- metadata +35 -3
@@ -1,39 +1,35 @@
|
|
1
|
-
|
1
|
+
<% raif_module_namespacing(["Conversations"]) do -%>
|
2
|
+
class <%= class_name.demodulize %> < Raif::ApplicationConversation
|
3
|
+
# Set the response format for the conversation. Options are :html, :text, or :json.
|
4
|
+
# If you set this to something other than :text, make sure to include instructions to the model in your system prompt
|
5
|
+
llm_response_format :<%= options[:response_format] %>
|
2
6
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
# Set the response format for the task. Options are :html, :text, or :json.
|
7
|
-
# If you set this to something other than :text, make sure to include instructions to the model in your system prompt
|
8
|
-
llm_response_format :<%= options[:response_format] %>
|
7
|
+
# If you want to always include a certain set of model tools with this conversation type,
|
8
|
+
# uncomment this callback to populate the available_model_tools attribute with your desired model tools.
|
9
|
+
# before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
11
|
+
# Override the methods below to customize the system prompt for this conversation type.
|
12
|
+
# def system_prompt_intro
|
13
|
+
# Raif.config.conversation_system_prompt_intro
|
14
|
+
# end
|
13
15
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
16
|
+
# def build_system_prompt
|
17
|
+
# <<~PROMPT
|
18
|
+
# #{system_prompt_intro}
|
19
|
+
# #{system_prompt_language_preference}
|
20
|
+
# PROMPT
|
21
|
+
# end
|
18
22
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
# PROMPT
|
24
|
-
# end
|
23
|
+
# Override this method to set the initial message shown to the user.
|
24
|
+
# def initial_chat_message
|
25
|
+
# I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
|
26
|
+
# end
|
25
27
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
# By default, it just passes the model response message through, but you can override
|
33
|
-
# for custom response message processing
|
34
|
-
# def process_model_response_message(message:, entry:)
|
35
|
-
# message
|
36
|
-
# end
|
37
|
-
end
|
28
|
+
# This method will be called when receing a model response to a Raif::ConversationEntry
|
29
|
+
# By default, it just passes the model response message through, but you can override
|
30
|
+
# for custom response message processing
|
31
|
+
# def process_model_response_message(message:, entry:)
|
32
|
+
# message
|
33
|
+
# end
|
38
34
|
end
|
39
|
-
end
|
35
|
+
<% end -%>
|
@@ -0,0 +1,70 @@
|
|
1
|
+
<% raif_module_namespacing(["Evals", "Conversations"]) do -%>
|
2
|
+
class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
|
3
|
+
# Run this eval set with:
|
4
|
+
# bundle exec raif evals ./<%= eval_set_file_path %>
|
5
|
+
|
6
|
+
# Setup method runs before each eval
|
7
|
+
setup do
|
8
|
+
# Common setup code
|
9
|
+
# @user = User.create!(email: "test@example.com")
|
10
|
+
# @conversation = Raif::Conversations::<%= class_name %>.create!(creator: @user)
|
11
|
+
end
|
12
|
+
|
13
|
+
# Teardown runs after each eval
|
14
|
+
teardown do
|
15
|
+
# Cleanup code
|
16
|
+
end
|
17
|
+
|
18
|
+
eval "<%= class_name %> responds appropriately to user greeting" do
|
19
|
+
# entry = @conversation.entries.create!(
|
20
|
+
# user_message: "Hello, how are you?",
|
21
|
+
# creator: @user
|
22
|
+
# )
|
23
|
+
|
24
|
+
# entry.process_entry!
|
25
|
+
|
26
|
+
# expect "generates a response" do
|
27
|
+
# entry.model_response_message.present?
|
28
|
+
# end
|
29
|
+
|
30
|
+
# expect "response is friendly" do
|
31
|
+
# entry.model_response_message.match?(/hello|hi|greetings/i)
|
32
|
+
# end
|
33
|
+
end
|
34
|
+
|
35
|
+
eval "<%= class_name %> maintains conversation context" do
|
36
|
+
# First message establishes context
|
37
|
+
# first_entry = @conversation.entries.create!(
|
38
|
+
# user_message: "My name is Alice",
|
39
|
+
# creator: @user
|
40
|
+
# )
|
41
|
+
# first_entry.process_entry!
|
42
|
+
|
43
|
+
# Second message references context
|
44
|
+
# second_entry = @conversation.entries.create!(
|
45
|
+
# user_message: "What's my name?",
|
46
|
+
# creator: @user
|
47
|
+
# )
|
48
|
+
# second_entry.process_entry!
|
49
|
+
|
50
|
+
# expect "remembers the user's name" do
|
51
|
+
# second_entry.model_response_message.include?("Alice")
|
52
|
+
# end
|
53
|
+
end
|
54
|
+
|
55
|
+
eval "<%= class_name %> handles tool invocations correctly" do
|
56
|
+
# Test if your conversation uses tools
|
57
|
+
# @conversation.update!(available_model_tools: [ "Raif::ModelTools::FetchUrl" ])
|
58
|
+
|
59
|
+
# entry = @conversation.entries.create!(
|
60
|
+
# user_message: "What can you tell me about the content of https://en.wikipedia.org/wiki/Moon",
|
61
|
+
# creator: @user
|
62
|
+
# )
|
63
|
+
|
64
|
+
# entry.process_entry!
|
65
|
+
|
66
|
+
# expect_tool_invocation(entry, "Raif::ModelTools::FetchUrl", with: { url: "https://en.wikipedia.org/wiki/Moon" })
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
<% end -%>
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "../base_generator"
|
4
|
+
|
5
|
+
module Raif
|
6
|
+
module Generators
|
7
|
+
class EvalSetGenerator < BaseGenerator
|
8
|
+
source_root File.expand_path("templates", __dir__)
|
9
|
+
|
10
|
+
def create_eval_set_file
|
11
|
+
template "eval_set.rb.tt", eval_set_file_path
|
12
|
+
end
|
13
|
+
|
14
|
+
def show_instructions
|
15
|
+
say "\nEval set created!"
|
16
|
+
say "To run this eval set: bundle exec raif evals ./#{eval_set_file_path}"
|
17
|
+
say "To run all eval sets: bundle exec raif evals"
|
18
|
+
say ""
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def eval_set_file_path
|
24
|
+
File.join("raif_evals", "eval_sets", class_path, "#{file_name}_eval_set.rb")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
<% raif_module_namespacing(["Evals"]) do -%>
|
2
|
+
class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
|
3
|
+
# Run this eval set with:
|
4
|
+
# bundle exec raif evals ./<%= eval_set_file_path %>
|
5
|
+
|
6
|
+
# Setup method runs before each eval
|
7
|
+
setup do
|
8
|
+
# Common setup code
|
9
|
+
end
|
10
|
+
|
11
|
+
# Teardown runs after each eval
|
12
|
+
teardown do
|
13
|
+
# Cleanup code
|
14
|
+
end
|
15
|
+
|
16
|
+
eval "description of your eval" do
|
17
|
+
# Your eval code here
|
18
|
+
# expect_tool_invocation, expect_no_tool_invocation, expect, etc.
|
19
|
+
end
|
20
|
+
end
|
21
|
+
<% end -%>
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails/generators"
|
4
|
+
|
5
|
+
module Raif
|
6
|
+
module Generators
|
7
|
+
module Evals
|
8
|
+
class SetupGenerator < Rails::Generators::Base
|
9
|
+
source_root File.expand_path("templates", __dir__)
|
10
|
+
|
11
|
+
def create_directories
|
12
|
+
empty_directory "raif_evals"
|
13
|
+
empty_directory "raif_evals/eval_sets"
|
14
|
+
empty_directory "raif_evals/files"
|
15
|
+
empty_directory "raif_evals/results"
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_setup_file
|
19
|
+
create_file "raif_evals/setup.rb", <<~EOS
|
20
|
+
#
|
21
|
+
# This file is loaded at the start of a run of your evals.
|
22
|
+
#
|
23
|
+
# Add any setup code that should run before your evals.
|
24
|
+
#
|
25
|
+
EOS
|
26
|
+
end
|
27
|
+
|
28
|
+
def create_gitignore
|
29
|
+
create_file "raif_evals/results/.gitignore", <<~EOS
|
30
|
+
*
|
31
|
+
!.gitignore
|
32
|
+
EOS
|
33
|
+
end
|
34
|
+
|
35
|
+
def show_instructions
|
36
|
+
say "\nRaif evals setup complete!", :green
|
37
|
+
say "You can create evals with: rails g raif:eval_set ExampleName"
|
38
|
+
say ""
|
39
|
+
say "Run evals with:"
|
40
|
+
say " bundle exec raif evals # Run all evals"
|
41
|
+
say " bundle exec raif evals ./raif_evals/eval_sets/my_eval_set.rb # Run specific eval set"
|
42
|
+
say ""
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -24,8 +24,23 @@ module Raif
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def add_engine_route
|
27
|
+
routes_file = "config/routes.rb"
|
28
|
+
|
29
|
+
if File.exist?(routes_file)
|
30
|
+
routes_content = File.read(routes_file)
|
31
|
+
if routes_content.include?("mount Raif::Engine")
|
32
|
+
say "Raif is already mounted in #{routes_file}, skipping route", :yellow
|
33
|
+
return
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
27
37
|
route 'mount Raif::Engine => "/raif"'
|
28
38
|
end
|
39
|
+
|
40
|
+
def setup_evals
|
41
|
+
say "\nSetting up Raif evals...", :green
|
42
|
+
generate "raif:evals:setup"
|
43
|
+
end
|
29
44
|
end
|
30
45
|
end
|
31
46
|
end
|
@@ -98,6 +98,9 @@ Raif.configure do |config|
|
|
98
98
|
# Or you can use a lambda to return a dynamic system prompt intro:
|
99
99
|
# config.task_system_prompt_intro = ->(task){ "You are a helpful assistant. Today's date is #{Date.today.strftime('%B %d, %Y')}." }
|
100
100
|
|
101
|
+
# Whether the creator association is optional for Raif::Task. Defaults to true.
|
102
|
+
# config.task_creator_optional = true
|
103
|
+
|
101
104
|
# The system prompt intro for Raif::Conversation instances. Defaults to "You are a helpful assistant who is collaborating with a teammate."
|
102
105
|
# config.conversation_system_prompt_intro = "You are a helpful assistant who is collaborating with a teammate."
|
103
106
|
# Or you can use a lambda to return a dynamic system prompt intro:
|
@@ -134,4 +137,12 @@ Raif.configure do |config|
|
|
134
137
|
# Whether LLM API requests are enabled. Defaults to true.
|
135
138
|
# Use this to globally disable requests to LLM APIs.
|
136
139
|
# config.llm_api_requests_enabled = true
|
140
|
+
|
141
|
+
# The default LLM model to use for LLM-as-judge evaluations.
|
142
|
+
# If not set, falls back to the default_llm_model_key.
|
143
|
+
# config.evals_default_llm_judge_model_key = ENV["RAIF_EVALS_DEFAULT_LLM_JUDGE_MODEL_KEY"].presence
|
144
|
+
|
145
|
+
# Whether to output verbose information during evaluation runs. Defaults to false.
|
146
|
+
# When true, provides more detailed output including individual test results.
|
147
|
+
# config.evals_verbose_output = false
|
137
148
|
end
|
@@ -1,17 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative "../base_generator"
|
4
|
+
|
3
5
|
module Raif
|
4
6
|
module Generators
|
5
|
-
class ModelToolGenerator <
|
7
|
+
class ModelToolGenerator < BaseGenerator
|
6
8
|
source_root File.expand_path("templates", __dir__)
|
7
9
|
|
8
10
|
desc "Creates a new model tool for the LLM to invoke in app/models/raif/model_tools"
|
9
11
|
|
10
12
|
def create_model_tool_file
|
11
|
-
template "model_tool.rb.tt", File.join("app/models/raif/model_tools", "#{file_name}.rb")
|
12
|
-
|
13
|
-
# Generate the view partial for the tool invocation
|
14
|
-
template "model_tool_invocation_partial.html.erb.tt", File.join("app/views/raif/model_tool_invocations", "_#{file_name}.html.erb")
|
13
|
+
template "model_tool.rb.tt", File.join("app/models/raif/model_tools", class_path, "#{file_name}.rb")
|
14
|
+
template "model_tool_invocation_partial.html.erb.tt", File.join("app/views/raif/model_tool_invocations", class_path, "_#{file_name}.html.erb")
|
15
15
|
end
|
16
16
|
|
17
17
|
def success_message
|
@@ -1,89 +1,89 @@
|
|
1
|
-
|
1
|
+
<% raif_module_namespacing(["ModelTools"]) do -%>
|
2
|
+
class <%= class_name.demodulize %> < Raif::ModelTool
|
3
|
+
# For example tool implementations, see:
|
4
|
+
# Wikipedia Search Tool: https://github.com/CultivateLabs/raif/blob/main/app/models/raif/model_tools/wikipedia_search.rb
|
5
|
+
# Fetch URL Tool: https://github.com/CultivateLabs/raif/blob/main/app/models/raif/model_tools/fetch_url.rb
|
2
6
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
# Fetch URL Tool: https://github.com/CultivateLabs/raif/blob/main/app/models/raif/model_tools/fetch_url.rb
|
7
|
-
|
8
|
-
tool_description do
|
9
|
-
"Description of your tool that will be provided to the LLM so it knows when to invoke it"
|
10
|
-
end
|
7
|
+
tool_description do
|
8
|
+
"Description of your tool that will be provided to the LLM so it knows when to invoke it"
|
9
|
+
end
|
11
10
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
#
|
16
|
-
# All attributes will be required and additionalProperties will be set to false.
|
17
|
-
#
|
18
|
-
# See https://docs.raif.ai/learn_more/json_schemas for more information about defining JSON schemas.
|
19
|
-
tool_arguments_schema do
|
20
|
-
# string :title, description: "The title of the operation", minLength: 3
|
11
|
+
# Define the schema for the arguments that the LLM should use when invoking your tool.
|
12
|
+
# It should be a valid JSON schema. When the model invokes your tool,
|
13
|
+
# the arguments it provides will be validated against this schema using JSON::Validator from the json-schema gem.
|
21
14
|
#
|
22
|
-
#
|
23
|
-
# boolean :is_red, description: "Whether the widget is red"
|
24
|
-
# integer :rating, description: "A rating of the widget from 1 to 10", minimum: 1, maximum: 10
|
25
|
-
# array :tags, description: "Associated tags" do
|
26
|
-
# items type: "string"
|
27
|
-
# end
|
28
|
-
# end
|
15
|
+
# All attributes will be required and additionalProperties will be set to false.
|
29
16
|
#
|
30
|
-
#
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
17
|
+
# See https://docs.raif.ai/learn_more/json_schemas for more information about defining JSON schemas.
|
18
|
+
tool_arguments_schema do
|
19
|
+
# string :title, description: "The title of the operation", minLength: 3
|
20
|
+
#
|
21
|
+
# object :widget, description: "A widget's description" do
|
22
|
+
# boolean :is_red, description: "Whether the widget is red"
|
23
|
+
# integer :rating, description: "A rating of the widget from 1 to 10", minimum: 1, maximum: 10
|
24
|
+
# array :tags, description: "Associated tags" do
|
25
|
+
# items type: "string"
|
26
|
+
# end
|
27
|
+
# end
|
28
|
+
#
|
29
|
+
# array :products, description: "List of products" do
|
30
|
+
# object do
|
31
|
+
# integer :id, description: "Product identifier"
|
32
|
+
# string :name, description: "Product name"
|
33
|
+
# number :price, description: "Product price", minimum: 0
|
34
|
+
# end
|
35
|
+
# end
|
36
|
+
end
|
38
37
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
38
|
+
# An example of how the LLM should invoke your tool. This should return a hash with name and arguments keys.
|
39
|
+
# `to_json` will be called on it and provided to the LLM as an example of how to invoke your tool.
|
40
|
+
example_model_invocation do
|
41
|
+
{
|
42
|
+
"name": tool_name,
|
43
|
+
"arguments": {}
|
44
|
+
}
|
45
|
+
end
|
47
46
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
47
|
+
class << self
|
48
|
+
# When your tool is invoked by the LLM in a Raif::Agent loop,
|
49
|
+
# the results of the tool invocation are provided back to the LLM as an observation.
|
50
|
+
# This method should return whatever you want provided to the LLM.
|
51
|
+
# For example, if you were implementing a GoogleSearch tool, this might return a JSON
|
52
|
+
# object containing search results for the query.
|
53
|
+
def observation_for_invocation(tool_invocation)
|
54
|
+
return "No results found" unless tool_invocation.result.present?
|
56
55
|
|
57
|
-
|
58
|
-
|
56
|
+
JSON.pretty_generate(tool_invocation.result)
|
57
|
+
end
|
59
58
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
59
|
+
# When your tool is invoked in a Raif::Conversation, should the result be automatically provided back to the model?
|
60
|
+
# When true, observation_for_invocation will be used to produce the observation provided to the model
|
61
|
+
def triggers_observation_to_model?
|
62
|
+
false
|
63
|
+
end
|
64
|
+
|
65
|
+
# When the LLM invokes your tool, this method will be called with a `Raif::ModelToolInvocation` record as an argument.
|
66
|
+
# It should handle the actual execution of the tool.
|
67
|
+
# For example, if you are implementing a GoogleSearch tool, this method should run the actual search
|
68
|
+
# and store the results in the tool_invocation's result JSON column.
|
69
|
+
def process_invocation(tool_invocation)
|
70
|
+
# Extract arguments from tool_invocation.tool_arguments
|
71
|
+
# query = tool_invocation.tool_arguments["query"]
|
72
|
+
#
|
73
|
+
# Process the invocation and perform the desired action
|
74
|
+
# ...
|
75
|
+
#
|
76
|
+
# Store the results in the tool_invocation
|
77
|
+
# tool_invocation.update!(
|
78
|
+
# result: {
|
79
|
+
# # Your result data structure
|
80
|
+
# }
|
81
|
+
# )
|
82
|
+
#
|
83
|
+
# Return the result
|
84
|
+
# tool_invocation.result
|
85
|
+
end
|
65
86
|
|
66
|
-
# When the LLM invokes your tool, this method will be called with a `Raif::ModelToolInvocation` record as an argument.
|
67
|
-
# It should handle the actual execution of the tool.
|
68
|
-
# For example, if you are implementing a GoogleSearch tool, this method should run the actual search
|
69
|
-
# and store the results in the tool_invocation's result JSON column.
|
70
|
-
def process_invocation(tool_invocation)
|
71
|
-
# Extract arguments from tool_invocation.tool_arguments
|
72
|
-
# query = tool_invocation.tool_arguments["query"]
|
73
|
-
#
|
74
|
-
# Process the invocation and perform the desired action
|
75
|
-
# ...
|
76
|
-
#
|
77
|
-
# Store the results in the tool_invocation
|
78
|
-
# tool_invocation.update!(
|
79
|
-
# result: {
|
80
|
-
# # Your result data structure
|
81
|
-
# }
|
82
|
-
# )
|
83
|
-
#
|
84
|
-
# Return the result
|
85
|
-
# tool_invocation.result
|
86
87
|
end
|
87
88
|
end
|
88
|
-
|
89
|
-
end
|
89
|
+
<% end -%>
|
@@ -1,8 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative "../base_generator"
|
4
|
+
|
3
5
|
module Raif
|
4
6
|
module Generators
|
5
|
-
class TaskGenerator <
|
7
|
+
class TaskGenerator < BaseGenerator
|
6
8
|
source_root File.expand_path("templates", __dir__)
|
7
9
|
|
8
10
|
class_option :response_format,
|
@@ -10,6 +12,11 @@ module Raif
|
|
10
12
|
default: "text",
|
11
13
|
desc: "Response format for the task (text, html, or json)"
|
12
14
|
|
15
|
+
class_option :skip_eval_set,
|
16
|
+
type: :boolean,
|
17
|
+
default: false,
|
18
|
+
desc: "Skip generating the corresponding eval set"
|
19
|
+
|
13
20
|
def create_application_task
|
14
21
|
template "application_task.rb.tt", "app/models/raif/application_task.rb" unless File.exist?("app/models/raif/application_task.rb")
|
15
22
|
end
|
@@ -18,11 +25,23 @@ module Raif
|
|
18
25
|
template "task.rb.tt", File.join("app/models/raif/tasks", class_path, "#{file_name}.rb")
|
19
26
|
end
|
20
27
|
|
28
|
+
def create_eval_set
|
29
|
+
return if options[:skip_eval_set]
|
30
|
+
|
31
|
+
template "task_eval_set.rb.tt", eval_set_file_path
|
32
|
+
end
|
33
|
+
|
34
|
+
def show_instructions
|
35
|
+
say "\nTask created!"
|
36
|
+
say ""
|
37
|
+
end
|
38
|
+
|
21
39
|
private
|
22
40
|
|
23
|
-
def
|
24
|
-
|
41
|
+
def eval_set_file_path
|
42
|
+
File.join("raif_evals", "eval_sets", "tasks", class_path, "#{file_name}_eval_set.rb")
|
25
43
|
end
|
44
|
+
|
26
45
|
end
|
27
46
|
end
|
28
47
|
end
|