raif 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/jobs/raif/conversation_entry_job.rb +1 -1
- data/app/models/raif/agents/re_act_step.rb +1 -2
- data/app/models/raif/concerns/has_llm.rb +1 -1
- data/app/models/raif/concerns/task_run_args.rb +62 -0
- data/app/models/raif/conversation.rb +5 -1
- data/app/models/raif/conversation_entry.rb +6 -8
- data/app/models/raif/llm.rb +1 -1
- data/app/models/raif/llms/open_router.rb +3 -1
- data/app/models/raif/task.rb +22 -9
- data/app/views/raif/conversation_entries/_form.html.erb +1 -1
- data/app/views/raif/conversations/_full_conversation.html.erb +3 -6
- data/app/views/raif/conversations/_initial_chat_message.html.erb +5 -0
- data/config/locales/en.yml +8 -0
- data/db/migrate/20250804013843_add_task_run_args_to_raif_tasks.rb +13 -0
- data/db/migrate/20250811171150_make_raif_task_creator_optional.rb +8 -0
- data/exe/raif +7 -0
- data/lib/generators/raif/agent/agent_generator.rb +22 -7
- data/lib/generators/raif/agent/templates/agent.rb.tt +20 -24
- data/lib/generators/raif/agent/templates/agent_eval_set.rb.tt +48 -0
- data/lib/generators/raif/agent/templates/application_agent.rb.tt +0 -2
- data/lib/generators/raif/base_generator.rb +19 -0
- data/lib/generators/raif/conversation/conversation_generator.rb +21 -2
- data/lib/generators/raif/conversation/templates/application_conversation.rb.tt +0 -2
- data/lib/generators/raif/conversation/templates/conversation.rb.tt +29 -33
- data/lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt +70 -0
- data/lib/generators/raif/eval_set/eval_set_generator.rb +28 -0
- data/lib/generators/raif/eval_set/templates/eval_set.rb.tt +21 -0
- data/lib/generators/raif/evals/setup/setup_generator.rb +47 -0
- data/lib/generators/raif/install/install_generator.rb +15 -0
- data/lib/generators/raif/install/templates/initializer.rb +11 -0
- data/lib/generators/raif/model_tool/model_tool_generator.rb +5 -5
- data/lib/generators/raif/model_tool/templates/model_tool.rb.tt +78 -78
- data/lib/generators/raif/model_tool/templates/model_tool_invocation_partial.html.erb.tt +1 -1
- data/lib/generators/raif/task/task_generator.rb +22 -3
- data/lib/generators/raif/task/templates/application_task.rb.tt +0 -2
- data/lib/generators/raif/task/templates/task.rb.tt +55 -59
- data/lib/generators/raif/task/templates/task_eval_set.rb.tt +54 -0
- data/lib/raif/cli/base.rb +39 -0
- data/lib/raif/cli/evals.rb +47 -0
- data/lib/raif/cli/evals_setup.rb +27 -0
- data/lib/raif/cli.rb +67 -0
- data/lib/raif/configuration.rb +20 -6
- data/lib/raif/evals/eval.rb +30 -0
- data/lib/raif/evals/eval_set.rb +111 -0
- data/lib/raif/evals/eval_sets/expectations.rb +53 -0
- data/lib/raif/evals/eval_sets/llm_judge_expectations.rb +255 -0
- data/lib/raif/evals/expectation_result.rb +39 -0
- data/lib/raif/evals/llm_judge.rb +32 -0
- data/lib/raif/evals/llm_judges/binary.rb +94 -0
- data/lib/raif/evals/llm_judges/comparative.rb +89 -0
- data/lib/raif/evals/llm_judges/scored.rb +63 -0
- data/lib/raif/evals/llm_judges/summarization.rb +166 -0
- data/lib/raif/evals/run.rb +201 -0
- data/lib/raif/evals/scoring_rubric.rb +174 -0
- data/lib/raif/evals.rb +26 -0
- data/lib/raif/llm_registry.rb +33 -0
- data/lib/raif/migration_checker.rb +3 -3
- data/lib/raif/utils/colors.rb +23 -0
- data/lib/raif/utils.rb +1 -0
- data/lib/raif/version.rb +1 -1
- data/lib/raif.rb +4 -0
- data/spec/support/current_temperature_test_tool.rb +34 -0
- data/spec/support/test_conversation.rb +1 -1
- metadata +35 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '074678c5fc61a6b08ddae200f82baaa948b0c26b0178a1c868c6d3e9d1ed6e0e'
|
4
|
+
data.tar.gz: bea8da28b1245ccaeb52a5f81fdc52f4932016d774b4ccdde20c3af61871c5c3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c7420da7d6676db141ffae3ddc43787631db1190bf0b617f7c284312c85832101fffca6fe0600d35b893ccb8586474d7c22d31a549605a8c51e09dadc132aed2
|
7
|
+
data.tar.gz: 8cd9681ec60a6d20b53aebedef7512051bf434f625354a3e05ec340419321765a2e9d18c381e4fa947565b99c38fc6ac108e037c6622e6d96b2f06f1f7bc3f35
|
@@ -16,7 +16,7 @@ module Raif
|
|
16
16
|
Turbo::StreamsChannel.broadcast_action_to(
|
17
17
|
conversation,
|
18
18
|
action: :raif_scroll_to_bottom,
|
19
|
-
target: dom_id(conversation, :entries)
|
19
|
+
target: ActionView::RecordIdentifier.dom_id(conversation, :entries)
|
20
20
|
)
|
21
21
|
rescue StandardError => e
|
22
22
|
logger.error "Error processing conversation entry: #{e.message}"
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Raif::Concerns::TaskRunArgs
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
class_attribute :_task_run_args, instance_writer: false, default: []
|
8
|
+
end
|
9
|
+
|
10
|
+
class_methods do
|
11
|
+
# DSL for declaring persistent task arguments that will be serialized to the database
|
12
|
+
# @param name [Symbol] The name of the argument
|
13
|
+
def task_run_arg(name)
|
14
|
+
# Ensure each class has its own array copy
|
15
|
+
self._task_run_args = _task_run_args.dup
|
16
|
+
_task_run_args << name.to_sym
|
17
|
+
|
18
|
+
# Define getter that pulls from task_run_args JSON
|
19
|
+
define_method(name) do
|
20
|
+
return instance_variable_get("@#{name}") if instance_variable_defined?("@#{name}")
|
21
|
+
|
22
|
+
value = task_run_args&.dig(name.to_s)
|
23
|
+
return unless value
|
24
|
+
|
25
|
+
# Deserialize GID if it's a string starting with gid://
|
26
|
+
deserialized = if value.is_a?(String) && value.start_with?("gid://")
|
27
|
+
begin
|
28
|
+
GlobalID::Locator.locate(value)
|
29
|
+
rescue ActiveRecord::RecordNotFound
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
else
|
33
|
+
value
|
34
|
+
end
|
35
|
+
|
36
|
+
instance_variable_set("@#{name}", deserialized)
|
37
|
+
end
|
38
|
+
|
39
|
+
# Define setter that stores in memory (for use during run)
|
40
|
+
define_method("#{name}=") do |value|
|
41
|
+
instance_variable_set("@#{name}", value)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# Transform run args into a hash that can be stored in the task_run_args database column
|
46
|
+
def serialize_task_run_args(args)
|
47
|
+
serialized_args = {}
|
48
|
+
_task_run_args.each do |arg_name|
|
49
|
+
next unless args.key?(arg_name)
|
50
|
+
|
51
|
+
value = args[arg_name]
|
52
|
+
serialized_args[arg_name.to_s] = if value.respond_to?(:to_global_id)
|
53
|
+
value.to_global_id.to_s
|
54
|
+
else
|
55
|
+
value
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
serialized_args
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -34,6 +34,10 @@ class Raif::Conversation < Raif::ApplicationRecord
|
|
34
34
|
I18n.t("#{self.class.name.underscore.gsub("/", ".")}.initial_chat_message")
|
35
35
|
end
|
36
36
|
|
37
|
+
def initial_chat_message_partial_path
|
38
|
+
"raif/conversations/initial_chat_message"
|
39
|
+
end
|
40
|
+
|
37
41
|
def prompt_model_for_entry_response(entry:, &block)
|
38
42
|
update(system_prompt: build_system_prompt)
|
39
43
|
|
@@ -59,7 +63,7 @@ class Raif::Conversation < Raif::ApplicationRecord
|
|
59
63
|
Airbrake.notify(notice)
|
60
64
|
end
|
61
65
|
|
62
|
-
|
66
|
+
nil
|
63
67
|
end
|
64
68
|
|
65
69
|
def process_model_response_message(message:, entry:)
|
@@ -63,7 +63,7 @@ class Raif::ConversationEntry < Raif::ApplicationRecord
|
|
63
63
|
broadcast_replace_to raif_conversation
|
64
64
|
end
|
65
65
|
|
66
|
-
if raif_model_completion.parsed_response.present? || raif_model_completion.response_tool_calls.present?
|
66
|
+
if raif_model_completion.present? && (raif_model_completion.parsed_response.present? || raif_model_completion.response_tool_calls.present?)
|
67
67
|
extract_message_and_invoke_tools!
|
68
68
|
create_entry_for_observation! if triggers_observation_to_model?
|
69
69
|
else
|
@@ -83,7 +83,7 @@ class Raif::ConversationEntry < Raif::ApplicationRecord
|
|
83
83
|
def create_entry_for_observation!
|
84
84
|
follow_up_entry = raif_conversation.entries.create!(creator: creator)
|
85
85
|
Raif::ConversationEntryJob.perform_later(conversation_entry: follow_up_entry)
|
86
|
-
follow_up_entry.broadcast_append_to raif_conversation, target: dom_id(raif_conversation, :entries)
|
86
|
+
follow_up_entry.broadcast_append_to raif_conversation, target: ActionView::RecordIdentifier.dom_id(raif_conversation, :entries)
|
87
87
|
end
|
88
88
|
|
89
89
|
private
|
@@ -94,13 +94,11 @@ private
|
|
94
94
|
self.model_response_message = raif_conversation.process_model_response_message(message: raif_model_completion.parsed_response, entry: self)
|
95
95
|
save!
|
96
96
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
next if tool_klass.nil?
|
97
|
+
raif_model_completion.response_tool_calls&.each do |tool_call|
|
98
|
+
tool_klass = available_model_tools_map[tool_call["name"]]
|
99
|
+
next if tool_klass.nil?
|
101
100
|
|
102
|
-
|
103
|
-
end
|
101
|
+
tool_klass.invoke_tool(tool_arguments: tool_call["arguments"], source: self)
|
104
102
|
end
|
105
103
|
|
106
104
|
completed!
|
data/app/models/raif/llm.rb
CHANGED
@@ -77,7 +77,7 @@ module Raif
|
|
77
77
|
temperature ||= default_temperature
|
78
78
|
max_completion_tokens ||= default_max_completion_tokens
|
79
79
|
|
80
|
-
model_completion = Raif::ModelCompletion.
|
80
|
+
model_completion = Raif::ModelCompletion.create!(
|
81
81
|
messages: format_messages(messages),
|
82
82
|
system_prompt: system_prompt,
|
83
83
|
response_format: response_format,
|
@@ -93,7 +93,9 @@ private
|
|
93
93
|
params[:stream_options] = { include_usage: true }
|
94
94
|
end
|
95
95
|
|
96
|
-
|
96
|
+
# OpenRouter will sometimes complain about combining response_format json and tool calling.
|
97
|
+
# If we're telling it to use the json_response tool, then the json_object response_format should be irrelevant.
|
98
|
+
if model_completion.response_format_json? && params[:tools].blank?
|
97
99
|
params[:response_format] = { type: "json_object" }
|
98
100
|
model_completion.response_format_parameter = "json_object"
|
99
101
|
end
|
data/app/models/raif/task.rb
CHANGED
@@ -9,10 +9,13 @@ module Raif
|
|
9
9
|
include Raif::Concerns::LlmResponseParsing
|
10
10
|
include Raif::Concerns::LlmTemperature
|
11
11
|
include Raif::Concerns::JsonSchemaDefinition
|
12
|
+
include Raif::Concerns::TaskRunArgs
|
12
13
|
|
13
14
|
llm_temperature 0.7
|
14
15
|
|
15
|
-
belongs_to :creator, polymorphic: true
|
16
|
+
belongs_to :creator, polymorphic: true, optional: true
|
17
|
+
|
18
|
+
validates :creator, presence: true, unless: -> { Raif.config.task_creator_optional }
|
16
19
|
|
17
20
|
has_one :raif_model_completion, as: :source, dependent: :destroy, class_name: "Raif::ModelCompletion"
|
18
21
|
|
@@ -32,6 +35,7 @@ module Raif
|
|
32
35
|
attr_accessor :files, :images
|
33
36
|
|
34
37
|
after_initialize -> { self.available_model_tools ||= [] }
|
38
|
+
after_initialize -> { self.task_run_args ||= {} }
|
35
39
|
|
36
40
|
def status
|
37
41
|
if completed_at?
|
@@ -48,15 +52,24 @@ module Raif
|
|
48
52
|
# The primary interface for running a task. It will hit the LLM with the task's prompt and system prompt and return a Raif::Task object.
|
49
53
|
# It will also create a new Raif::ModelCompletion record.
|
50
54
|
#
|
51
|
-
# @param creator [Object] The creator of the task (polymorphic association)
|
55
|
+
# @param creator [Object, nil] The creator of the task (polymorphic association), optional
|
52
56
|
# @param available_model_tools [Array<Class>] Optional array of model tool classes that will be provided to the LLM for it to invoke.
|
53
57
|
# @param llm_model_key [Symbol, String] Optional key for the LLM model to use. If blank, Raif.config.default_llm_model_key will be used.
|
54
58
|
# @param images [Array] Optional array of Raif::ModelImageInput objects to include with the prompt.
|
55
59
|
# @param files [Array] Optional array of Raif::ModelFileInput objects to include with the prompt.
|
56
60
|
# @param args [Hash] Additional arguments to pass to the instance of the task that is created.
|
57
61
|
# @return [Raif::Task, nil] The task instance that was created and run.
|
58
|
-
def self.run(creator
|
59
|
-
task = new(
|
62
|
+
def self.run(creator: nil, available_model_tools: [], llm_model_key: nil, images: [], files: [], **args)
|
63
|
+
task = new(
|
64
|
+
creator: creator,
|
65
|
+
llm_model_key: llm_model_key,
|
66
|
+
available_model_tools: available_model_tools,
|
67
|
+
started_at: Time.current,
|
68
|
+
images: images,
|
69
|
+
files: files,
|
70
|
+
task_run_args: serialize_task_run_args(args),
|
71
|
+
**args
|
72
|
+
)
|
60
73
|
|
61
74
|
task.save!
|
62
75
|
task.run
|
@@ -109,19 +122,19 @@ module Raif
|
|
109
122
|
|
110
123
|
# Returns the LLM prompt for the task.
|
111
124
|
#
|
112
|
-
# @param creator [Object] The creator of the task (polymorphic association)
|
125
|
+
# @param creator [Object, nil] The creator of the task (polymorphic association), optional
|
113
126
|
# @param args [Hash] Additional arguments to pass to the instance of the task that is created.
|
114
127
|
# @return [String] The LLM prompt for the task.
|
115
|
-
def self.prompt(creator
|
128
|
+
def self.prompt(creator: nil, **args)
|
116
129
|
new(creator:, **args).build_prompt
|
117
130
|
end
|
118
131
|
|
119
132
|
# Returns the LLM system prompt for the task.
|
120
133
|
#
|
121
|
-
# @param creator [Object] The creator of the task (polymorphic association)
|
134
|
+
# @param creator [Object, nil] The creator of the task (polymorphic association), optional
|
122
135
|
# @param args [Hash] Additional arguments to pass to the instance of the task that is created.
|
123
136
|
# @return [String] The LLM system prompt for the task.
|
124
|
-
def self.system_prompt(creator
|
137
|
+
def self.system_prompt(creator: nil, **args)
|
125
138
|
new(creator:, **args).build_system_prompt
|
126
139
|
end
|
127
140
|
|
@@ -170,7 +183,7 @@ module Raif
|
|
170
183
|
end
|
171
184
|
|
172
185
|
def populate_prompts
|
173
|
-
self.requested_language_key ||= creator
|
186
|
+
self.requested_language_key ||= creator&.preferred_language_key if creator&.respond_to?(:preferred_language_key)
|
174
187
|
self.prompt = build_prompt
|
175
188
|
self.system_prompt = build_system_prompt
|
176
189
|
end
|
@@ -10,7 +10,7 @@
|
|
10
10
|
<% end %>
|
11
11
|
<% end %>
|
12
12
|
|
13
|
-
<div class="d-flex px-2">
|
13
|
+
<div class="d-flex px-2 align-items-center">
|
14
14
|
<%= f.text_field :user_message,
|
15
15
|
class: "form-control me-2",
|
16
16
|
placeholder: conversation_entry.raif_user_tool_invocation&.message_input_placeholder.presence || t("raif.common.type_your_message"),
|
@@ -1,14 +1,11 @@
|
|
1
1
|
<%= turbo_stream_from conversation %>
|
2
2
|
|
3
|
-
<div id="<%= dom_id(conversation, :entries) %>" class="flex-grow-1 overflow-auto" data-controller="raif--conversations">
|
4
|
-
<%= render
|
5
|
-
content: conversation.initial_chat_message,
|
6
|
-
message_type: :model_response %>
|
7
|
-
|
3
|
+
<div id="<%= dom_id(conversation, :entries) %>" class="flex-grow-1 overflow-auto raif-conversation-entries-container" data-controller="raif--conversations">
|
4
|
+
<%= render conversation.initial_chat_message_partial_path, conversation: conversation %>
|
8
5
|
<%= render conversation.entries.oldest_first %>
|
9
6
|
</div>
|
10
7
|
|
11
|
-
<div id="<%= dom_id(conversation, :entry_input) %>">
|
8
|
+
<div id="<%= dom_id(conversation, :entry_input) %>" class="raif-conversation-entry-input-container">
|
12
9
|
<%= render "raif/conversation_entries/form_with_available_tools",
|
13
10
|
conversation: conversation,
|
14
11
|
conversation_entry: Raif::ConversationEntry.new %>
|
data/config/locales/en.yml
CHANGED
@@ -68,6 +68,9 @@ en:
|
|
68
68
|
open_ai_gpt_4_1_nano: OpenAI GPT-4.1 Nano
|
69
69
|
open_ai_gpt_4o: OpenAI GPT-4o
|
70
70
|
open_ai_gpt_4o_mini: OpenAI GPT-4o Mini
|
71
|
+
open_ai_gpt_5: OpenAI GPT-5
|
72
|
+
open_ai_gpt_5_mini: OpenAI GPT-5 Mini
|
73
|
+
open_ai_gpt_5_nano: OpenAI GPT-5 Nano
|
71
74
|
open_ai_o1: OpenAI o1
|
72
75
|
open_ai_o1_mini: OpenAI o1 Mini
|
73
76
|
open_ai_o3: OpenAI o3
|
@@ -79,6 +82,9 @@ en:
|
|
79
82
|
open_ai_responses_gpt_4_1_nano: OpenAI GPT-4.1 Nano (Responses API)
|
80
83
|
open_ai_responses_gpt_4o: OpenAI GPT-4o (Responses API)
|
81
84
|
open_ai_responses_gpt_4o_mini: OpenAI GPT-4o Mini (Responses API)
|
85
|
+
open_ai_responses_gpt_5: OpenAI GPT-5 (Responses API)
|
86
|
+
open_ai_responses_gpt_5_mini: OpenAI GPT-5 Mini (Responses API)
|
87
|
+
open_ai_responses_gpt_5_nano: OpenAI GPT-5 Nano (Responses API)
|
82
88
|
open_ai_responses_o1: OpenAI o1 (Responses API)
|
83
89
|
open_ai_responses_o1_mini: OpenAI o1 Mini (Responses API)
|
84
90
|
open_ai_responses_o1_pro: OpenAI o1 Pro (Responses API)
|
@@ -93,4 +99,6 @@ en:
|
|
93
99
|
open_router_llama_3_3_70b_instruct: Meta Llama 3.3 70B Instruct (via OpenRouter)
|
94
100
|
open_router_llama_4_maverick: Meta Llama 4 Maverick (via OpenRouter)
|
95
101
|
open_router_llama_4_scout: Meta Llama 4 Scout (via OpenRouter)
|
102
|
+
open_router_open_ai_gpt_oss_120b: OpenAI GPT-OSS 120B (via OpenRouter)
|
103
|
+
open_router_open_ai_gpt_oss_20b: OpenAI GPT-OSS 20B (via OpenRouter)
|
96
104
|
raif_test_llm: Raif Test LLM
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class AddTaskRunArgsToRaifTasks < ActiveRecord::Migration[7.1]
|
4
|
+
def change
|
5
|
+
json_column_type = if connection.adapter_name.downcase.include?("postgresql")
|
6
|
+
:jsonb
|
7
|
+
else
|
8
|
+
:json
|
9
|
+
end
|
10
|
+
|
11
|
+
add_column :raif_tasks, :task_run_args, json_column_type
|
12
|
+
end
|
13
|
+
end
|
data/exe/raif
ADDED
@@ -1,32 +1,47 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative "../base_generator"
|
4
|
+
|
3
5
|
module Raif
|
4
6
|
module Generators
|
5
|
-
class AgentGenerator <
|
7
|
+
class AgentGenerator < BaseGenerator
|
6
8
|
source_root File.expand_path("templates", __dir__)
|
7
9
|
desc "Creates a new Raif::Agent subclass in app/models/raif/agents"
|
8
10
|
|
11
|
+
class_option :skip_eval_set,
|
12
|
+
type: :boolean,
|
13
|
+
default: false,
|
14
|
+
desc: "Skip generating the corresponding eval set"
|
15
|
+
|
9
16
|
def create_application_agent
|
10
17
|
template "application_agent.rb.tt", "app/models/raif/application_agent.rb" unless File.exist?("app/models/raif/application_agent.rb")
|
11
18
|
end
|
12
19
|
|
13
20
|
def create_agent
|
14
|
-
template "agent.rb.tt", "app/models/raif/agents
|
21
|
+
template "agent.rb.tt", File.join("app/models/raif/agents", class_path, "#{file_name}.rb")
|
15
22
|
end
|
16
23
|
|
17
24
|
def create_directory
|
18
25
|
empty_directory "app/models/raif/agents" unless File.directory?("app/models/raif/agents")
|
19
26
|
end
|
20
27
|
|
21
|
-
|
28
|
+
def create_eval_set
|
29
|
+
return if options[:skip_eval_set]
|
30
|
+
|
31
|
+
template "agent_eval_set.rb.tt", eval_set_file_path
|
32
|
+
end
|
22
33
|
|
23
|
-
def
|
24
|
-
|
34
|
+
def show_instructions
|
35
|
+
say "\nAgent created!"
|
36
|
+
say ""
|
25
37
|
end
|
26
38
|
|
27
|
-
|
28
|
-
|
39
|
+
private
|
40
|
+
|
41
|
+
def eval_set_file_path
|
42
|
+
File.join("raif_evals", "eval_sets", "agents", class_path, "#{file_name}_eval_set.rb")
|
29
43
|
end
|
44
|
+
|
30
45
|
end
|
31
46
|
end
|
32
47
|
end
|
@@ -1,28 +1,24 @@
|
|
1
|
-
|
1
|
+
<% raif_module_namespacing(["Agents"]) do -%>
|
2
|
+
class <%= class_name.demodulize %> < Raif::ApplicationAgent
|
3
|
+
# If you want to always include a certain set of model tools with this agent type,
|
4
|
+
# uncomment this callback to populate the available_model_tools attribute with your desired model tools.
|
5
|
+
# def populate_default_model_tools
|
6
|
+
# self.available_model_tools = [
|
7
|
+
# Raif::ModelTools::WikipediaSearch,
|
8
|
+
# Raif::ModelTools::FetchUrl
|
9
|
+
# ]
|
10
|
+
# end
|
2
11
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
#
|
7
|
-
|
8
|
-
# def populate_default_model_tools
|
9
|
-
# self.available_model_tools ||= [
|
10
|
-
# Raif::ModelTools::WikipediaSearchTool,
|
11
|
-
# Raif::ModelTools::FetchUrlTool
|
12
|
-
# ]
|
13
|
-
# end
|
14
|
-
|
15
|
-
# Enter your agent's system prompt here. Alternatively, you can change your agent's superclass
|
16
|
-
# to an existing agent types (like Raif::Agents::ReActAgent) to utilize an existing system prompt.
|
17
|
-
def build_system_prompt
|
18
|
-
# TODO: Implement your system prompt here
|
19
|
-
end
|
12
|
+
# Enter your agent's system prompt here. Alternatively, you can change your agent's superclass
|
13
|
+
# to an existing agent types (like Raif::Agents::ReActAgent) to utilize an existing system prompt.
|
14
|
+
def build_system_prompt
|
15
|
+
# TODO: Implement your system prompt here
|
16
|
+
end
|
20
17
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
end
|
18
|
+
# Each iteration of the agent loop will generate a new Raif::ModelCompletion record and
|
19
|
+
# then call this method with it as an argument.
|
20
|
+
def process_iteration_model_completion(model_completion)
|
21
|
+
# TODO: Implement your iteration processing here
|
26
22
|
end
|
27
23
|
end
|
28
|
-
end
|
24
|
+
<% end -%>
|
@@ -0,0 +1,48 @@
|
|
1
|
+
<% raif_module_namespacing(["Evals", "Agents"]) do -%>
|
2
|
+
class <%= class_name.demodulize %>EvalSet < Raif::Evals::EvalSet
|
3
|
+
# Run this eval set with:
|
4
|
+
# bundle exec raif evals ./<%= eval_set_file_path %>
|
5
|
+
|
6
|
+
# Setup method runs before each eval
|
7
|
+
setup do
|
8
|
+
# Common setup code
|
9
|
+
# @user = User.create!(email: "test@example.com")
|
10
|
+
end
|
11
|
+
|
12
|
+
# Teardown runs after each eval
|
13
|
+
teardown do
|
14
|
+
# Cleanup code
|
15
|
+
end
|
16
|
+
|
17
|
+
eval "<%= class_name %> completes task successfully" do
|
18
|
+
# agent = Raif::Agents::<%= class_name %>.create!(
|
19
|
+
# creator: @user,
|
20
|
+
# task: "Your specific task here",
|
21
|
+
# available_model_tools: [] # Add your tools here if needed
|
22
|
+
# )
|
23
|
+
|
24
|
+
# agent.run!
|
25
|
+
|
26
|
+
# expect "agent completes successfully" do
|
27
|
+
# agent.completed?
|
28
|
+
# end
|
29
|
+
|
30
|
+
# expect "produces expected output" do
|
31
|
+
# agent.final_answer.include?("expected content")
|
32
|
+
# end
|
33
|
+
end
|
34
|
+
|
35
|
+
eval "<%= class_name %> uses tools correctly" do
|
36
|
+
# agent = Raif::Agents::<%= class_name %>.create!(
|
37
|
+
# creator: @user,
|
38
|
+
# task: "A task that requires tool usage",
|
39
|
+
# available_model_tools: ["expected_tool_name"]
|
40
|
+
# )
|
41
|
+
|
42
|
+
# agent.run!
|
43
|
+
|
44
|
+
# expect_tool_invocation(agent, "expected_tool_name")
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
<% end -%>
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Raif
|
4
|
+
class BaseGenerator < Rails::Generators::NamedBase
|
5
|
+
private
|
6
|
+
|
7
|
+
def raif_module_namespacing(intermediate_modules = [], &block)
|
8
|
+
content = capture(&block).rstrip
|
9
|
+
|
10
|
+
modules_names = intermediate_modules + class_path.map(&:camelize)
|
11
|
+
modules_names.reverse.each do |module_name|
|
12
|
+
content = indent "module #{module_name}\n#{content}\nend", 2
|
13
|
+
end
|
14
|
+
|
15
|
+
concat("module Raif\n#{content}\nend\n")
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
@@ -1,8 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative "../base_generator"
|
4
|
+
|
3
5
|
module Raif
|
4
6
|
module Generators
|
5
|
-
class ConversationGenerator <
|
7
|
+
class ConversationGenerator < BaseGenerator
|
6
8
|
source_root File.expand_path("templates", __dir__)
|
7
9
|
|
8
10
|
desc "Creates a new conversation type in the app/models/raif/conversations directory"
|
@@ -12,19 +14,30 @@ module Raif
|
|
12
14
|
default: "text",
|
13
15
|
desc: "Response format for the task (text, html, or json)"
|
14
16
|
|
17
|
+
class_option :skip_eval_set,
|
18
|
+
type: :boolean,
|
19
|
+
default: false,
|
20
|
+
desc: "Skip generating the corresponding eval set"
|
21
|
+
|
15
22
|
def create_application_conversation
|
16
23
|
template "application_conversation.rb.tt",
|
17
24
|
"app/models/raif/application_conversation.rb" unless File.exist?("app/models/raif/application_conversation.rb")
|
18
25
|
end
|
19
26
|
|
20
27
|
def create_conversation_file
|
21
|
-
template "conversation.rb.tt", File.join("app/models/raif/conversations", "#{file_name}.rb")
|
28
|
+
template "conversation.rb.tt", File.join("app/models/raif/conversations", class_path, "#{file_name}.rb")
|
22
29
|
end
|
23
30
|
|
24
31
|
def create_directory
|
25
32
|
empty_directory "app/models/raif/conversations" unless File.directory?("app/models/raif/conversations")
|
26
33
|
end
|
27
34
|
|
35
|
+
def create_eval_set
|
36
|
+
return if options[:skip_eval_set]
|
37
|
+
|
38
|
+
template "conversation_eval_set.rb.tt", eval_set_file_path
|
39
|
+
end
|
40
|
+
|
28
41
|
def success_message
|
29
42
|
say_status :success, "Conversation type created successfully", :green
|
30
43
|
say "\nYou can now implement your conversation type in:"
|
@@ -32,6 +45,12 @@ module Raif
|
|
32
45
|
say "\nDon't forget to add it to the config.conversation_types in your Raif configuration"
|
33
46
|
say "For example: config.conversation_types += ['Raif::Conversations::#{class_name}']\n\n"
|
34
47
|
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def eval_set_file_path
|
52
|
+
File.join("raif_evals", "eval_sets", "conversations", class_path, "#{file_name}_eval_set.rb")
|
53
|
+
end
|
35
54
|
end
|
36
55
|
end
|
37
56
|
end
|