raif 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -7
- data/app/assets/builds/raif.css +4 -1
- data/app/assets/builds/raif_admin.css +52 -2
- data/app/assets/builds/raif_admin_sprockets.js +2709 -0
- data/app/assets/javascript/raif/admin/copy_to_clipboard_controller.js +132 -0
- data/app/assets/javascript/raif/admin/cost_estimate_controller.js +80 -0
- data/app/assets/javascript/raif/admin/judge_config_controller.js +23 -0
- data/app/assets/javascript/raif/admin/select_all_checkboxes_controller.js +33 -0
- data/app/assets/javascript/raif/admin/sortable_table_controller.js +51 -0
- data/app/assets/javascript/raif/admin/table_search_controller.js +15 -0
- data/app/assets/javascript/raif/admin/tom_select_controller.js +33 -0
- data/app/assets/javascript/raif/controllers/conversations_controller.js +1 -1
- data/app/assets/javascript/raif_admin.js +23 -0
- data/app/assets/javascript/raif_admin_sprockets.js +24 -0
- data/app/assets/stylesheets/raif/admin/conversation.scss +16 -0
- data/app/assets/stylesheets/raif/conversations.scss +3 -0
- data/app/assets/stylesheets/raif.scss +2 -1
- data/app/assets/stylesheets/raif_admin.scss +50 -1
- data/app/controllers/raif/admin/agents_controller.rb +27 -1
- data/app/controllers/raif/admin/application_controller.rb +16 -0
- data/app/controllers/raif/admin/configs_controller.rb +95 -0
- data/app/controllers/raif/admin/llms_controller.rb +27 -0
- data/app/controllers/raif/admin/model_completions_controller.rb +24 -1
- data/app/controllers/raif/admin/model_tool_invocations_controller.rb +7 -1
- data/app/controllers/raif/admin/prompt_studio/agents_controller.rb +25 -0
- data/app/controllers/raif/admin/prompt_studio/base_controller.rb +32 -0
- data/app/controllers/raif/admin/prompt_studio/batch_runs_controller.rb +102 -0
- data/app/controllers/raif/admin/prompt_studio/conversations_controller.rb +25 -0
- data/app/controllers/raif/admin/prompt_studio/tasks_controller.rb +64 -0
- data/app/controllers/raif/admin/stats/model_tool_invocations_controller.rb +21 -0
- data/app/controllers/raif/admin/stats/tasks_controller.rb +15 -6
- data/app/controllers/raif/admin/stats_controller.rb +32 -3
- data/app/controllers/raif/admin/tasks_controller.rb +5 -0
- data/app/controllers/raif/conversation_entries_controller.rb +1 -0
- data/app/controllers/raif/conversations_controller.rb +10 -2
- data/app/helpers/raif/application_helper.rb +40 -0
- data/app/jobs/raif/conversation_entry_job.rb +8 -6
- data/app/jobs/raif/prompt_studio_batch_run_item_job.rb +11 -0
- data/app/jobs/raif/prompt_studio_batch_run_job.rb +15 -0
- data/app/jobs/raif/prompt_studio_task_run_job.rb +36 -0
- data/app/models/raif/admin/task_stat.rb +7 -0
- data/app/models/raif/agent.rb +98 -6
- data/app/models/raif/agents/native_tool_calling_agent.rb +179 -52
- data/app/models/raif/application_record.rb +18 -0
- data/app/models/raif/concerns/agent_inference_stats.rb +35 -0
- data/app/models/raif/concerns/has_prompt_templates.rb +88 -0
- data/app/models/raif/concerns/has_runtime_duration.rb +41 -0
- data/app/models/raif/concerns/json_schema_definition.rb +54 -6
- data/app/models/raif/concerns/llm_prompt_caching.rb +20 -0
- data/app/models/raif/concerns/llms/anthropic/message_formatting.rb +34 -0
- data/app/models/raif/concerns/llms/anthropic/response_tool_calls.rb +24 -0
- data/app/models/raif/concerns/llms/anthropic/tool_formatting.rb +8 -0
- data/app/models/raif/concerns/llms/bedrock/message_formatting.rb +43 -0
- data/app/models/raif/concerns/llms/bedrock/response_tool_calls.rb +26 -0
- data/app/models/raif/concerns/llms/bedrock/tool_formatting.rb +8 -0
- data/app/models/raif/concerns/llms/google/message_formatting.rb +112 -0
- data/app/models/raif/concerns/llms/google/response_tool_calls.rb +32 -0
- data/app/models/raif/concerns/llms/google/tool_formatting.rb +76 -0
- data/app/models/raif/concerns/llms/message_formatting.rb +41 -5
- data/app/models/raif/concerns/llms/open_ai/json_schema_validation.rb +3 -3
- data/app/models/raif/concerns/llms/open_ai_completions/message_formatting.rb +22 -0
- data/app/models/raif/concerns/llms/open_ai_completions/response_tool_calls.rb +22 -0
- data/app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb +8 -0
- data/app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb +17 -0
- data/app/models/raif/concerns/llms/open_ai_responses/response_tool_calls.rb +26 -0
- data/app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb +8 -0
- data/app/models/raif/concerns/provider_managed_tool_calls.rb +162 -0
- data/app/models/raif/concerns/run_with.rb +127 -0
- data/app/models/raif/conversation.rb +112 -8
- data/app/models/raif/conversation_entry.rb +38 -4
- data/app/models/raif/embedding_model.rb +2 -1
- data/app/models/raif/embedding_models/bedrock.rb +10 -1
- data/app/models/raif/embedding_models/google.rb +37 -0
- data/app/models/raif/embedding_models/open_ai.rb +1 -1
- data/app/models/raif/evals/llm_judge.rb +70 -0
- data/{lib → app/models}/raif/evals/llm_judges/binary.rb +41 -3
- data/{lib → app/models}/raif/evals/llm_judges/comparative.rb +41 -3
- data/{lib → app/models}/raif/evals/llm_judges/scored.rb +39 -1
- data/{lib → app/models}/raif/evals/llm_judges/summarization.rb +40 -2
- data/app/models/raif/llm.rb +104 -4
- data/app/models/raif/llms/anthropic.rb +32 -22
- data/app/models/raif/llms/bedrock.rb +64 -24
- data/app/models/raif/llms/google.rb +166 -0
- data/app/models/raif/llms/open_ai_base.rb +23 -5
- data/app/models/raif/llms/open_ai_completions.rb +14 -12
- data/app/models/raif/llms/open_ai_responses.rb +14 -17
- data/app/models/raif/llms/open_router.rb +16 -15
- data/app/models/raif/model_completion.rb +103 -1
- data/app/models/raif/model_tool.rb +55 -5
- data/app/models/raif/model_tool_invocation.rb +68 -6
- data/app/models/raif/model_tools/agent_final_answer.rb +2 -7
- data/app/models/raif/model_tools/provider_managed/code_execution.rb +4 -0
- data/app/models/raif/model_tools/provider_managed/image_generation.rb +4 -0
- data/app/models/raif/model_tools/provider_managed/web_search.rb +4 -0
- data/app/models/raif/prompt_studio_batch_run.rb +155 -0
- data/app/models/raif/prompt_studio_batch_run_item.rb +220 -0
- data/app/models/raif/streaming_responses/bedrock.rb +60 -1
- data/app/models/raif/streaming_responses/google.rb +71 -0
- data/app/models/raif/task.rb +85 -18
- data/app/models/raif/user_tool_invocation.rb +19 -0
- data/app/views/layouts/raif/admin.html.erb +43 -2
- data/app/views/raif/admin/agents/_agent.html.erb +9 -0
- data/app/views/raif/admin/agents/_conversation_message.html.erb +28 -6
- data/app/views/raif/admin/agents/index.html.erb +50 -0
- data/app/views/raif/admin/agents/show.html.erb +50 -1
- data/app/views/raif/admin/configs/show.html.erb +117 -0
- data/app/views/raif/admin/conversations/_conversation_entry.html.erb +29 -34
- data/app/views/raif/admin/conversations/show.html.erb +2 -0
- data/app/views/raif/admin/llms/index.html.erb +110 -0
- data/app/views/raif/admin/model_completions/_model_completion.html.erb +10 -5
- data/app/views/raif/admin/model_completions/index.html.erb +40 -1
- data/app/views/raif/admin/model_completions/show.html.erb +256 -84
- data/app/views/raif/admin/model_tool_invocations/index.html.erb +22 -1
- data/app/views/raif/admin/model_tool_invocations/show.html.erb +18 -0
- data/app/views/raif/admin/model_tools/_list.html.erb +16 -0
- data/app/views/raif/admin/model_tools/_model_tool.html.erb +36 -0
- data/app/views/raif/admin/prompt_studio/agents/index.html.erb +56 -0
- data/app/views/raif/admin/prompt_studio/agents/show.html.erb +57 -0
- data/app/views/raif/admin/prompt_studio/batch_runs/_batch_run_item.html.erb +54 -0
- data/app/views/raif/admin/prompt_studio/batch_runs/_judge_config_fields.html.erb +76 -0
- data/app/views/raif/admin/prompt_studio/batch_runs/_judge_detail_modal.html.erb +27 -0
- data/app/views/raif/admin/prompt_studio/batch_runs/_modal.html.erb +35 -0
- data/app/views/raif/admin/prompt_studio/batch_runs/_progress.html.erb +78 -0
- data/app/views/raif/admin/prompt_studio/batch_runs/show.html.erb +49 -0
- data/app/views/raif/admin/prompt_studio/conversations/index.html.erb +48 -0
- data/app/views/raif/admin/prompt_studio/conversations/show.html.erb +36 -0
- data/app/views/raif/admin/prompt_studio/shared/_nav_tabs.html.erb +17 -0
- data/app/views/raif/admin/prompt_studio/shared/_prompt_comparison.html.erb +87 -0
- data/app/views/raif/admin/prompt_studio/shared/_type_filter.html.erb +54 -0
- data/app/views/raif/admin/prompt_studio/tasks/_task_result.html.erb +145 -0
- data/app/views/raif/admin/prompt_studio/tasks/_task_row.html.erb +12 -0
- data/app/views/raif/admin/prompt_studio/tasks/_task_type_filter.html.erb +58 -0
- data/app/views/raif/admin/prompt_studio/tasks/_tasks_table.html.erb +22 -0
- data/app/views/raif/admin/prompt_studio/tasks/index.html.erb +35 -0
- data/app/views/raif/admin/prompt_studio/tasks/show.html.erb +19 -0
- data/app/views/raif/admin/stats/_stats_tile.html.erb +34 -0
- data/app/views/raif/admin/stats/index.html.erb +71 -88
- data/app/views/raif/admin/stats/model_tool_invocations/index.html.erb +43 -0
- data/app/views/raif/admin/stats/tasks/index.html.erb +20 -6
- data/app/views/raif/admin/tasks/_task.html.erb +1 -0
- data/app/views/raif/admin/tasks/index.html.erb +23 -6
- data/app/views/raif/admin/tasks/show.html.erb +56 -3
- data/app/views/raif/conversation_entries/_form.html.erb +3 -0
- data/app/views/raif/conversation_entries/_message.html.erb +10 -6
- data/app/views/raif/conversations/_conversation.html.erb +10 -0
- data/app/views/raif/conversations/_entry_processed.turbo_stream.erb +12 -0
- data/app/views/raif/conversations/index.html.erb +23 -0
- data/config/importmap.rb +8 -0
- data/config/locales/admin.en.yml +161 -1
- data/config/locales/en.yml +67 -4
- data/config/routes.rb +10 -0
- data/db/migrate/20250904194456_add_generating_entry_response_to_raif_conversations.rb +7 -0
- data/db/migrate/20250911125234_add_source_to_raif_tasks.rb +7 -0
- data/db/migrate/20251020005853_add_source_to_raif_agents.rb +7 -0
- data/db/migrate/20251020011346_rename_task_run_args_to_run_with.rb +7 -0
- data/db/migrate/20251020011405_add_run_with_to_raif_agents.rb +13 -0
- data/db/migrate/20251024160119_add_llm_messages_max_length_to_raif_conversations.rb +14 -0
- data/db/migrate/20251124185033_add_provider_tool_call_id_to_raif_model_tool_invocations.rb +7 -0
- data/db/migrate/20251128202941_add_tool_choice_to_raif_model_completions.rb +7 -0
- data/db/migrate/20260118144846_add_source_to_raif_conversations.rb +7 -0
- data/db/migrate/20260119000000_add_failure_tracking_to_raif_model_completions.rb +10 -0
- data/db/migrate/20260119000001_add_completed_at_to_raif_model_completions.rb +8 -0
- data/db/migrate/20260119000002_add_started_at_to_raif_model_completions.rb +8 -0
- data/db/migrate/20260307000000_add_prompt_studio_run_to_raif_tasks.rb +7 -0
- data/db/migrate/20260308000000_create_raif_prompt_studio_batch_runs.rb +27 -0
- data/db/migrate/20260308000001_create_raif_prompt_studio_batch_run_items.rb +24 -0
- data/db/migrate/20260407000000_add_cache_token_columns_to_raif_model_completions.rb +8 -0
- data/lib/generators/raif/agent/agent_generator.rb +18 -0
- data/lib/generators/raif/agent/templates/agent.rb.tt +7 -5
- data/lib/generators/raif/agent/templates/application_agent.rb.tt +1 -1
- data/lib/generators/raif/agent/templates/system_prompt.erb.tt +3 -0
- data/lib/generators/raif/conversation/conversation_generator.rb +19 -1
- data/lib/generators/raif/conversation/templates/conversation.rb.tt +6 -0
- data/lib/generators/raif/conversation/templates/system_prompt.erb.tt +4 -0
- data/lib/generators/raif/install/templates/initializer.rb +117 -8
- data/lib/generators/raif/task/task_generator.rb +18 -0
- data/lib/generators/raif/task/templates/prompt.erb.tt +4 -0
- data/lib/generators/raif/task/templates/task.rb.tt +10 -9
- data/lib/raif/configuration.rb +47 -2
- data/lib/raif/embedding_model_registry.rb +8 -0
- data/lib/raif/engine.rb +24 -1
- data/lib/raif/errors/blank_response_error.rb +8 -0
- data/lib/raif/errors/instance_dependent_schema_error.rb +8 -0
- data/lib/raif/errors/prompt_template_error.rb +15 -0
- data/lib/raif/errors/streaming_error.rb +6 -3
- data/lib/raif/errors.rb +3 -0
- data/lib/raif/evals/run.rb +1 -0
- data/lib/raif/evals.rb +0 -6
- data/lib/raif/json_schema_builder.rb +14 -0
- data/lib/raif/llm_registry.rb +433 -42
- data/lib/raif/messages.rb +180 -0
- data/lib/raif/prompt_studio_comparison_builder.rb +138 -0
- data/lib/raif/token_estimator.rb +28 -0
- data/lib/raif/version.rb +1 -1
- data/lib/raif.rb +11 -0
- data/lib/tasks/annotate_rb.rake +10 -0
- data/spec/support/rspec_helpers.rb +15 -9
- data/spec/support/test_task.rb +9 -0
- data/spec/support/test_template_task.rb +41 -0
- metadata +108 -15
- data/app/models/raif/agents/re_act_agent.rb +0 -127
- data/app/models/raif/agents/re_act_step.rb +0 -32
- data/app/models/raif/concerns/task_run_args.rb +0 -62
- data/lib/raif/evals/llm_judge.rb +0 -32
- /data/{lib → app/models}/raif/evals/scoring_rubric.rb +0 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class AddRunWithToRaifAgents < ActiveRecord::Migration[7.1]
|
|
4
|
+
def change
|
|
5
|
+
json_column_type = if connection.adapter_name.downcase.include?("postgresql")
|
|
6
|
+
:jsonb
|
|
7
|
+
else
|
|
8
|
+
:json
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
add_column :raif_agents, :run_with, json_column_type
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class AddLlmMessagesMaxLengthToRaifConversations < ActiveRecord::Migration[7.1]
|
|
4
|
+
def change
|
|
5
|
+
add_column :raif_conversations, :llm_messages_max_length, :integer
|
|
6
|
+
|
|
7
|
+
reversible do |dir|
|
|
8
|
+
dir.up do
|
|
9
|
+
# Set default value for existing conversations
|
|
10
|
+
execute "UPDATE raif_conversations SET llm_messages_max_length = 50 WHERE llm_messages_max_length IS NULL"
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class AddFailureTrackingToRaifModelCompletions < ActiveRecord::Migration[7.1]
|
|
4
|
+
def change
|
|
5
|
+
add_column :raif_model_completions, :failed_at, :datetime
|
|
6
|
+
add_column :raif_model_completions, :failure_error, :string
|
|
7
|
+
add_column :raif_model_completions, :failure_reason, :text
|
|
8
|
+
add_index :raif_model_completions, :failed_at
|
|
9
|
+
end
|
|
10
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreateRaifPromptStudioBatchRuns < ActiveRecord::Migration[7.1]
|
|
4
|
+
def change
|
|
5
|
+
json_column_type = if connection.adapter_name.downcase.include?("postgresql")
|
|
6
|
+
:jsonb
|
|
7
|
+
else
|
|
8
|
+
:json
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
create_table :raif_prompt_studio_batch_runs do |t|
|
|
12
|
+
t.string :task_type, null: false
|
|
13
|
+
t.string :llm_model_key, null: false
|
|
14
|
+
t.string :judge_type
|
|
15
|
+
t.string :judge_llm_model_key
|
|
16
|
+
t.send json_column_type, :judge_config, null: false
|
|
17
|
+
t.integer :total_count, default: 0, null: false
|
|
18
|
+
t.integer :completed_count, default: 0, null: false
|
|
19
|
+
t.integer :failed_count, default: 0, null: false
|
|
20
|
+
t.datetime :started_at
|
|
21
|
+
t.datetime :completed_at
|
|
22
|
+
t.datetime :failed_at
|
|
23
|
+
|
|
24
|
+
t.timestamps
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class CreateRaifPromptStudioBatchRunItems < ActiveRecord::Migration[7.1]
|
|
4
|
+
def change
|
|
5
|
+
json_column_type = if connection.adapter_name.downcase.include?("postgresql")
|
|
6
|
+
:jsonb
|
|
7
|
+
else
|
|
8
|
+
:json
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
create_table :raif_prompt_studio_batch_run_items do |t|
|
|
12
|
+
t.references :batch_run, null: false, foreign_key: { to_table: :raif_prompt_studio_batch_runs }
|
|
13
|
+
t.references :source_task, null: false, foreign_key: { to_table: :raif_tasks }
|
|
14
|
+
t.references :result_task, foreign_key: { to_table: :raif_tasks }
|
|
15
|
+
t.references :judge_task, foreign_key: { to_table: :raif_tasks }
|
|
16
|
+
t.string :status, default: "pending", null: false
|
|
17
|
+
t.column :metadata, json_column_type
|
|
18
|
+
|
|
19
|
+
t.timestamps
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
add_index :raif_prompt_studio_batch_run_items, :status
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class AddCacheTokenColumnsToRaifModelCompletions < ActiveRecord::Migration[7.1]
|
|
4
|
+
def change
|
|
5
|
+
add_column :raif_model_completions, :cache_read_input_tokens, :integer
|
|
6
|
+
add_column :raif_model_completions, :cache_creation_input_tokens, :integer
|
|
7
|
+
end
|
|
8
|
+
end
|
|
@@ -13,6 +13,11 @@ module Raif
|
|
|
13
13
|
default: false,
|
|
14
14
|
desc: "Skip generating the corresponding eval set"
|
|
15
15
|
|
|
16
|
+
class_option :skip_prompt_template,
|
|
17
|
+
type: :boolean,
|
|
18
|
+
default: false,
|
|
19
|
+
desc: "Skip generating the system prompt template file"
|
|
20
|
+
|
|
16
21
|
def create_application_agent
|
|
17
22
|
template "application_agent.rb.tt", "app/models/raif/application_agent.rb" unless File.exist?("app/models/raif/application_agent.rb")
|
|
18
23
|
end
|
|
@@ -21,6 +26,12 @@ module Raif
|
|
|
21
26
|
template "agent.rb.tt", File.join("app/models/raif/agents", class_path, "#{file_name}.rb")
|
|
22
27
|
end
|
|
23
28
|
|
|
29
|
+
def create_system_prompt_template
|
|
30
|
+
return if options[:skip_prompt_template]
|
|
31
|
+
|
|
32
|
+
template "system_prompt.erb.tt", system_prompt_template_file_path
|
|
33
|
+
end
|
|
34
|
+
|
|
24
35
|
def create_directory
|
|
25
36
|
empty_directory "app/models/raif/agents" unless File.directory?("app/models/raif/agents")
|
|
26
37
|
end
|
|
@@ -33,6 +44,9 @@ module Raif
|
|
|
33
44
|
|
|
34
45
|
def show_instructions
|
|
35
46
|
say "\nAgent created!"
|
|
47
|
+
unless options[:skip_prompt_template]
|
|
48
|
+
say " System prompt template: #{system_prompt_template_file_path}"
|
|
49
|
+
end
|
|
36
50
|
say ""
|
|
37
51
|
end
|
|
38
52
|
|
|
@@ -42,6 +56,10 @@ module Raif
|
|
|
42
56
|
File.join("raif_evals", "eval_sets", "agents", class_path, "#{file_name}_eval_set.rb")
|
|
43
57
|
end
|
|
44
58
|
|
|
59
|
+
def system_prompt_template_file_path
|
|
60
|
+
File.join("app/views/raif/agents", class_path, "#{file_name}.system_prompt.erb")
|
|
61
|
+
end
|
|
62
|
+
|
|
45
63
|
end
|
|
46
64
|
end
|
|
47
65
|
end
|
|
@@ -9,11 +9,13 @@
|
|
|
9
9
|
# ]
|
|
10
10
|
# end
|
|
11
11
|
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
# System prompt is defined in app/views/raif/agents/<%= class_path.any? ? class_path.join("/") + "/" : "" %><%= file_name %>.system_prompt.erb
|
|
13
|
+
# Alternatively, you can change your agent's superclass to an existing agent type
|
|
14
|
+
# (like Raif::Agents::NativeToolCallingAgent) to utilize an existing system prompt,
|
|
15
|
+
# or override build_system_prompt directly:
|
|
16
|
+
# def build_system_prompt
|
|
17
|
+
# "Your system prompt here"
|
|
18
|
+
# end
|
|
17
19
|
|
|
18
20
|
# Each iteration of the agent loop will generate a new Raif::ModelCompletion record and
|
|
19
21
|
# then call this method with it as an argument.
|
|
@@ -19,6 +19,11 @@ module Raif
|
|
|
19
19
|
default: false,
|
|
20
20
|
desc: "Skip generating the corresponding eval set"
|
|
21
21
|
|
|
22
|
+
class_option :skip_prompt_template,
|
|
23
|
+
type: :boolean,
|
|
24
|
+
default: false,
|
|
25
|
+
desc: "Skip generating the system prompt template file"
|
|
26
|
+
|
|
22
27
|
def create_application_conversation
|
|
23
28
|
template "application_conversation.rb.tt",
|
|
24
29
|
"app/models/raif/application_conversation.rb" unless File.exist?("app/models/raif/application_conversation.rb")
|
|
@@ -28,6 +33,12 @@ module Raif
|
|
|
28
33
|
template "conversation.rb.tt", File.join("app/models/raif/conversations", class_path, "#{file_name}.rb")
|
|
29
34
|
end
|
|
30
35
|
|
|
36
|
+
def create_system_prompt_template
|
|
37
|
+
return if options[:skip_prompt_template]
|
|
38
|
+
|
|
39
|
+
template "system_prompt.erb.tt", system_prompt_template_file_path
|
|
40
|
+
end
|
|
41
|
+
|
|
31
42
|
def create_directory
|
|
32
43
|
empty_directory "app/models/raif/conversations" unless File.directory?("app/models/raif/conversations")
|
|
33
44
|
end
|
|
@@ -41,7 +52,10 @@ module Raif
|
|
|
41
52
|
def success_message
|
|
42
53
|
say_status :success, "Conversation type created successfully", :green
|
|
43
54
|
say "\nYou can now implement your conversation type in:"
|
|
44
|
-
say " app/models/raif/conversations/#{file_name}.rb
|
|
55
|
+
say " app/models/raif/conversations/#{file_name}.rb"
|
|
56
|
+
unless options[:skip_prompt_template]
|
|
57
|
+
say " System prompt template: #{system_prompt_template_file_path}"
|
|
58
|
+
end
|
|
45
59
|
say "\nDon't forget to add it to the config.conversation_types in your Raif configuration"
|
|
46
60
|
say "For example: config.conversation_types += ['Raif::Conversations::#{class_name}']\n\n"
|
|
47
61
|
end
|
|
@@ -51,6 +65,10 @@ module Raif
|
|
|
51
65
|
def eval_set_file_path
|
|
52
66
|
File.join("raif_evals", "eval_sets", "conversations", class_path, "#{file_name}_eval_set.rb")
|
|
53
67
|
end
|
|
68
|
+
|
|
69
|
+
def system_prompt_template_file_path
|
|
70
|
+
File.join("app/views/raif/conversations", class_path, "#{file_name}.system_prompt.erb")
|
|
71
|
+
end
|
|
54
72
|
end
|
|
55
73
|
end
|
|
56
74
|
end
|
|
@@ -8,6 +8,12 @@
|
|
|
8
8
|
# uncomment this callback to populate the available_model_tools attribute with your desired model tools.
|
|
9
9
|
# before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
|
|
10
10
|
|
|
11
|
+
# Use this callback to run custom logic before prompting the model for an entry response.
|
|
12
|
+
# The block receives the entry as an argument, and `self` is the conversation instance.
|
|
13
|
+
# before_prompt_model_for_entry_response do |entry|
|
|
14
|
+
# # Custom logic here
|
|
15
|
+
# end
|
|
16
|
+
|
|
11
17
|
# Override the methods below to customize the system prompt for this conversation type.
|
|
12
18
|
# def system_prompt_intro
|
|
13
19
|
# Raif.config.conversation_system_prompt_intro
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
<%% # Define the system prompt for Raif::Conversations::<%= class_name %> here. %>
|
|
2
|
+
<%% # All instance methods are available in this template. %>
|
|
3
|
+
<%% # You can also use Rails view helpers and render partials. %>
|
|
4
|
+
<%% # If this file is removed, the default system prompt from Raif.config.conversation_system_prompt_intro will be used. %>
|
|
@@ -10,6 +10,22 @@ Raif.configure do |config|
|
|
|
10
10
|
# Whether OpenAI embedding models are enabled.
|
|
11
11
|
# config.open_ai_embedding_models_enabled = ENV["OPENAI_API_KEY"].present?
|
|
12
12
|
|
|
13
|
+
# The base URL for OpenAI API requests.
|
|
14
|
+
# Set this if you want to use the OpenAI adapter with a different provider (e.g. for using Azure instead of OpenAI)
|
|
15
|
+
# config.open_ai_base_url = "https://api.openai.com/v1"
|
|
16
|
+
|
|
17
|
+
# The base URL for OpenAI embedding API requests.
|
|
18
|
+
# Set this if you want to use a different provider for embeddings (e.g. Ollama, vLLM, or other OpenAI-compatible APIs)
|
|
19
|
+
# config.open_ai_embedding_base_url = "https://api.openai.com/v1"
|
|
20
|
+
|
|
21
|
+
# When set, this will be included as an api-version parameter in any OpenAI API requests (e.g. for using Azure instead of OpenAI)
|
|
22
|
+
# config.open_ai_api_version = nil
|
|
23
|
+
|
|
24
|
+
# The authentication header style for OpenAI API requests. Defaults to :bearer
|
|
25
|
+
# Use :bearer for standard OpenAI API (Authorization: Bearer <token>)
|
|
26
|
+
# Use :api_key for Azure OpenAI API (api-key: <token>)
|
|
27
|
+
# config.open_ai_auth_header_style = :bearer
|
|
28
|
+
|
|
13
29
|
# Your Anthropic API key. Defaults to ENV["ANTHROPIC_API_KEY"]
|
|
14
30
|
# config.anthropic_api_key = ENV["ANTHROPIC_API_KEY"]
|
|
15
31
|
|
|
@@ -40,36 +56,119 @@ Raif.configure do |config|
|
|
|
40
56
|
# The site URL to include in OpenRouter API requests headers. Optional.
|
|
41
57
|
# config.open_router_site_url = "https://myapp.com"
|
|
42
58
|
|
|
59
|
+
# Your Google AI API key. Defaults to ENV["GOOGLE_AI_API_KEY"].presence || ENV["GOOGLE_API_KEY"]
|
|
60
|
+
# config.google_api_key = ENV["GOOGLE_AI_API_KEY"].presence || ENV["GOOGLE_API_KEY"]
|
|
61
|
+
|
|
62
|
+
# Whether Google models are enabled.
|
|
63
|
+
# config.google_models_enabled = ENV["GOOGLE_AI_API_KEY"].present? || ENV["GOOGLE_API_KEY"].present?
|
|
64
|
+
|
|
65
|
+
# Whether Google embedding models are enabled. Defaults to false
|
|
66
|
+
# config.google_embedding_models_enabled = false
|
|
67
|
+
|
|
43
68
|
# The default LLM model to use. Defaults to "open_ai_gpt_4o"
|
|
44
69
|
# Available keys:
|
|
70
|
+
# open_ai_gpt_5_4
|
|
71
|
+
# open_ai_gpt_5_2
|
|
72
|
+
# open_ai_gpt_5_1
|
|
73
|
+
# open_ai_gpt_5
|
|
74
|
+
# open_ai_gpt_5_mini
|
|
75
|
+
# open_ai_gpt_5_nano
|
|
45
76
|
# open_ai_gpt_4_1
|
|
46
77
|
# open_ai_gpt_4_1_mini
|
|
47
78
|
# open_ai_gpt_4_1_nano
|
|
48
|
-
# open_ai_gpt_4o_mini
|
|
49
79
|
# open_ai_gpt_4o
|
|
80
|
+
# open_ai_gpt_4o_mini
|
|
50
81
|
# open_ai_gpt_3_5_turbo
|
|
82
|
+
# open_ai_o4_mini
|
|
83
|
+
# open_ai_o3
|
|
84
|
+
# open_ai_o3_mini
|
|
85
|
+
# open_ai_o1
|
|
86
|
+
# open_ai_o1_mini
|
|
87
|
+
# open_ai_responses_gpt_5_4
|
|
88
|
+
# open_ai_responses_gpt_5_4_pro
|
|
89
|
+
# open_ai_responses_gpt_5_2
|
|
90
|
+
# open_ai_responses_gpt_5_2_pro
|
|
91
|
+
# open_ai_responses_gpt_5_1
|
|
92
|
+
# open_ai_responses_gpt_5
|
|
93
|
+
# open_ai_responses_gpt_5_pro
|
|
94
|
+
# open_ai_responses_gpt_5_mini
|
|
95
|
+
# open_ai_responses_gpt_5_nano
|
|
51
96
|
# open_ai_responses_gpt_4_1
|
|
52
97
|
# open_ai_responses_gpt_4_1_mini
|
|
53
98
|
# open_ai_responses_gpt_4_1_nano
|
|
54
|
-
# open_ai_responses_gpt_4o_mini
|
|
55
99
|
# open_ai_responses_gpt_4o
|
|
56
|
-
#
|
|
100
|
+
# open_ai_responses_gpt_4o_mini
|
|
101
|
+
# open_ai_responses_gpt_3_5_turbo
|
|
102
|
+
# open_ai_responses_o4_mini
|
|
103
|
+
# open_ai_responses_o3
|
|
104
|
+
# open_ai_responses_o3_pro
|
|
105
|
+
# open_ai_responses_o3_mini
|
|
106
|
+
# open_ai_responses_o1
|
|
107
|
+
# open_ai_responses_o1_pro
|
|
108
|
+
# anthropic_claude_4_7_opus
|
|
109
|
+
# anthropic_claude_4_6_opus
|
|
110
|
+
# anthropic_claude_4_6_sonnet
|
|
111
|
+
# anthropic_claude_4_5_opus
|
|
112
|
+
# anthropic_claude_4_5_sonnet
|
|
113
|
+
# anthropic_claude_4_5_haiku
|
|
114
|
+
# anthropic_claude_4_1_opus
|
|
115
|
+
# anthropic_claude_4_opus
|
|
116
|
+
# anthropic_claude_4_sonnet
|
|
57
117
|
# anthropic_claude_3_7_sonnet
|
|
58
118
|
# anthropic_claude_3_5_sonnet
|
|
59
119
|
# anthropic_claude_3_5_haiku
|
|
60
120
|
# anthropic_claude_3_opus
|
|
61
|
-
#
|
|
121
|
+
# bedrock_claude_4_7_opus
|
|
122
|
+
# bedrock_claude_4_6_opus
|
|
123
|
+
# bedrock_claude_4_6_sonnet
|
|
124
|
+
# bedrock_claude_4_5_opus
|
|
125
|
+
# bedrock_claude_4_5_sonnet
|
|
126
|
+
# bedrock_claude_4_5_haiku
|
|
127
|
+
# bedrock_claude_4_1_opus
|
|
128
|
+
# bedrock_claude_4_opus
|
|
129
|
+
# bedrock_claude_4_sonnet
|
|
62
130
|
# bedrock_claude_3_7_sonnet
|
|
131
|
+
# bedrock_claude_3_5_sonnet
|
|
63
132
|
# bedrock_claude_3_5_haiku
|
|
64
133
|
# bedrock_claude_3_opus
|
|
65
134
|
# bedrock_amazon_nova_micro
|
|
66
135
|
# bedrock_amazon_nova_lite
|
|
67
136
|
# bedrock_amazon_nova_pro
|
|
137
|
+
# bedrock_deepseek_v3_2
|
|
138
|
+
# bedrock_deepseek_r1
|
|
139
|
+
# bedrock_gpt_oss_120b
|
|
140
|
+
# bedrock_gpt_oss_20b
|
|
68
141
|
# open_router_claude_3_7_sonnet
|
|
69
|
-
# open_router_llama_3_3_70b_instruct
|
|
70
|
-
# open_router_llama_3_1_8b_instruct
|
|
71
|
-
# open_router_gemini_2_0_flash
|
|
72
142
|
# open_router_deepseek_chat_v3
|
|
143
|
+
# open_router_deepseek_v3_1
|
|
144
|
+
# open_router_deepseek_v3_2
|
|
145
|
+
# open_router_gemini_2_0_flash
|
|
146
|
+
# open_router_gemini_2_5_flash
|
|
147
|
+
# open_router_gemini_2_5_pro
|
|
148
|
+
# open_router_gemini_3_pro_preview
|
|
149
|
+
# open_router_grok_4
|
|
150
|
+
# open_router_grok_4_1_fast
|
|
151
|
+
# open_router_kimi_k2_thinking
|
|
152
|
+
# open_router_kimi_k2_5
|
|
153
|
+
# open_router_llama_3_1_8b_instruct
|
|
154
|
+
# open_router_llama_3_3_70b_instruct
|
|
155
|
+
# open_router_llama_4_maverick
|
|
156
|
+
# open_router_llama_4_scout
|
|
157
|
+
# open_router_minimax_m2
|
|
158
|
+
# open_router_minimax_m2_1
|
|
159
|
+
# open_router_minimax_m2_5
|
|
160
|
+
# open_router_mistral_large_3_2512
|
|
161
|
+
# open_router_mistral_small_3_2_24b
|
|
162
|
+
# open_router_gemini_3_1_pro_preview
|
|
163
|
+
# open_router_gemini_3_1_flash_lite_preview
|
|
164
|
+
# open_router_open_ai_gpt_oss_120b
|
|
165
|
+
# open_router_open_ai_gpt_oss_20b
|
|
166
|
+
# google_gemini_3_1_pro
|
|
167
|
+
# google_gemini_3_1_flash_lite
|
|
168
|
+
# google_gemini_3_0_pro
|
|
169
|
+
# google_gemini_3_0_flash
|
|
170
|
+
# google_gemini_2_5_pro
|
|
171
|
+
# google_gemini_2_5_flash
|
|
73
172
|
#
|
|
74
173
|
# config.default_llm_model_key = "open_ai_gpt_4o"
|
|
75
174
|
|
|
@@ -80,6 +179,7 @@ Raif.configure do |config|
|
|
|
80
179
|
# open_ai_text_embedding_3_large
|
|
81
180
|
# open_ai_text_embedding_ada_002
|
|
82
181
|
# bedrock_titan_embed_text_v2
|
|
182
|
+
# google_gemini_embedding_2
|
|
83
183
|
#
|
|
84
184
|
# config.default_embedding_model_key = "open_ai_text_embedding_3_small"
|
|
85
185
|
|
|
@@ -118,10 +218,14 @@ Raif.configure do |config|
|
|
|
118
218
|
# If you want to use a custom controller that inherits from Raif::ConversationEntriesController, you can set it here.
|
|
119
219
|
# config.conversation_entries_controller = "Raif::ConversationEntriesController"
|
|
120
220
|
|
|
221
|
+
# The default maximum number of conversation entries to include in LLM messages. Defaults to 50.
|
|
222
|
+
# Set to nil to include all entries. Each conversation can override this with its own llm_messages_max_length attribute.
|
|
223
|
+
# config.conversation_llm_messages_max_length_default = 50
|
|
224
|
+
|
|
121
225
|
# The method to call to get the current user. Defaults to :current_user
|
|
122
226
|
# config.current_user_method = :current_user
|
|
123
227
|
|
|
124
|
-
# The agent types that are available. Defaults to Set.new(["Raif::Agents::
|
|
228
|
+
# The agent types that are available. Defaults to Set.new(["Raif::Agents::NativeToolCallingAgent"])
|
|
125
229
|
# If you want to use custom agent types that inherits from Raif::Agent, you can add them here.
|
|
126
230
|
# config.agent_types += ["MyAgent"]
|
|
127
231
|
|
|
@@ -138,6 +242,11 @@ Raif.configure do |config|
|
|
|
138
242
|
# Use this to globally disable requests to LLM APIs.
|
|
139
243
|
# config.llm_api_requests_enabled = true
|
|
140
244
|
|
|
245
|
+
# Timeout settings for LLM API requests (in seconds). All default to nil (use Faraday defaults).
|
|
246
|
+
# config.request_open_timeout = nil # Time to wait for a connection to be opened
|
|
247
|
+
# config.request_read_timeout = nil # Time to wait for data to be read
|
|
248
|
+
# config.request_write_timeout = nil # Time to wait for data to be written
|
|
249
|
+
|
|
141
250
|
# The default LLM model to use for LLM-as-judge evaluations.
|
|
142
251
|
# If not set, falls back to the default_llm_model_key.
|
|
143
252
|
# config.evals_default_llm_judge_model_key = ENV["RAIF_EVALS_DEFAULT_LLM_JUDGE_MODEL_KEY"].presence
|
|
@@ -17,6 +17,11 @@ module Raif
|
|
|
17
17
|
default: false,
|
|
18
18
|
desc: "Skip generating the corresponding eval set"
|
|
19
19
|
|
|
20
|
+
class_option :skip_prompt_template,
|
|
21
|
+
type: :boolean,
|
|
22
|
+
default: false,
|
|
23
|
+
desc: "Skip generating the prompt template file"
|
|
24
|
+
|
|
20
25
|
def create_application_task
|
|
21
26
|
template "application_task.rb.tt", "app/models/raif/application_task.rb" unless File.exist?("app/models/raif/application_task.rb")
|
|
22
27
|
end
|
|
@@ -25,6 +30,12 @@ module Raif
|
|
|
25
30
|
template "task.rb.tt", File.join("app/models/raif/tasks", class_path, "#{file_name}.rb")
|
|
26
31
|
end
|
|
27
32
|
|
|
33
|
+
def create_prompt_template
|
|
34
|
+
return if options[:skip_prompt_template]
|
|
35
|
+
|
|
36
|
+
template "prompt.erb.tt", prompt_template_file_path
|
|
37
|
+
end
|
|
38
|
+
|
|
28
39
|
def create_eval_set
|
|
29
40
|
return if options[:skip_eval_set]
|
|
30
41
|
|
|
@@ -33,6 +44,9 @@ module Raif
|
|
|
33
44
|
|
|
34
45
|
def show_instructions
|
|
35
46
|
say "\nTask created!"
|
|
47
|
+
unless options[:skip_prompt_template]
|
|
48
|
+
say " Prompt template: #{prompt_template_file_path}"
|
|
49
|
+
end
|
|
36
50
|
say ""
|
|
37
51
|
end
|
|
38
52
|
|
|
@@ -42,6 +56,10 @@ module Raif
|
|
|
42
56
|
File.join("raif_evals", "eval_sets", "tasks", class_path, "#{file_name}_eval_set.rb")
|
|
43
57
|
end
|
|
44
58
|
|
|
59
|
+
def prompt_template_file_path
|
|
60
|
+
File.join("app/views/raif/tasks", class_path, "#{file_name}.prompt.erb")
|
|
61
|
+
end
|
|
62
|
+
|
|
45
63
|
end
|
|
46
64
|
end
|
|
47
65
|
end
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
<%% # Implement the LLM prompt for Raif::Tasks::<%= class_name %> here. %>
|
|
2
|
+
<%% # All instance methods and run_with attributes are available in this template. %>
|
|
3
|
+
<%% # You can also use Rails view helpers (content_tag, strip_tags, truncate, etc.) %>
|
|
4
|
+
<%% # and render partials with: <%%= render partial: "raif/shared/my_partial" %> %>
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# Define any attributes that are needed for the task.
|
|
18
18
|
# You can then pass them when running the task and they will be available in build_prompt:
|
|
19
19
|
# Raif::Tasks::<%= class_name %>.run(your_attribute: "some value")
|
|
20
|
-
#
|
|
20
|
+
# run_with :your_attribute
|
|
21
21
|
<%- if options[:response_format] == "json" -%>
|
|
22
22
|
|
|
23
23
|
# Define a JSON schema that the model's response should adhere to
|
|
@@ -44,14 +44,15 @@
|
|
|
44
44
|
end
|
|
45
45
|
<%- end -%>
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
47
|
+
# Prompt is defined in app/views/raif/tasks/<%= class_path.any? ? class_path.join("/") + "/" : "" %><%= file_name %>.prompt.erb
|
|
48
|
+
# You can also define a system prompt template at:
|
|
49
|
+
# app/views/raif/tasks/<%= class_path.any? ? class_path.join("/") + "/" : "" %><%= file_name %>.system_prompt.erb
|
|
50
|
+
#
|
|
51
|
+
# Alternatively, override build_prompt and/or build_system_prompt methods directly:
|
|
52
|
+
# def build_prompt
|
|
53
|
+
# "Your prompt here"
|
|
54
|
+
# end
|
|
55
|
+
#
|
|
55
56
|
# def build_system_prompt
|
|
56
57
|
# super + "\nAdditional system instructions..."
|
|
57
58
|
# end
|