raif 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +8 -7
  3. data/app/assets/builds/raif.css +4 -1
  4. data/app/assets/builds/raif_admin.css +52 -2
  5. data/app/assets/builds/raif_admin_sprockets.js +2709 -0
  6. data/app/assets/javascript/raif/admin/copy_to_clipboard_controller.js +132 -0
  7. data/app/assets/javascript/raif/admin/cost_estimate_controller.js +80 -0
  8. data/app/assets/javascript/raif/admin/judge_config_controller.js +23 -0
  9. data/app/assets/javascript/raif/admin/select_all_checkboxes_controller.js +33 -0
  10. data/app/assets/javascript/raif/admin/sortable_table_controller.js +51 -0
  11. data/app/assets/javascript/raif/admin/table_search_controller.js +15 -0
  12. data/app/assets/javascript/raif/admin/tom_select_controller.js +33 -0
  13. data/app/assets/javascript/raif/controllers/conversations_controller.js +1 -1
  14. data/app/assets/javascript/raif_admin.js +23 -0
  15. data/app/assets/javascript/raif_admin_sprockets.js +24 -0
  16. data/app/assets/stylesheets/raif/admin/conversation.scss +16 -0
  17. data/app/assets/stylesheets/raif/conversations.scss +3 -0
  18. data/app/assets/stylesheets/raif.scss +2 -1
  19. data/app/assets/stylesheets/raif_admin.scss +50 -1
  20. data/app/controllers/raif/admin/agents_controller.rb +27 -1
  21. data/app/controllers/raif/admin/application_controller.rb +16 -0
  22. data/app/controllers/raif/admin/configs_controller.rb +95 -0
  23. data/app/controllers/raif/admin/llms_controller.rb +27 -0
  24. data/app/controllers/raif/admin/model_completions_controller.rb +24 -1
  25. data/app/controllers/raif/admin/model_tool_invocations_controller.rb +7 -1
  26. data/app/controllers/raif/admin/prompt_studio/agents_controller.rb +25 -0
  27. data/app/controllers/raif/admin/prompt_studio/base_controller.rb +32 -0
  28. data/app/controllers/raif/admin/prompt_studio/batch_runs_controller.rb +102 -0
  29. data/app/controllers/raif/admin/prompt_studio/conversations_controller.rb +25 -0
  30. data/app/controllers/raif/admin/prompt_studio/tasks_controller.rb +64 -0
  31. data/app/controllers/raif/admin/stats/model_tool_invocations_controller.rb +21 -0
  32. data/app/controllers/raif/admin/stats/tasks_controller.rb +15 -6
  33. data/app/controllers/raif/admin/stats_controller.rb +32 -3
  34. data/app/controllers/raif/admin/tasks_controller.rb +5 -0
  35. data/app/controllers/raif/conversation_entries_controller.rb +1 -0
  36. data/app/controllers/raif/conversations_controller.rb +10 -2
  37. data/app/helpers/raif/application_helper.rb +40 -0
  38. data/app/jobs/raif/conversation_entry_job.rb +8 -6
  39. data/app/jobs/raif/prompt_studio_batch_run_item_job.rb +11 -0
  40. data/app/jobs/raif/prompt_studio_batch_run_job.rb +15 -0
  41. data/app/jobs/raif/prompt_studio_task_run_job.rb +36 -0
  42. data/app/models/raif/admin/task_stat.rb +7 -0
  43. data/app/models/raif/agent.rb +98 -6
  44. data/app/models/raif/agents/native_tool_calling_agent.rb +179 -52
  45. data/app/models/raif/application_record.rb +18 -0
  46. data/app/models/raif/concerns/agent_inference_stats.rb +35 -0
  47. data/app/models/raif/concerns/has_prompt_templates.rb +88 -0
  48. data/app/models/raif/concerns/has_runtime_duration.rb +41 -0
  49. data/app/models/raif/concerns/json_schema_definition.rb +54 -6
  50. data/app/models/raif/concerns/llm_prompt_caching.rb +20 -0
  51. data/app/models/raif/concerns/llms/anthropic/message_formatting.rb +34 -0
  52. data/app/models/raif/concerns/llms/anthropic/response_tool_calls.rb +24 -0
  53. data/app/models/raif/concerns/llms/anthropic/tool_formatting.rb +8 -0
  54. data/app/models/raif/concerns/llms/bedrock/message_formatting.rb +43 -0
  55. data/app/models/raif/concerns/llms/bedrock/response_tool_calls.rb +26 -0
  56. data/app/models/raif/concerns/llms/bedrock/tool_formatting.rb +8 -0
  57. data/app/models/raif/concerns/llms/google/message_formatting.rb +112 -0
  58. data/app/models/raif/concerns/llms/google/response_tool_calls.rb +32 -0
  59. data/app/models/raif/concerns/llms/google/tool_formatting.rb +76 -0
  60. data/app/models/raif/concerns/llms/message_formatting.rb +41 -5
  61. data/app/models/raif/concerns/llms/open_ai/json_schema_validation.rb +3 -3
  62. data/app/models/raif/concerns/llms/open_ai_completions/message_formatting.rb +22 -0
  63. data/app/models/raif/concerns/llms/open_ai_completions/response_tool_calls.rb +22 -0
  64. data/app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb +8 -0
  65. data/app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb +17 -0
  66. data/app/models/raif/concerns/llms/open_ai_responses/response_tool_calls.rb +26 -0
  67. data/app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb +8 -0
  68. data/app/models/raif/concerns/provider_managed_tool_calls.rb +162 -0
  69. data/app/models/raif/concerns/run_with.rb +127 -0
  70. data/app/models/raif/conversation.rb +112 -8
  71. data/app/models/raif/conversation_entry.rb +38 -4
  72. data/app/models/raif/embedding_model.rb +2 -1
  73. data/app/models/raif/embedding_models/bedrock.rb +10 -1
  74. data/app/models/raif/embedding_models/google.rb +37 -0
  75. data/app/models/raif/embedding_models/open_ai.rb +1 -1
  76. data/app/models/raif/evals/llm_judge.rb +70 -0
  77. data/{lib → app/models}/raif/evals/llm_judges/binary.rb +41 -3
  78. data/{lib → app/models}/raif/evals/llm_judges/comparative.rb +41 -3
  79. data/{lib → app/models}/raif/evals/llm_judges/scored.rb +39 -1
  80. data/{lib → app/models}/raif/evals/llm_judges/summarization.rb +40 -2
  81. data/app/models/raif/llm.rb +104 -4
  82. data/app/models/raif/llms/anthropic.rb +32 -22
  83. data/app/models/raif/llms/bedrock.rb +64 -24
  84. data/app/models/raif/llms/google.rb +166 -0
  85. data/app/models/raif/llms/open_ai_base.rb +23 -5
  86. data/app/models/raif/llms/open_ai_completions.rb +14 -12
  87. data/app/models/raif/llms/open_ai_responses.rb +14 -17
  88. data/app/models/raif/llms/open_router.rb +16 -15
  89. data/app/models/raif/model_completion.rb +103 -1
  90. data/app/models/raif/model_tool.rb +55 -5
  91. data/app/models/raif/model_tool_invocation.rb +68 -6
  92. data/app/models/raif/model_tools/agent_final_answer.rb +2 -7
  93. data/app/models/raif/model_tools/provider_managed/code_execution.rb +4 -0
  94. data/app/models/raif/model_tools/provider_managed/image_generation.rb +4 -0
  95. data/app/models/raif/model_tools/provider_managed/web_search.rb +4 -0
  96. data/app/models/raif/prompt_studio_batch_run.rb +155 -0
  97. data/app/models/raif/prompt_studio_batch_run_item.rb +220 -0
  98. data/app/models/raif/streaming_responses/bedrock.rb +60 -1
  99. data/app/models/raif/streaming_responses/google.rb +71 -0
  100. data/app/models/raif/task.rb +85 -18
  101. data/app/models/raif/user_tool_invocation.rb +19 -0
  102. data/app/views/layouts/raif/admin.html.erb +43 -2
  103. data/app/views/raif/admin/agents/_agent.html.erb +9 -0
  104. data/app/views/raif/admin/agents/_conversation_message.html.erb +28 -6
  105. data/app/views/raif/admin/agents/index.html.erb +50 -0
  106. data/app/views/raif/admin/agents/show.html.erb +50 -1
  107. data/app/views/raif/admin/configs/show.html.erb +117 -0
  108. data/app/views/raif/admin/conversations/_conversation_entry.html.erb +29 -34
  109. data/app/views/raif/admin/conversations/show.html.erb +2 -0
  110. data/app/views/raif/admin/llms/index.html.erb +110 -0
  111. data/app/views/raif/admin/model_completions/_model_completion.html.erb +10 -5
  112. data/app/views/raif/admin/model_completions/index.html.erb +40 -1
  113. data/app/views/raif/admin/model_completions/show.html.erb +256 -84
  114. data/app/views/raif/admin/model_tool_invocations/index.html.erb +22 -1
  115. data/app/views/raif/admin/model_tool_invocations/show.html.erb +18 -0
  116. data/app/views/raif/admin/model_tools/_list.html.erb +16 -0
  117. data/app/views/raif/admin/model_tools/_model_tool.html.erb +36 -0
  118. data/app/views/raif/admin/prompt_studio/agents/index.html.erb +56 -0
  119. data/app/views/raif/admin/prompt_studio/agents/show.html.erb +57 -0
  120. data/app/views/raif/admin/prompt_studio/batch_runs/_batch_run_item.html.erb +54 -0
  121. data/app/views/raif/admin/prompt_studio/batch_runs/_judge_config_fields.html.erb +76 -0
  122. data/app/views/raif/admin/prompt_studio/batch_runs/_judge_detail_modal.html.erb +27 -0
  123. data/app/views/raif/admin/prompt_studio/batch_runs/_modal.html.erb +35 -0
  124. data/app/views/raif/admin/prompt_studio/batch_runs/_progress.html.erb +78 -0
  125. data/app/views/raif/admin/prompt_studio/batch_runs/show.html.erb +49 -0
  126. data/app/views/raif/admin/prompt_studio/conversations/index.html.erb +48 -0
  127. data/app/views/raif/admin/prompt_studio/conversations/show.html.erb +36 -0
  128. data/app/views/raif/admin/prompt_studio/shared/_nav_tabs.html.erb +17 -0
  129. data/app/views/raif/admin/prompt_studio/shared/_prompt_comparison.html.erb +87 -0
  130. data/app/views/raif/admin/prompt_studio/shared/_type_filter.html.erb +54 -0
  131. data/app/views/raif/admin/prompt_studio/tasks/_task_result.html.erb +145 -0
  132. data/app/views/raif/admin/prompt_studio/tasks/_task_row.html.erb +12 -0
  133. data/app/views/raif/admin/prompt_studio/tasks/_task_type_filter.html.erb +58 -0
  134. data/app/views/raif/admin/prompt_studio/tasks/_tasks_table.html.erb +22 -0
  135. data/app/views/raif/admin/prompt_studio/tasks/index.html.erb +35 -0
  136. data/app/views/raif/admin/prompt_studio/tasks/show.html.erb +19 -0
  137. data/app/views/raif/admin/stats/_stats_tile.html.erb +34 -0
  138. data/app/views/raif/admin/stats/index.html.erb +71 -88
  139. data/app/views/raif/admin/stats/model_tool_invocations/index.html.erb +43 -0
  140. data/app/views/raif/admin/stats/tasks/index.html.erb +20 -6
  141. data/app/views/raif/admin/tasks/_task.html.erb +1 -0
  142. data/app/views/raif/admin/tasks/index.html.erb +23 -6
  143. data/app/views/raif/admin/tasks/show.html.erb +56 -3
  144. data/app/views/raif/conversation_entries/_form.html.erb +3 -0
  145. data/app/views/raif/conversation_entries/_message.html.erb +10 -6
  146. data/app/views/raif/conversations/_conversation.html.erb +10 -0
  147. data/app/views/raif/conversations/_entry_processed.turbo_stream.erb +12 -0
  148. data/app/views/raif/conversations/index.html.erb +23 -0
  149. data/config/importmap.rb +8 -0
  150. data/config/locales/admin.en.yml +161 -1
  151. data/config/locales/en.yml +67 -4
  152. data/config/routes.rb +10 -0
  153. data/db/migrate/20250904194456_add_generating_entry_response_to_raif_conversations.rb +7 -0
  154. data/db/migrate/20250911125234_add_source_to_raif_tasks.rb +7 -0
  155. data/db/migrate/20251020005853_add_source_to_raif_agents.rb +7 -0
  156. data/db/migrate/20251020011346_rename_task_run_args_to_run_with.rb +7 -0
  157. data/db/migrate/20251020011405_add_run_with_to_raif_agents.rb +13 -0
  158. data/db/migrate/20251024160119_add_llm_messages_max_length_to_raif_conversations.rb +14 -0
  159. data/db/migrate/20251124185033_add_provider_tool_call_id_to_raif_model_tool_invocations.rb +7 -0
  160. data/db/migrate/20251128202941_add_tool_choice_to_raif_model_completions.rb +7 -0
  161. data/db/migrate/20260118144846_add_source_to_raif_conversations.rb +7 -0
  162. data/db/migrate/20260119000000_add_failure_tracking_to_raif_model_completions.rb +10 -0
  163. data/db/migrate/20260119000001_add_completed_at_to_raif_model_completions.rb +8 -0
  164. data/db/migrate/20260119000002_add_started_at_to_raif_model_completions.rb +8 -0
  165. data/db/migrate/20260307000000_add_prompt_studio_run_to_raif_tasks.rb +7 -0
  166. data/db/migrate/20260308000000_create_raif_prompt_studio_batch_runs.rb +27 -0
  167. data/db/migrate/20260308000001_create_raif_prompt_studio_batch_run_items.rb +24 -0
  168. data/db/migrate/20260407000000_add_cache_token_columns_to_raif_model_completions.rb +8 -0
  169. data/lib/generators/raif/agent/agent_generator.rb +18 -0
  170. data/lib/generators/raif/agent/templates/agent.rb.tt +7 -5
  171. data/lib/generators/raif/agent/templates/application_agent.rb.tt +1 -1
  172. data/lib/generators/raif/agent/templates/system_prompt.erb.tt +3 -0
  173. data/lib/generators/raif/conversation/conversation_generator.rb +19 -1
  174. data/lib/generators/raif/conversation/templates/conversation.rb.tt +6 -0
  175. data/lib/generators/raif/conversation/templates/system_prompt.erb.tt +4 -0
  176. data/lib/generators/raif/install/templates/initializer.rb +117 -8
  177. data/lib/generators/raif/task/task_generator.rb +18 -0
  178. data/lib/generators/raif/task/templates/prompt.erb.tt +4 -0
  179. data/lib/generators/raif/task/templates/task.rb.tt +10 -9
  180. data/lib/raif/configuration.rb +47 -2
  181. data/lib/raif/embedding_model_registry.rb +8 -0
  182. data/lib/raif/engine.rb +24 -1
  183. data/lib/raif/errors/blank_response_error.rb +8 -0
  184. data/lib/raif/errors/instance_dependent_schema_error.rb +8 -0
  185. data/lib/raif/errors/prompt_template_error.rb +15 -0
  186. data/lib/raif/errors/streaming_error.rb +6 -3
  187. data/lib/raif/errors.rb +3 -0
  188. data/lib/raif/evals/run.rb +1 -0
  189. data/lib/raif/evals.rb +0 -6
  190. data/lib/raif/json_schema_builder.rb +14 -0
  191. data/lib/raif/llm_registry.rb +433 -42
  192. data/lib/raif/messages.rb +180 -0
  193. data/lib/raif/prompt_studio_comparison_builder.rb +138 -0
  194. data/lib/raif/token_estimator.rb +28 -0
  195. data/lib/raif/version.rb +1 -1
  196. data/lib/raif.rb +11 -0
  197. data/lib/tasks/annotate_rb.rake +10 -0
  198. data/spec/support/rspec_helpers.rb +15 -9
  199. data/spec/support/test_task.rb +9 -0
  200. data/spec/support/test_template_task.rb +41 -0
  201. metadata +108 -15
  202. data/app/models/raif/agents/re_act_agent.rb +0 -127
  203. data/app/models/raif/agents/re_act_step.rb +0 -32
  204. data/app/models/raif/concerns/task_run_args.rb +0 -62
  205. data/lib/raif/evals/llm_judge.rb +0 -32
  206. /data/{lib → app/models}/raif/evals/scoring_rubric.rb +0 -0
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddSourceToRaifAgents < ActiveRecord::Migration[7.1]
4
+ def change
5
+ add_reference :raif_agents, :source, polymorphic: true, index: true
6
+ end
7
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ class RenameTaskRunArgsToRunWith < ActiveRecord::Migration[7.1]
4
+ def change
5
+ rename_column :raif_tasks, :task_run_args, :run_with
6
+ end
7
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddRunWithToRaifAgents < ActiveRecord::Migration[7.1]
4
+ def change
5
+ json_column_type = if connection.adapter_name.downcase.include?("postgresql")
6
+ :jsonb
7
+ else
8
+ :json
9
+ end
10
+
11
+ add_column :raif_agents, :run_with, json_column_type
12
+ end
13
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddLlmMessagesMaxLengthToRaifConversations < ActiveRecord::Migration[7.1]
4
+ def change
5
+ add_column :raif_conversations, :llm_messages_max_length, :integer
6
+
7
+ reversible do |dir|
8
+ dir.up do
9
+ # Set default value for existing conversations
10
+ execute "UPDATE raif_conversations SET llm_messages_max_length = 50 WHERE llm_messages_max_length IS NULL"
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddProviderToolCallIdToRaifModelToolInvocations < ActiveRecord::Migration[7.2]
4
+ def change
5
+ add_column :raif_model_tool_invocations, :provider_tool_call_id, :string
6
+ end
7
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddToolChoiceToRaifModelCompletions < ActiveRecord::Migration[7.2]
4
+ def change
5
+ add_column :raif_model_completions, :tool_choice, :string
6
+ end
7
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddSourceToRaifConversations < ActiveRecord::Migration[7.1]
4
+ def change
5
+ add_reference :raif_conversations, :source, polymorphic: true, index: true
6
+ end
7
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddFailureTrackingToRaifModelCompletions < ActiveRecord::Migration[7.1]
4
+ def change
5
+ add_column :raif_model_completions, :failed_at, :datetime
6
+ add_column :raif_model_completions, :failure_error, :string
7
+ add_column :raif_model_completions, :failure_reason, :text
8
+ add_index :raif_model_completions, :failed_at
9
+ end
10
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddCompletedAtToRaifModelCompletions < ActiveRecord::Migration[7.1]
4
+ def change
5
+ add_column :raif_model_completions, :completed_at, :datetime
6
+ add_index :raif_model_completions, :completed_at
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddStartedAtToRaifModelCompletions < ActiveRecord::Migration[7.1]
4
+ def change
5
+ add_column :raif_model_completions, :started_at, :datetime
6
+ add_index :raif_model_completions, :started_at
7
+ end
8
+ end
@@ -0,0 +1,7 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddPromptStudioRunToRaifTasks < ActiveRecord::Migration[7.1]
4
+ def change
5
+ add_column :raif_tasks, :prompt_studio_run, :boolean, default: false, null: false
6
+ end
7
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ class CreateRaifPromptStudioBatchRuns < ActiveRecord::Migration[7.1]
4
+ def change
5
+ json_column_type = if connection.adapter_name.downcase.include?("postgresql")
6
+ :jsonb
7
+ else
8
+ :json
9
+ end
10
+
11
+ create_table :raif_prompt_studio_batch_runs do |t|
12
+ t.string :task_type, null: false
13
+ t.string :llm_model_key, null: false
14
+ t.string :judge_type
15
+ t.string :judge_llm_model_key
16
+ t.send json_column_type, :judge_config, null: false
17
+ t.integer :total_count, default: 0, null: false
18
+ t.integer :completed_count, default: 0, null: false
19
+ t.integer :failed_count, default: 0, null: false
20
+ t.datetime :started_at
21
+ t.datetime :completed_at
22
+ t.datetime :failed_at
23
+
24
+ t.timestamps
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,24 @@
1
+ # frozen_string_literal: true
2
+
3
+ class CreateRaifPromptStudioBatchRunItems < ActiveRecord::Migration[7.1]
4
+ def change
5
+ json_column_type = if connection.adapter_name.downcase.include?("postgresql")
6
+ :jsonb
7
+ else
8
+ :json
9
+ end
10
+
11
+ create_table :raif_prompt_studio_batch_run_items do |t|
12
+ t.references :batch_run, null: false, foreign_key: { to_table: :raif_prompt_studio_batch_runs }
13
+ t.references :source_task, null: false, foreign_key: { to_table: :raif_tasks }
14
+ t.references :result_task, foreign_key: { to_table: :raif_tasks }
15
+ t.references :judge_task, foreign_key: { to_table: :raif_tasks }
16
+ t.string :status, default: "pending", null: false
17
+ t.column :metadata, json_column_type
18
+
19
+ t.timestamps
20
+ end
21
+
22
+ add_index :raif_prompt_studio_batch_run_items, :status
23
+ end
24
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddCacheTokenColumnsToRaifModelCompletions < ActiveRecord::Migration[7.1]
4
+ def change
5
+ add_column :raif_model_completions, :cache_read_input_tokens, :integer
6
+ add_column :raif_model_completions, :cache_creation_input_tokens, :integer
7
+ end
8
+ end
@@ -13,6 +13,11 @@ module Raif
13
13
  default: false,
14
14
  desc: "Skip generating the corresponding eval set"
15
15
 
16
+ class_option :skip_prompt_template,
17
+ type: :boolean,
18
+ default: false,
19
+ desc: "Skip generating the system prompt template file"
20
+
16
21
  def create_application_agent
17
22
  template "application_agent.rb.tt", "app/models/raif/application_agent.rb" unless File.exist?("app/models/raif/application_agent.rb")
18
23
  end
@@ -21,6 +26,12 @@ module Raif
21
26
  template "agent.rb.tt", File.join("app/models/raif/agents", class_path, "#{file_name}.rb")
22
27
  end
23
28
 
29
+ def create_system_prompt_template
30
+ return if options[:skip_prompt_template]
31
+
32
+ template "system_prompt.erb.tt", system_prompt_template_file_path
33
+ end
34
+
24
35
  def create_directory
25
36
  empty_directory "app/models/raif/agents" unless File.directory?("app/models/raif/agents")
26
37
  end
@@ -33,6 +44,9 @@ module Raif
33
44
 
34
45
  def show_instructions
35
46
  say "\nAgent created!"
47
+ unless options[:skip_prompt_template]
48
+ say " System prompt template: #{system_prompt_template_file_path}"
49
+ end
36
50
  say ""
37
51
  end
38
52
 
@@ -42,6 +56,10 @@ module Raif
42
56
  File.join("raif_evals", "eval_sets", "agents", class_path, "#{file_name}_eval_set.rb")
43
57
  end
44
58
 
59
+ def system_prompt_template_file_path
60
+ File.join("app/views/raif/agents", class_path, "#{file_name}.system_prompt.erb")
61
+ end
62
+
45
63
  end
46
64
  end
47
65
  end
@@ -9,11 +9,13 @@
9
9
  # ]
10
10
  # end
11
11
 
12
- # Enter your agent's system prompt here. Alternatively, you can change your agent's superclass
13
- # to an existing agent types (like Raif::Agents::ReActAgent) to utilize an existing system prompt.
14
- def build_system_prompt
15
- # TODO: Implement your system prompt here
16
- end
12
+ # System prompt is defined in app/views/raif/agents/<%= class_path.any? ? class_path.join("/") + "/" : "" %><%= file_name %>.system_prompt.erb
13
+ # Alternatively, you can change your agent's superclass to an existing agent type
14
+ # (like Raif::Agents::NativeToolCallingAgent) to utilize an existing system prompt,
15
+ # or override build_system_prompt directly:
16
+ # def build_system_prompt
17
+ # "Your system prompt here"
18
+ # end
17
19
 
18
20
  # Each iteration of the agent loop will generate a new Raif::ModelCompletion record and
19
21
  # then call this method with it as an argument.
@@ -1,5 +1,5 @@
1
1
  module Raif
2
- class ApplicationAgent < Raif::Agent
2
+ class ApplicationAgent < Raif::Agents::NativeToolCallingAgent
3
3
  # Add any shared agent behavior here
4
4
  end
5
5
  end
@@ -0,0 +1,3 @@
1
+ <%% # Define the system prompt for Raif::Agents::<%= class_name %> here. %>
2
+ <%% # All instance methods and run_with attributes are available in this template. %>
3
+ <%% # You can also use Rails view helpers and render partials. %>
@@ -19,6 +19,11 @@ module Raif
19
19
  default: false,
20
20
  desc: "Skip generating the corresponding eval set"
21
21
 
22
+ class_option :skip_prompt_template,
23
+ type: :boolean,
24
+ default: false,
25
+ desc: "Skip generating the system prompt template file"
26
+
22
27
  def create_application_conversation
23
28
  template "application_conversation.rb.tt",
24
29
  "app/models/raif/application_conversation.rb" unless File.exist?("app/models/raif/application_conversation.rb")
@@ -28,6 +33,12 @@ module Raif
28
33
  template "conversation.rb.tt", File.join("app/models/raif/conversations", class_path, "#{file_name}.rb")
29
34
  end
30
35
 
36
+ def create_system_prompt_template
37
+ return if options[:skip_prompt_template]
38
+
39
+ template "system_prompt.erb.tt", system_prompt_template_file_path
40
+ end
41
+
31
42
  def create_directory
32
43
  empty_directory "app/models/raif/conversations" unless File.directory?("app/models/raif/conversations")
33
44
  end
@@ -41,7 +52,10 @@ module Raif
41
52
  def success_message
42
53
  say_status :success, "Conversation type created successfully", :green
43
54
  say "\nYou can now implement your conversation type in:"
44
- say " app/models/raif/conversations/#{file_name}.rb\n\n"
55
+ say " app/models/raif/conversations/#{file_name}.rb"
56
+ unless options[:skip_prompt_template]
57
+ say " System prompt template: #{system_prompt_template_file_path}"
58
+ end
45
59
  say "\nDon't forget to add it to the config.conversation_types in your Raif configuration"
46
60
  say "For example: config.conversation_types += ['Raif::Conversations::#{class_name}']\n\n"
47
61
  end
@@ -51,6 +65,10 @@ module Raif
51
65
  def eval_set_file_path
52
66
  File.join("raif_evals", "eval_sets", "conversations", class_path, "#{file_name}_eval_set.rb")
53
67
  end
68
+
69
+ def system_prompt_template_file_path
70
+ File.join("app/views/raif/conversations", class_path, "#{file_name}.system_prompt.erb")
71
+ end
54
72
  end
55
73
  end
56
74
  end
@@ -8,6 +8,12 @@
8
8
  # uncomment this callback to populate the available_model_tools attribute with your desired model tools.
9
9
  # before_create -> { self.available_model_tools = ["Raif::ModelTools::Example"] }
10
10
 
11
+ # Use this callback to run custom logic before prompting the model for an entry response.
12
+ # The block receives the entry as an argument, and `self` is the conversation instance.
13
+ # before_prompt_model_for_entry_response do |entry|
14
+ # # Custom logic here
15
+ # end
16
+
11
17
  # Override the methods below to customize the system prompt for this conversation type.
12
18
  # def system_prompt_intro
13
19
  # Raif.config.conversation_system_prompt_intro
@@ -0,0 +1,4 @@
1
+ <%% # Define the system prompt for Raif::Conversations::<%= class_name %> here. %>
2
+ <%% # All instance methods are available in this template. %>
3
+ <%% # You can also use Rails view helpers and render partials. %>
4
+ <%% # If this file is removed, the default system prompt from Raif.config.conversation_system_prompt_intro will be used. %>
@@ -10,6 +10,22 @@ Raif.configure do |config|
10
10
  # Whether OpenAI embedding models are enabled.
11
11
  # config.open_ai_embedding_models_enabled = ENV["OPENAI_API_KEY"].present?
12
12
 
13
+ # The base URL for OpenAI API requests.
14
+ # Set this if you want to use the OpenAI adapter with a different provider (e.g. for using Azure instead of OpenAI)
15
+ # config.open_ai_base_url = "https://api.openai.com/v1"
16
+
17
+ # The base URL for OpenAI embedding API requests.
18
+ # Set this if you want to use a different provider for embeddings (e.g. Ollama, vLLM, or other OpenAI-compatible APIs)
19
+ # config.open_ai_embedding_base_url = "https://api.openai.com/v1"
20
+
21
+ # When set, this will be included as an api-version parameter in any OpenAI API requests (e.g. for using Azure instead of OpenAI)
22
+ # config.open_ai_api_version = nil
23
+
24
+ # The authentication header style for OpenAI API requests. Defaults to :bearer
25
+ # Use :bearer for standard OpenAI API (Authorization: Bearer <token>)
26
+ # Use :api_key for Azure OpenAI API (api-key: <token>)
27
+ # config.open_ai_auth_header_style = :bearer
28
+
13
29
  # Your Anthropic API key. Defaults to ENV["ANTHROPIC_API_KEY"]
14
30
  # config.anthropic_api_key = ENV["ANTHROPIC_API_KEY"]
15
31
 
@@ -40,36 +56,119 @@ Raif.configure do |config|
40
56
  # The site URL to include in OpenRouter API requests headers. Optional.
41
57
  # config.open_router_site_url = "https://myapp.com"
42
58
 
59
+ # Your Google AI API key. Defaults to ENV["GOOGLE_AI_API_KEY"].presence || ENV["GOOGLE_API_KEY"]
60
+ # config.google_api_key = ENV["GOOGLE_AI_API_KEY"].presence || ENV["GOOGLE_API_KEY"]
61
+
62
+ # Whether Google models are enabled.
63
+ # config.google_models_enabled = ENV["GOOGLE_AI_API_KEY"].present? || ENV["GOOGLE_API_KEY"].present?
64
+
65
+ # Whether Google embedding models are enabled. Defaults to false
66
+ # config.google_embedding_models_enabled = false
67
+
43
68
  # The default LLM model to use. Defaults to "open_ai_gpt_4o"
44
69
  # Available keys:
70
+ # open_ai_gpt_5_4
71
+ # open_ai_gpt_5_2
72
+ # open_ai_gpt_5_1
73
+ # open_ai_gpt_5
74
+ # open_ai_gpt_5_mini
75
+ # open_ai_gpt_5_nano
45
76
  # open_ai_gpt_4_1
46
77
  # open_ai_gpt_4_1_mini
47
78
  # open_ai_gpt_4_1_nano
48
- # open_ai_gpt_4o_mini
49
79
  # open_ai_gpt_4o
80
+ # open_ai_gpt_4o_mini
50
81
  # open_ai_gpt_3_5_turbo
82
+ # open_ai_o4_mini
83
+ # open_ai_o3
84
+ # open_ai_o3_mini
85
+ # open_ai_o1
86
+ # open_ai_o1_mini
87
+ # open_ai_responses_gpt_5_4
88
+ # open_ai_responses_gpt_5_4_pro
89
+ # open_ai_responses_gpt_5_2
90
+ # open_ai_responses_gpt_5_2_pro
91
+ # open_ai_responses_gpt_5_1
92
+ # open_ai_responses_gpt_5
93
+ # open_ai_responses_gpt_5_pro
94
+ # open_ai_responses_gpt_5_mini
95
+ # open_ai_responses_gpt_5_nano
51
96
  # open_ai_responses_gpt_4_1
52
97
  # open_ai_responses_gpt_4_1_mini
53
98
  # open_ai_responses_gpt_4_1_nano
54
- # open_ai_responses_gpt_4o_mini
55
99
  # open_ai_responses_gpt_4o
56
- # open_ai_gpt_3_5_turbo
100
+ # open_ai_responses_gpt_4o_mini
101
+ # open_ai_responses_gpt_3_5_turbo
102
+ # open_ai_responses_o4_mini
103
+ # open_ai_responses_o3
104
+ # open_ai_responses_o3_pro
105
+ # open_ai_responses_o3_mini
106
+ # open_ai_responses_o1
107
+ # open_ai_responses_o1_pro
108
+ # anthropic_claude_4_7_opus
109
+ # anthropic_claude_4_6_opus
110
+ # anthropic_claude_4_6_sonnet
111
+ # anthropic_claude_4_5_opus
112
+ # anthropic_claude_4_5_sonnet
113
+ # anthropic_claude_4_5_haiku
114
+ # anthropic_claude_4_1_opus
115
+ # anthropic_claude_4_opus
116
+ # anthropic_claude_4_sonnet
57
117
  # anthropic_claude_3_7_sonnet
58
118
  # anthropic_claude_3_5_sonnet
59
119
  # anthropic_claude_3_5_haiku
60
120
  # anthropic_claude_3_opus
61
- # bedrock_claude_3_5_sonnet
121
+ # bedrock_claude_4_7_opus
122
+ # bedrock_claude_4_6_opus
123
+ # bedrock_claude_4_6_sonnet
124
+ # bedrock_claude_4_5_opus
125
+ # bedrock_claude_4_5_sonnet
126
+ # bedrock_claude_4_5_haiku
127
+ # bedrock_claude_4_1_opus
128
+ # bedrock_claude_4_opus
129
+ # bedrock_claude_4_sonnet
62
130
  # bedrock_claude_3_7_sonnet
131
+ # bedrock_claude_3_5_sonnet
63
132
  # bedrock_claude_3_5_haiku
64
133
  # bedrock_claude_3_opus
65
134
  # bedrock_amazon_nova_micro
66
135
  # bedrock_amazon_nova_lite
67
136
  # bedrock_amazon_nova_pro
137
+ # bedrock_deepseek_v3_2
138
+ # bedrock_deepseek_r1
139
+ # bedrock_gpt_oss_120b
140
+ # bedrock_gpt_oss_20b
68
141
  # open_router_claude_3_7_sonnet
69
- # open_router_llama_3_3_70b_instruct
70
- # open_router_llama_3_1_8b_instruct
71
- # open_router_gemini_2_0_flash
72
142
  # open_router_deepseek_chat_v3
143
+ # open_router_deepseek_v3_1
144
+ # open_router_deepseek_v3_2
145
+ # open_router_gemini_2_0_flash
146
+ # open_router_gemini_2_5_flash
147
+ # open_router_gemini_2_5_pro
148
+ # open_router_gemini_3_pro_preview
149
+ # open_router_grok_4
150
+ # open_router_grok_4_1_fast
151
+ # open_router_kimi_k2_thinking
152
+ # open_router_kimi_k2_5
153
+ # open_router_llama_3_1_8b_instruct
154
+ # open_router_llama_3_3_70b_instruct
155
+ # open_router_llama_4_maverick
156
+ # open_router_llama_4_scout
157
+ # open_router_minimax_m2
158
+ # open_router_minimax_m2_1
159
+ # open_router_minimax_m2_5
160
+ # open_router_mistral_large_3_2512
161
+ # open_router_mistral_small_3_2_24b
162
+ # open_router_gemini_3_1_pro_preview
163
+ # open_router_gemini_3_1_flash_lite_preview
164
+ # open_router_open_ai_gpt_oss_120b
165
+ # open_router_open_ai_gpt_oss_20b
166
+ # google_gemini_3_1_pro
167
+ # google_gemini_3_1_flash_lite
168
+ # google_gemini_3_0_pro
169
+ # google_gemini_3_0_flash
170
+ # google_gemini_2_5_pro
171
+ # google_gemini_2_5_flash
73
172
  #
74
173
  # config.default_llm_model_key = "open_ai_gpt_4o"
75
174
 
@@ -80,6 +179,7 @@ Raif.configure do |config|
80
179
  # open_ai_text_embedding_3_large
81
180
  # open_ai_text_embedding_ada_002
82
181
  # bedrock_titan_embed_text_v2
182
+ # google_gemini_embedding_2
83
183
  #
84
184
  # config.default_embedding_model_key = "open_ai_text_embedding_3_small"
85
185
 
@@ -118,10 +218,14 @@ Raif.configure do |config|
118
218
  # If you want to use a custom controller that inherits from Raif::ConversationEntriesController, you can set it here.
119
219
  # config.conversation_entries_controller = "Raif::ConversationEntriesController"
120
220
 
221
+ # The default maximum number of conversation entries to include in LLM messages. Defaults to 50.
222
+ # Set to nil to include all entries. Each conversation can override this with its own llm_messages_max_length attribute.
223
+ # config.conversation_llm_messages_max_length_default = 50
224
+
121
225
  # The method to call to get the current user. Defaults to :current_user
122
226
  # config.current_user_method = :current_user
123
227
 
124
- # The agent types that are available. Defaults to Set.new(["Raif::Agents::ReActAgent", "Raif::Agents::NativeToolCallingAgent"])
228
+ # The agent types that are available. Defaults to Set.new(["Raif::Agents::NativeToolCallingAgent"])
125
229
  # If you want to use custom agent types that inherits from Raif::Agent, you can add them here.
126
230
  # config.agent_types += ["MyAgent"]
127
231
 
@@ -138,6 +242,11 @@ Raif.configure do |config|
138
242
  # Use this to globally disable requests to LLM APIs.
139
243
  # config.llm_api_requests_enabled = true
140
244
 
245
+ # Timeout settings for LLM API requests (in seconds). All default to nil (use Faraday defaults).
246
+ # config.request_open_timeout = nil # Time to wait for a connection to be opened
247
+ # config.request_read_timeout = nil # Time to wait for data to be read
248
+ # config.request_write_timeout = nil # Time to wait for data to be written
249
+
141
250
  # The default LLM model to use for LLM-as-judge evaluations.
142
251
  # If not set, falls back to the default_llm_model_key.
143
252
  # config.evals_default_llm_judge_model_key = ENV["RAIF_EVALS_DEFAULT_LLM_JUDGE_MODEL_KEY"].presence
@@ -17,6 +17,11 @@ module Raif
17
17
  default: false,
18
18
  desc: "Skip generating the corresponding eval set"
19
19
 
20
+ class_option :skip_prompt_template,
21
+ type: :boolean,
22
+ default: false,
23
+ desc: "Skip generating the prompt template file"
24
+
20
25
  def create_application_task
21
26
  template "application_task.rb.tt", "app/models/raif/application_task.rb" unless File.exist?("app/models/raif/application_task.rb")
22
27
  end
@@ -25,6 +30,12 @@ module Raif
25
30
  template "task.rb.tt", File.join("app/models/raif/tasks", class_path, "#{file_name}.rb")
26
31
  end
27
32
 
33
+ def create_prompt_template
34
+ return if options[:skip_prompt_template]
35
+
36
+ template "prompt.erb.tt", prompt_template_file_path
37
+ end
38
+
28
39
  def create_eval_set
29
40
  return if options[:skip_eval_set]
30
41
 
@@ -33,6 +44,9 @@ module Raif
33
44
 
34
45
  def show_instructions
35
46
  say "\nTask created!"
47
+ unless options[:skip_prompt_template]
48
+ say " Prompt template: #{prompt_template_file_path}"
49
+ end
36
50
  say ""
37
51
  end
38
52
 
@@ -42,6 +56,10 @@ module Raif
42
56
  File.join("raif_evals", "eval_sets", "tasks", class_path, "#{file_name}_eval_set.rb")
43
57
  end
44
58
 
59
+ def prompt_template_file_path
60
+ File.join("app/views/raif/tasks", class_path, "#{file_name}.prompt.erb")
61
+ end
62
+
45
63
  end
46
64
  end
47
65
  end
@@ -0,0 +1,4 @@
1
+ <%% # Implement the LLM prompt for Raif::Tasks::<%= class_name %> here. %>
2
+ <%% # All instance methods and run_with attributes are available in this template. %>
3
+ <%% # You can also use Rails view helpers (content_tag, strip_tags, truncate, etc.) %>
4
+ <%% # and render partials with: <%%= render partial: "raif/shared/my_partial" %> %>
@@ -17,7 +17,7 @@
17
17
  # Define any attributes that are needed for the task.
18
18
  # You can then pass them when running the task and they will be available in build_prompt:
19
19
  # Raif::Tasks::<%= class_name %>.run(your_attribute: "some value")
20
- # task_run_arg :your_attribute
20
+ # run_with :your_attribute
21
21
  <%- if options[:response_format] == "json" -%>
22
22
 
23
23
  # Define a JSON schema that the model's response should adhere to
@@ -44,14 +44,15 @@
44
44
  end
45
45
  <%- end -%>
46
46
 
47
- def build_prompt
48
- # Implement the LLM prompt for this task.
49
- raise NotImplementedError, "Implement #build_prompt in #{self.class.name}"
50
- end
51
-
52
- # Optional: Override build_system_prompt if you need custom system instructions.
53
- # The default implementation, which you'll get if you call super, will use Raif.config.task_system_prompt_intro
54
- # and append the system_prompt_language_preference if the task's requested_language_key is set.
47
+ # Prompt is defined in app/views/raif/tasks/<%= class_path.any? ? class_path.join("/") + "/" : "" %><%= file_name %>.prompt.erb
48
+ # You can also define a system prompt template at:
49
+ # app/views/raif/tasks/<%= class_path.any? ? class_path.join("/") + "/" : "" %><%= file_name %>.system_prompt.erb
50
+ #
51
+ # Alternatively, override build_prompt and/or build_system_prompt methods directly:
52
+ # def build_prompt
53
+ # "Your prompt here"
54
+ # end
55
+ #
55
56
  # def build_system_prompt
56
57
  # super + "\nAdditional system instructions..."
57
58
  # end