raif 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +8 -7
  3. data/app/assets/builds/raif.css +4 -1
  4. data/app/assets/builds/raif_admin.css +52 -2
  5. data/app/assets/builds/raif_admin_sprockets.js +2709 -0
  6. data/app/assets/javascript/raif/admin/copy_to_clipboard_controller.js +132 -0
  7. data/app/assets/javascript/raif/admin/cost_estimate_controller.js +80 -0
  8. data/app/assets/javascript/raif/admin/judge_config_controller.js +23 -0
  9. data/app/assets/javascript/raif/admin/select_all_checkboxes_controller.js +33 -0
  10. data/app/assets/javascript/raif/admin/sortable_table_controller.js +51 -0
  11. data/app/assets/javascript/raif/admin/table_search_controller.js +15 -0
  12. data/app/assets/javascript/raif/admin/tom_select_controller.js +33 -0
  13. data/app/assets/javascript/raif/controllers/conversations_controller.js +1 -1
  14. data/app/assets/javascript/raif_admin.js +23 -0
  15. data/app/assets/javascript/raif_admin_sprockets.js +24 -0
  16. data/app/assets/stylesheets/raif/admin/conversation.scss +16 -0
  17. data/app/assets/stylesheets/raif/conversations.scss +3 -0
  18. data/app/assets/stylesheets/raif.scss +2 -1
  19. data/app/assets/stylesheets/raif_admin.scss +50 -1
  20. data/app/controllers/raif/admin/agents_controller.rb +27 -1
  21. data/app/controllers/raif/admin/application_controller.rb +16 -0
  22. data/app/controllers/raif/admin/configs_controller.rb +95 -0
  23. data/app/controllers/raif/admin/llms_controller.rb +27 -0
  24. data/app/controllers/raif/admin/model_completions_controller.rb +24 -1
  25. data/app/controllers/raif/admin/model_tool_invocations_controller.rb +7 -1
  26. data/app/controllers/raif/admin/prompt_studio/agents_controller.rb +25 -0
  27. data/app/controllers/raif/admin/prompt_studio/base_controller.rb +32 -0
  28. data/app/controllers/raif/admin/prompt_studio/batch_runs_controller.rb +102 -0
  29. data/app/controllers/raif/admin/prompt_studio/conversations_controller.rb +25 -0
  30. data/app/controllers/raif/admin/prompt_studio/tasks_controller.rb +64 -0
  31. data/app/controllers/raif/admin/stats/model_tool_invocations_controller.rb +21 -0
  32. data/app/controllers/raif/admin/stats/tasks_controller.rb +15 -6
  33. data/app/controllers/raif/admin/stats_controller.rb +32 -3
  34. data/app/controllers/raif/admin/tasks_controller.rb +5 -0
  35. data/app/controllers/raif/conversation_entries_controller.rb +1 -0
  36. data/app/controllers/raif/conversations_controller.rb +10 -2
  37. data/app/helpers/raif/application_helper.rb +40 -0
  38. data/app/jobs/raif/conversation_entry_job.rb +8 -6
  39. data/app/jobs/raif/prompt_studio_batch_run_item_job.rb +11 -0
  40. data/app/jobs/raif/prompt_studio_batch_run_job.rb +15 -0
  41. data/app/jobs/raif/prompt_studio_task_run_job.rb +36 -0
  42. data/app/models/raif/admin/task_stat.rb +7 -0
  43. data/app/models/raif/agent.rb +98 -6
  44. data/app/models/raif/agents/native_tool_calling_agent.rb +179 -52
  45. data/app/models/raif/application_record.rb +18 -0
  46. data/app/models/raif/concerns/agent_inference_stats.rb +35 -0
  47. data/app/models/raif/concerns/has_prompt_templates.rb +88 -0
  48. data/app/models/raif/concerns/has_runtime_duration.rb +41 -0
  49. data/app/models/raif/concerns/json_schema_definition.rb +54 -6
  50. data/app/models/raif/concerns/llm_prompt_caching.rb +20 -0
  51. data/app/models/raif/concerns/llms/anthropic/message_formatting.rb +34 -0
  52. data/app/models/raif/concerns/llms/anthropic/response_tool_calls.rb +24 -0
  53. data/app/models/raif/concerns/llms/anthropic/tool_formatting.rb +8 -0
  54. data/app/models/raif/concerns/llms/bedrock/message_formatting.rb +43 -0
  55. data/app/models/raif/concerns/llms/bedrock/response_tool_calls.rb +26 -0
  56. data/app/models/raif/concerns/llms/bedrock/tool_formatting.rb +8 -0
  57. data/app/models/raif/concerns/llms/google/message_formatting.rb +112 -0
  58. data/app/models/raif/concerns/llms/google/response_tool_calls.rb +32 -0
  59. data/app/models/raif/concerns/llms/google/tool_formatting.rb +76 -0
  60. data/app/models/raif/concerns/llms/message_formatting.rb +41 -5
  61. data/app/models/raif/concerns/llms/open_ai/json_schema_validation.rb +3 -3
  62. data/app/models/raif/concerns/llms/open_ai_completions/message_formatting.rb +22 -0
  63. data/app/models/raif/concerns/llms/open_ai_completions/response_tool_calls.rb +22 -0
  64. data/app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb +8 -0
  65. data/app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb +17 -0
  66. data/app/models/raif/concerns/llms/open_ai_responses/response_tool_calls.rb +26 -0
  67. data/app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb +8 -0
  68. data/app/models/raif/concerns/provider_managed_tool_calls.rb +162 -0
  69. data/app/models/raif/concerns/run_with.rb +127 -0
  70. data/app/models/raif/conversation.rb +112 -8
  71. data/app/models/raif/conversation_entry.rb +38 -4
  72. data/app/models/raif/embedding_model.rb +2 -1
  73. data/app/models/raif/embedding_models/bedrock.rb +10 -1
  74. data/app/models/raif/embedding_models/google.rb +37 -0
  75. data/app/models/raif/embedding_models/open_ai.rb +1 -1
  76. data/app/models/raif/evals/llm_judge.rb +70 -0
  77. data/{lib → app/models}/raif/evals/llm_judges/binary.rb +41 -3
  78. data/{lib → app/models}/raif/evals/llm_judges/comparative.rb +41 -3
  79. data/{lib → app/models}/raif/evals/llm_judges/scored.rb +39 -1
  80. data/{lib → app/models}/raif/evals/llm_judges/summarization.rb +40 -2
  81. data/app/models/raif/llm.rb +104 -4
  82. data/app/models/raif/llms/anthropic.rb +32 -22
  83. data/app/models/raif/llms/bedrock.rb +64 -24
  84. data/app/models/raif/llms/google.rb +166 -0
  85. data/app/models/raif/llms/open_ai_base.rb +23 -5
  86. data/app/models/raif/llms/open_ai_completions.rb +14 -12
  87. data/app/models/raif/llms/open_ai_responses.rb +14 -17
  88. data/app/models/raif/llms/open_router.rb +16 -15
  89. data/app/models/raif/model_completion.rb +103 -1
  90. data/app/models/raif/model_tool.rb +55 -5
  91. data/app/models/raif/model_tool_invocation.rb +68 -6
  92. data/app/models/raif/model_tools/agent_final_answer.rb +2 -7
  93. data/app/models/raif/model_tools/provider_managed/code_execution.rb +4 -0
  94. data/app/models/raif/model_tools/provider_managed/image_generation.rb +4 -0
  95. data/app/models/raif/model_tools/provider_managed/web_search.rb +4 -0
  96. data/app/models/raif/prompt_studio_batch_run.rb +155 -0
  97. data/app/models/raif/prompt_studio_batch_run_item.rb +220 -0
  98. data/app/models/raif/streaming_responses/bedrock.rb +60 -1
  99. data/app/models/raif/streaming_responses/google.rb +71 -0
  100. data/app/models/raif/task.rb +85 -18
  101. data/app/models/raif/user_tool_invocation.rb +19 -0
  102. data/app/views/layouts/raif/admin.html.erb +43 -2
  103. data/app/views/raif/admin/agents/_agent.html.erb +9 -0
  104. data/app/views/raif/admin/agents/_conversation_message.html.erb +28 -6
  105. data/app/views/raif/admin/agents/index.html.erb +50 -0
  106. data/app/views/raif/admin/agents/show.html.erb +50 -1
  107. data/app/views/raif/admin/configs/show.html.erb +117 -0
  108. data/app/views/raif/admin/conversations/_conversation_entry.html.erb +29 -34
  109. data/app/views/raif/admin/conversations/show.html.erb +2 -0
  110. data/app/views/raif/admin/llms/index.html.erb +110 -0
  111. data/app/views/raif/admin/model_completions/_model_completion.html.erb +10 -5
  112. data/app/views/raif/admin/model_completions/index.html.erb +40 -1
  113. data/app/views/raif/admin/model_completions/show.html.erb +256 -84
  114. data/app/views/raif/admin/model_tool_invocations/index.html.erb +22 -1
  115. data/app/views/raif/admin/model_tool_invocations/show.html.erb +18 -0
  116. data/app/views/raif/admin/model_tools/_list.html.erb +16 -0
  117. data/app/views/raif/admin/model_tools/_model_tool.html.erb +36 -0
  118. data/app/views/raif/admin/prompt_studio/agents/index.html.erb +56 -0
  119. data/app/views/raif/admin/prompt_studio/agents/show.html.erb +57 -0
  120. data/app/views/raif/admin/prompt_studio/batch_runs/_batch_run_item.html.erb +54 -0
  121. data/app/views/raif/admin/prompt_studio/batch_runs/_judge_config_fields.html.erb +76 -0
  122. data/app/views/raif/admin/prompt_studio/batch_runs/_judge_detail_modal.html.erb +27 -0
  123. data/app/views/raif/admin/prompt_studio/batch_runs/_modal.html.erb +35 -0
  124. data/app/views/raif/admin/prompt_studio/batch_runs/_progress.html.erb +78 -0
  125. data/app/views/raif/admin/prompt_studio/batch_runs/show.html.erb +49 -0
  126. data/app/views/raif/admin/prompt_studio/conversations/index.html.erb +48 -0
  127. data/app/views/raif/admin/prompt_studio/conversations/show.html.erb +36 -0
  128. data/app/views/raif/admin/prompt_studio/shared/_nav_tabs.html.erb +17 -0
  129. data/app/views/raif/admin/prompt_studio/shared/_prompt_comparison.html.erb +87 -0
  130. data/app/views/raif/admin/prompt_studio/shared/_type_filter.html.erb +54 -0
  131. data/app/views/raif/admin/prompt_studio/tasks/_task_result.html.erb +145 -0
  132. data/app/views/raif/admin/prompt_studio/tasks/_task_row.html.erb +12 -0
  133. data/app/views/raif/admin/prompt_studio/tasks/_task_type_filter.html.erb +58 -0
  134. data/app/views/raif/admin/prompt_studio/tasks/_tasks_table.html.erb +22 -0
  135. data/app/views/raif/admin/prompt_studio/tasks/index.html.erb +35 -0
  136. data/app/views/raif/admin/prompt_studio/tasks/show.html.erb +19 -0
  137. data/app/views/raif/admin/stats/_stats_tile.html.erb +34 -0
  138. data/app/views/raif/admin/stats/index.html.erb +71 -88
  139. data/app/views/raif/admin/stats/model_tool_invocations/index.html.erb +43 -0
  140. data/app/views/raif/admin/stats/tasks/index.html.erb +20 -6
  141. data/app/views/raif/admin/tasks/_task.html.erb +1 -0
  142. data/app/views/raif/admin/tasks/index.html.erb +23 -6
  143. data/app/views/raif/admin/tasks/show.html.erb +56 -3
  144. data/app/views/raif/conversation_entries/_form.html.erb +3 -0
  145. data/app/views/raif/conversation_entries/_message.html.erb +10 -6
  146. data/app/views/raif/conversations/_conversation.html.erb +10 -0
  147. data/app/views/raif/conversations/_entry_processed.turbo_stream.erb +12 -0
  148. data/app/views/raif/conversations/index.html.erb +23 -0
  149. data/config/importmap.rb +8 -0
  150. data/config/locales/admin.en.yml +161 -1
  151. data/config/locales/en.yml +67 -4
  152. data/config/routes.rb +10 -0
  153. data/db/migrate/20250904194456_add_generating_entry_response_to_raif_conversations.rb +7 -0
  154. data/db/migrate/20250911125234_add_source_to_raif_tasks.rb +7 -0
  155. data/db/migrate/20251020005853_add_source_to_raif_agents.rb +7 -0
  156. data/db/migrate/20251020011346_rename_task_run_args_to_run_with.rb +7 -0
  157. data/db/migrate/20251020011405_add_run_with_to_raif_agents.rb +13 -0
  158. data/db/migrate/20251024160119_add_llm_messages_max_length_to_raif_conversations.rb +14 -0
  159. data/db/migrate/20251124185033_add_provider_tool_call_id_to_raif_model_tool_invocations.rb +7 -0
  160. data/db/migrate/20251128202941_add_tool_choice_to_raif_model_completions.rb +7 -0
  161. data/db/migrate/20260118144846_add_source_to_raif_conversations.rb +7 -0
  162. data/db/migrate/20260119000000_add_failure_tracking_to_raif_model_completions.rb +10 -0
  163. data/db/migrate/20260119000001_add_completed_at_to_raif_model_completions.rb +8 -0
  164. data/db/migrate/20260119000002_add_started_at_to_raif_model_completions.rb +8 -0
  165. data/db/migrate/20260307000000_add_prompt_studio_run_to_raif_tasks.rb +7 -0
  166. data/db/migrate/20260308000000_create_raif_prompt_studio_batch_runs.rb +27 -0
  167. data/db/migrate/20260308000001_create_raif_prompt_studio_batch_run_items.rb +24 -0
  168. data/db/migrate/20260407000000_add_cache_token_columns_to_raif_model_completions.rb +8 -0
  169. data/lib/generators/raif/agent/agent_generator.rb +18 -0
  170. data/lib/generators/raif/agent/templates/agent.rb.tt +7 -5
  171. data/lib/generators/raif/agent/templates/application_agent.rb.tt +1 -1
  172. data/lib/generators/raif/agent/templates/system_prompt.erb.tt +3 -0
  173. data/lib/generators/raif/conversation/conversation_generator.rb +19 -1
  174. data/lib/generators/raif/conversation/templates/conversation.rb.tt +6 -0
  175. data/lib/generators/raif/conversation/templates/system_prompt.erb.tt +4 -0
  176. data/lib/generators/raif/install/templates/initializer.rb +117 -8
  177. data/lib/generators/raif/task/task_generator.rb +18 -0
  178. data/lib/generators/raif/task/templates/prompt.erb.tt +4 -0
  179. data/lib/generators/raif/task/templates/task.rb.tt +10 -9
  180. data/lib/raif/configuration.rb +47 -2
  181. data/lib/raif/embedding_model_registry.rb +8 -0
  182. data/lib/raif/engine.rb +24 -1
  183. data/lib/raif/errors/blank_response_error.rb +8 -0
  184. data/lib/raif/errors/instance_dependent_schema_error.rb +8 -0
  185. data/lib/raif/errors/prompt_template_error.rb +15 -0
  186. data/lib/raif/errors/streaming_error.rb +6 -3
  187. data/lib/raif/errors.rb +3 -0
  188. data/lib/raif/evals/run.rb +1 -0
  189. data/lib/raif/evals.rb +0 -6
  190. data/lib/raif/json_schema_builder.rb +14 -0
  191. data/lib/raif/llm_registry.rb +433 -42
  192. data/lib/raif/messages.rb +180 -0
  193. data/lib/raif/prompt_studio_comparison_builder.rb +138 -0
  194. data/lib/raif/token_estimator.rb +28 -0
  195. data/lib/raif/version.rb +1 -1
  196. data/lib/raif.rb +11 -0
  197. data/lib/tasks/annotate_rb.rake +10 -0
  198. data/spec/support/rspec_helpers.rb +15 -9
  199. data/spec/support/test_task.rb +9 -0
  200. data/spec/support/test_template_task.rb +41 -0
  201. metadata +108 -15
  202. data/app/models/raif/agents/re_act_agent.rb +0 -127
  203. data/app/models/raif/agents/re_act_step.rb +0 -32
  204. data/app/models/raif/concerns/task_run_args.rb +0 -62
  205. data/lib/raif/evals/llm_judge.rb +0 -32
  206. /data/{lib → app/models}/raif/evals/scoring_rubric.rb +0 -0
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Raif::StreamingResponses::Google
4
+
5
+ def initialize
6
+ @response_json = { "candidates" => [{ "content" => { "parts" => [] } }], "usageMetadata" => {} }
7
+ @finish_reason = nil
8
+ end
9
+
10
+ def process_streaming_event(event_type, event)
11
+ delta = nil
12
+
13
+ # Google streams complete candidate objects, so we need to extract the new text
14
+ candidates = event["candidates"]
15
+ if candidates.present?
16
+ candidate = candidates[0]
17
+
18
+ # Check for finish reason
19
+ @finish_reason = candidate["finishReason"] if candidate["finishReason"].present?
20
+
21
+ # Process content parts
22
+ parts = candidate.dig("content", "parts")
23
+ delta = process_content_parts(parts) if parts.present?
24
+ end
25
+
26
+ # Update usage metadata
27
+ usage_metadata = event["usageMetadata"]
28
+ @response_json["usageMetadata"] = usage_metadata if usage_metadata.present?
29
+
30
+ [delta, @finish_reason]
31
+ end
32
+
33
+ def current_response_json
34
+ @response_json
35
+ end
36
+
37
+ private
38
+
39
+ def process_content_parts(parts)
40
+ delta = nil
41
+
42
+ parts.each_with_index do |part, index|
43
+ if part.key?("text")
44
+ delta = part["text"]
45
+ accumulate_text_part(part, index)
46
+ else
47
+ # For non-text parts (e.g., functionCall), just store directly.
48
+ # Note: This works because we don't enable streamFunctionCallArguments in the API request.
49
+ # Without that opt-in flag, function calls arrive complete in a single chunk.
50
+ # If streaming function call arguments is enabled in the future (Gemini 3+ models),
51
+ # this would need to accumulate partialArgs similar to how we accumulate text.
52
+ @response_json["candidates"][0]["content"]["parts"][index] = part
53
+ end
54
+ end
55
+
56
+ delta
57
+ end
58
+
59
+ def accumulate_text_part(part, index)
60
+ existing_part = @response_json.dig("candidates", 0, "content", "parts", index)
61
+
62
+ if existing_part.present? && existing_part.key?("text")
63
+ # Accumulate text from incremental chunks
64
+ existing_part["text"] += part["text"]
65
+ else
66
+ # First text chunk for this index
67
+ @response_json["candidates"][0]["content"]["parts"][index] = part.dup
68
+ end
69
+ end
70
+
71
+ end
@@ -1,19 +1,62 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # == Schema Information
4
+ #
5
+ # Table name: raif_tasks
6
+ #
7
+ # id :bigint not null, primary key
8
+ # available_model_tools :jsonb not null
9
+ # completed_at :datetime
10
+ # creator_type :string
11
+ # failed_at :datetime
12
+ # llm_model_key :string not null
13
+ # prompt :text
14
+ # prompt_studio_run :boolean default(FALSE), not null
15
+ # raw_response :text
16
+ # requested_language_key :string
17
+ # response_format :integer default("text"), not null
18
+ # run_with :jsonb
19
+ # source_type :string
20
+ # started_at :datetime
21
+ # system_prompt :text
22
+ # type :string not null
23
+ # created_at :datetime not null
24
+ # updated_at :datetime not null
25
+ # creator_id :bigint
26
+ # source_id :bigint
27
+ #
28
+ # Indexes
29
+ #
30
+ # index_raif_tasks_on_completed_at (completed_at)
31
+ # index_raif_tasks_on_created_at (created_at)
32
+ # index_raif_tasks_on_creator (creator_type,creator_id)
33
+ # index_raif_tasks_on_failed_at (failed_at)
34
+ # index_raif_tasks_on_source (source_type,source_id)
35
+ # index_raif_tasks_on_started_at (started_at)
36
+ # index_raif_tasks_on_type (type)
37
+ # index_raif_tasks_on_type_and_completed_at (type,completed_at)
38
+ # index_raif_tasks_on_type_and_failed_at (type,failed_at)
39
+ # index_raif_tasks_on_type_and_started_at (type,started_at)
40
+ #
3
41
  module Raif
4
42
  class Task < Raif::ApplicationRecord
43
+ prepend Raif::Concerns::HasPromptTemplates
44
+
5
45
  include Raif::Concerns::HasLlm
6
46
  include Raif::Concerns::HasRequestedLanguage
7
47
  include Raif::Concerns::HasAvailableModelTools
48
+ include Raif::Concerns::HasRuntimeDuration
8
49
  include Raif::Concerns::InvokesModelTools
9
50
  include Raif::Concerns::LlmResponseParsing
10
51
  include Raif::Concerns::LlmTemperature
52
+ include Raif::Concerns::LlmPromptCaching
11
53
  include Raif::Concerns::JsonSchemaDefinition
12
- include Raif::Concerns::TaskRunArgs
54
+ include Raif::Concerns::RunWith
13
55
 
14
56
  llm_temperature 0.7
15
57
 
16
58
  belongs_to :creator, polymorphic: true, optional: true
59
+ belongs_to :source, polymorphic: true, optional: true
17
60
 
18
61
  validates :creator, presence: true, unless: -> { Raif.config.task_creator_optional }
19
62
 
@@ -25,8 +68,6 @@ module Raif
25
68
 
26
69
  normalizes :prompt, :system_prompt, with: ->(text){ text&.strip }
27
70
 
28
- delegate :json_response_schema, to: :class
29
-
30
71
  scope :completed, -> { where.not(completed_at: nil) }
31
72
  scope :failed, -> { where.not(failed_at: nil) }
32
73
  scope :in_progress, -> { where.not(started_at: nil).where(completed_at: nil, failed_at: nil) }
@@ -35,7 +76,7 @@ module Raif
35
76
  attr_accessor :files, :images
36
77
 
37
78
  after_initialize -> { self.available_model_tools ||= [] }
38
- after_initialize -> { self.task_run_args ||= {} }
79
+ after_initialize -> { self.run_with ||= {} }
39
80
 
40
81
  def status
41
82
  if completed_at?
@@ -59,7 +100,7 @@ module Raif
59
100
  # @param files [Array] Optional array of Raif::ModelFileInput objects to include with the prompt.
60
101
  # @param args [Hash] Additional arguments to pass to the instance of the task that is created.
61
102
  # @return [Raif::Task, nil] The task instance that was created and run.
62
- def self.run(creator: nil, available_model_tools: [], llm_model_key: nil, images: [], files: [], **args)
103
+ def self.run(creator: nil, available_model_tools: [], llm_model_key: nil, images: [], files: [], **args, &block)
63
104
  task = new(
64
105
  creator: creator,
65
106
  llm_model_key: llm_model_key,
@@ -67,12 +108,11 @@ module Raif
67
108
  started_at: Time.current,
68
109
  images: images,
69
110
  files: files,
70
- task_run_args: serialize_task_run_args(args),
71
111
  **args
72
112
  )
73
113
 
74
114
  task.save!
75
- task.run
115
+ task.run(&block)
76
116
  task
77
117
  rescue StandardError => e
78
118
  task&.failed!
@@ -91,11 +131,17 @@ module Raif
91
131
  task
92
132
  end
93
133
 
94
- def run(skip_prompt_population: false)
134
+ def run(skip_prompt_population: false, &block)
95
135
  update_columns(started_at: Time.current) if started_at.nil?
96
136
 
97
137
  populate_prompts unless skip_prompt_population
98
- messages = [{ "role" => "user", "content" => message_content }]
138
+
139
+ streaming_block = if block_given?
140
+ proc do |model_completion, delta, sse_event|
141
+ update_columns(raw_response: model_completion.raw_response, updated_at: Time.current)
142
+ block.call(model_completion, delta, sse_event)
143
+ end
144
+ end
99
145
 
100
146
  mc = llm.chat(
101
147
  messages: messages,
@@ -103,7 +149,10 @@ module Raif
103
149
  system_prompt: system_prompt,
104
150
  response_format: response_format.to_sym,
105
151
  available_model_tools: available_model_tools,
106
- temperature: self.class.temperature
152
+ temperature: self.class.temperature,
153
+ anthropic_prompt_caching_enabled: self.class.anthropic_prompt_caching_enabled,
154
+ bedrock_prompt_caching_enabled: self.class.bedrock_prompt_caching_enabled,
155
+ &streaming_block
107
156
  )
108
157
 
109
158
  self.raif_model_completion = mc.becomes(Raif::ModelCompletion)
@@ -115,9 +164,13 @@ module Raif
115
164
  self
116
165
  end
117
166
 
118
- def re_run
167
+ def re_run(&block)
119
168
  update_columns(started_at: Time.current)
120
- run(skip_prompt_population: true)
169
+ run(skip_prompt_population: true, &block)
170
+ end
171
+
172
+ def messages
173
+ [{ "role" => "user", "content" => message_content }]
121
174
  end
122
175
 
123
176
  # Returns the LLM prompt for the task.
@@ -146,6 +199,22 @@ module Raif
146
199
  end
147
200
  end
148
201
 
202
+ # Instance method to get the JSON response schema
203
+ # For instance-dependent schemas, builds the schema with this instance as context
204
+ # For class-level schemas, returns the class-level schema
205
+ def json_response_schema
206
+ schema_for_instance(:json_response)
207
+ end
208
+
209
+ # Returns additional attributes to assign when creating tasks in Prompt Studio
210
+ # (reruns, batch runs, and judge tasks). Override in your ApplicationTask or
211
+ # task subclass to include app-specific attributes.
212
+ #
213
+ # @return [Hash] additional attributes to assign to the new task
214
+ def prompt_studio_task_attributes
215
+ {}
216
+ end
217
+
149
218
  def build_prompt
150
219
  raise NotImplementedError, "Raif::Task subclasses must implement #build_prompt"
151
220
  end
@@ -189,15 +258,13 @@ module Raif
189
258
  end
190
259
 
191
260
  def process_model_tool_invocations
192
- return unless response_format_json?
193
- return unless parsed_response.is_a?(Hash)
194
- return unless parsed_response["tools"].present? && parsed_response["tools"].is_a?(Array)
261
+ return unless raif_model_completion&.response_tool_calls.present?
195
262
 
196
- parsed_response["tools"].each do |t|
197
- tool_klass = available_model_tools_map[t["name"]]
263
+ raif_model_completion.response_tool_calls.each do |tool_call|
264
+ tool_klass = available_model_tools_map[tool_call["name"]]
198
265
  next unless tool_klass
199
266
 
200
- tool_klass.invoke_tool(tool_arguments: t["arguments"], source: self)
267
+ tool_klass.invoke_tool(provider_tool_call_id: tool_call["provider_tool_call_id"], tool_arguments: tool_call["arguments"], source: self)
201
268
  end
202
269
  end
203
270
 
@@ -1,5 +1,24 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # == Schema Information
4
+ #
5
+ # Table name: raif_user_tool_invocations
6
+ #
7
+ # id :bigint not null, primary key
8
+ # tool_settings :jsonb not null
9
+ # type :string not null
10
+ # created_at :datetime not null
11
+ # updated_at :datetime not null
12
+ # raif_conversation_entry_id :bigint not null
13
+ #
14
+ # Indexes
15
+ #
16
+ # index_raif_user_tool_invocations_on_raif_conversation_entry_id (raif_conversation_entry_id)
17
+ #
18
+ # Foreign Keys
19
+ #
20
+ # fk_rails_... (raif_conversation_entry_id => raif_conversation_entries.id)
21
+ #
3
22
  class Raif::UserToolInvocation < Raif::ApplicationRecord
4
23
  belongs_to :raif_conversation_entry, class_name: "Raif::ConversationEntry"
5
24
 
@@ -5,14 +5,23 @@
5
5
  <meta name="viewport" content="width=device-width,initial-scale=1">
6
6
  <%= csrf_meta_tags %>
7
7
  <%= csp_meta_tag %>
8
+ <%= action_cable_meta_tag if respond_to?(:action_cable_meta_tag) %>
8
9
 
9
10
  <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-T3c6CoIi6uLrA9TneNEoa7RxnatzjcDSCmG1MXxSR1GAsXEV/Dwwykc2MPK8M2HN" crossorigin="anonymous">
10
11
  <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-C6RzsynM9kWDrMNeT87bh95OGNyZPhcTNXj1NW7RuBCsyN/o0jlpcV8Qyq46cDfL" crossorigin="anonymous" nonce="<%= request.content_security_policy_nonce %>"></script>
11
12
 
13
+ <link href="https://cdn.jsdelivr.net/npm/tom-select@2.4.3/dist/css/tom-select.bootstrap5.min.css" rel="stylesheet">
14
+ <script src="https://cdn.jsdelivr.net/npm/tom-select@2.4.3/dist/js/tom-select.complete.min.js" nonce="<%= request.content_security_policy_nonce %>"></script>
15
+
16
+ <% if respond_to?(:javascript_importmap_tags) %>
17
+ <%= javascript_importmap_tags "raif_admin" %>
18
+ <% else %>
19
+ <%= javascript_include_tag "raif_admin_sprockets", nonce: true %>
20
+ <% end %>
12
21
  <%= stylesheet_link_tag "raif_admin" %>
13
22
  </head>
14
23
 
15
- <body class="raif-admin">
24
+ <body class="raif-admin" data-controller="raif--copy-to-clipboard">
16
25
  <nav class="navbar navbar-expand-md navbar-dark bg-dark">
17
26
  <div class="container-fluid">
18
27
  <a class="navbar-brand fw-bold" href="<%= raif.admin_tasks_path %>">
@@ -35,7 +44,7 @@
35
44
  <div class="position-sticky pt-3">
36
45
  <ul class="nav flex-column">
37
46
  <li class="nav-item">
38
- <a class="nav-link <%= current_page?(raif.admin_stats_path) ? "active" : "" %>" href="<%= raif.admin_stats_path %>">
47
+ <a class="nav-link <%= params[:controller].start_with?("raif/admin/stats") ? "active" : "" %>" href="<%= raif.admin_stats_path %>">
39
48
  <span class="d-inline-block me-2">
40
49
  <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-bar-chart" viewBox="0 0 16 16">
41
50
  <path d="M4 11H2v3h2v-3zm5-4H7v7h2V7zm5-5v12h-2V2h2zm-2-1a1 1 0 0 0-1 1v12a1 1 0 0 0 1 1h2a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1h-2zM6 7a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1v7a1 1 0 0 1-1 1H7a1 1 0 0 1-1-1V7zm-5 4a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1v3a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1v-3z" />
@@ -96,6 +105,38 @@
96
105
  <%= t("raif.admin.common.agents") %>
97
106
  </a>
98
107
  </li>
108
+ <li class="nav-item">
109
+ <a class="nav-link <%= current_page?(raif.admin_llms_path) ? "active" : "" %>" href="<%= raif.admin_llms_path %>">
110
+ <span class="d-inline-block me-2">
111
+ <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-currency-dollar" viewBox="0 0 16 16">
112
+ <path d="M4 10.781c.148 1.667 1.513 2.85 3.591 3.003V15h1.043v-1.216c2.27-.179 3.678-1.438 3.678-3.3 0-1.59-.947-2.51-2.956-3.028l-.722-.187V3.467c1.122.11 1.879.714 2.07 1.616h1.47c-.166-1.6-1.54-2.748-3.54-2.875V1H7.591v1.233c-1.939.23-3.27 1.472-3.27 3.156 0 1.454.966 2.483 2.661 2.917l.61.162v4.031c-1.149-.17-1.94-.8-2.131-1.718H4zm3.391-3.836c-1.043-.263-1.6-.825-1.6-1.616 0-.944.704-1.641 1.8-1.828v3.495l-.2-.05zm1.591 1.872c1.287.323 1.852.859 1.852 1.769 0 1.097-.826 1.828-2.2 1.939V8.73l.348.086z" />
113
+ </svg>
114
+ </span>
115
+ <%= t("raif.admin.common.llms") %>
116
+ </a>
117
+ </li>
118
+ <li class="nav-item">
119
+ <a class="nav-link <%= params[:controller].start_with?("raif/admin/prompt_studio") ? "active" : "" %>" href="<%= raif.admin_prompt_studio_tasks_path %>">
120
+ <span class="d-inline-block me-2">
121
+ <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-pencil-square" viewBox="0 0 16 16">
122
+ <path d="M15.502 1.94a.5.5 0 0 1 0 .706L14.459 3.69l-2-2L13.502.646a.5.5 0 0 1 .707 0l1.293 1.293zm-1.75 2.456-2-2L4.939 9.21a.5.5 0 0 0-.121.196l-.805 2.414a.25.25 0 0 0 .316.316l2.414-.805a.5.5 0 0 0 .196-.12l6.813-6.814z" />
123
+ <path fill-rule="evenodd" d="M1 13.5A1.5 1.5 0 0 0 2.5 15h11a1.5 1.5 0 0 0 1.5-1.5v-6a.5.5 0 0 0-1 0v6a.5.5 0 0 1-.5.5h-11a.5.5 0 0 1-.5-.5v-11a.5.5 0 0 1 .5-.5H9a.5.5 0 0 0 0-1H2.5A1.5 1.5 0 0 0 1 2.5v11z" />
124
+ </svg>
125
+ </span>
126
+ <%= t("raif.admin.prompt_studio.common.prompt_studio") %>
127
+ </a>
128
+ </li>
129
+ <li class="nav-item">
130
+ <a class="nav-link <%= current_page?(raif.admin_config_path) ? "active" : "" %>" href="<%= raif.admin_config_path %>">
131
+ <span class="d-inline-block me-2">
132
+ <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-gear" viewBox="0 0 16 16">
133
+ <path d="M8 4.754a3.246 3.246 0 1 0 0 6.492 3.246 3.246 0 0 0 0-6.492zM5.754 8a2.246 2.246 0 1 1 4.492 0 2.246 2.246 0 0 1-4.492 0z" />
134
+ <path d="M9.796 1.343c-.527-1.79-3.065-1.79-3.592 0l-.094.319a.873.873 0 0 1-1.255.52l-.292-.16c-1.64-.892-3.433.902-2.54 2.541l.159.292a.873.873 0 0 1-.52 1.255l-.319.094c-1.79.527-1.79 3.065 0 3.592l.319.094a.873.873 0 0 1 .52 1.255l-.16.292c-.892 1.64.901 3.434 2.541 2.54l.292-.159a.873.873 0 0 1 1.255.52l.094.319c.527 1.79 3.065 1.79 3.592 0l.094-.319a.873.873 0 0 1 1.255-.52l.292.16c1.64.893 3.434-.902 2.54-2.541l-.159-.292a.873.873 0 0 1 .52-1.255l.319-.094c1.79-.527 1.79-3.065 0-3.592l-.319-.094a.873.873 0 0 1-.52-1.255l.16-.292c.893-1.64-.902-3.433-2.541-2.54l-.292.159a.873.873 0 0 1-1.255-.52l-.094-.319z" />
135
+ </svg>
136
+ </span>
137
+ <%= t("raif.admin.common.config") %>
138
+ </a>
139
+ </li>
99
140
  </ul>
100
141
  </div>
101
142
  </div>
@@ -1,5 +1,6 @@
1
1
  <tr id="<%= dom_id(agent) %>" class="raif-agent">
2
2
  <td><%= link_to "##{agent.id}", raif.admin_agent_path(agent) %></td>
3
+ <td><small class="text-muted"><%= agent.type.demodulize %></small></td>
3
4
  <td><small class="text-muted"><%= agent.created_at.rfc822 %></small></td>
4
5
  <td><small class="text-muted"><%= truncate(agent.task, length: 100) %></small></td>
5
6
  <td>
@@ -13,6 +14,14 @@
13
14
  <span class="badge bg-secondary"><%= t("raif.admin.common.pending") %></span>
14
15
  <% end %>
15
16
  </td>
17
+ <td><small class="text-muted"><%= agent.runtime_duration %></small></td>
16
18
  <td><%= agent.iteration_count %> / <%= agent.max_iterations %></td>
19
+ <td>
20
+ <% if agent.total_cost && agent.total_cost > 0 %>
21
+ <small><%= "$" %><%= number_with_precision(agent.total_cost, precision: 6) %></small>
22
+ <% else %>
23
+ -
24
+ <% end %>
25
+ </td>
17
26
  <td><small class="text-muted"><%= truncate(agent.final_answer, length: 100) if agent.final_answer %></small></td>
18
27
  </tr>
@@ -1,15 +1,37 @@
1
- <div class="message <%= message["role"] %> mb-3">
1
+ <% message_type = message["type"] || message["role"] %>
2
+ <div class="message <%= message_type %> mb-3">
2
3
  <div class="message-header d-flex justify-content-between align-items-center mb-2 px-2">
3
- <strong class="<%= message["role"] == "user" ? "text-primary" : "text-success" %>">
4
- <%= message["role"].capitalize %>
5
- <% if message[:counter] == 0 && message["role"] == 'user' %>
6
- (<%= t("raif.admin.common.initial_task") %>)
4
+ <strong class="<%= conversation_message_header_class(message) %>">
5
+ <% if message_type == "tool_call" %>
6
+ <%= t("raif.admin.common.tool_call") %>: <%= message["name"] %>
7
+ <% elsif message_type == "tool_call_result" %>
8
+ <%= t("raif.admin.common.tool_result") %>
9
+ <% else %>
10
+ <%= message["role"]&.capitalize %>
11
+ <% if message[:counter] == 0 && message["role"] == 'user' %>
12
+ (<%= t("raif.admin.common.initial_task") %>)
13
+ <% end %>
14
+ <% end %>
15
+ <% if is_final_answer %>
16
+ <span class="badge bg-info ms-2"><%= t("raif.admin.common.final_answer") %></span>
7
17
  <% end %>
8
18
  </strong>
9
19
  <span class="badge bg-secondary">#<%= message_count %></span>
10
20
  </div>
11
21
 
12
22
  <div class="message-content px-3 py-2">
13
- <code><%= message["content"] %></code>
23
+ <% if message_type == "tool_call" %>
24
+ <% if message["assistant_message"].present? %>
25
+ <div class="mb-2"><code><%= message["assistant_message"] %></code></div>
26
+ <% end %>
27
+ <div>
28
+ <strong><%= t("raif.admin.common.arguments") %>:</strong>
29
+ <code><%= message["arguments"].to_json %></code>
30
+ </div>
31
+ <% elsif message_type == "tool_call_result" %>
32
+ <code><%= message["result"].is_a?(String) ? message["result"] : JSON.pretty_generate(message["result"]) %></code>
33
+ <% else %>
34
+ <code><%= message["content"] %></code>
35
+ <% end %>
14
36
  </div>
15
37
  </div>
@@ -2,16 +2,66 @@
2
2
 
3
3
  <div class="row">
4
4
  <div class="col-12">
5
+ <%= form_tag raif.admin_agents_path, method: :get, class: "mb-4" do %>
6
+ <div class="row align-items-end">
7
+ <div class="col-md-3">
8
+ <div class="form-group">
9
+ <label for="agent_type"><%= t("raif.admin.common.type") %></label>
10
+ <%= select_tag :agent_type,
11
+ options_for_select(
12
+ [[t("raif.admin.common.all"), "all"]] + @agent_types.map { |type| [type, type] },
13
+ @selected_type
14
+ ),
15
+ { class: "form-select", data: { controller: "raif--tom-select" } } %>
16
+ </div>
17
+ </div>
18
+ <div class="col-md-3">
19
+ <div class="form-group">
20
+ <label for="status"><%= t("raif.admin.common.status") %></label>
21
+ <%= select_tag :status,
22
+ options_for_select(
23
+ [
24
+ [t("raif.admin.common.all"), :all],
25
+ [t("raif.admin.common.completed"), :completed],
26
+ [t("raif.admin.common.failed"), :failed],
27
+ [t("raif.admin.common.running"), :running],
28
+ [t("raif.admin.common.pending"), :pending]
29
+ ],
30
+ @selected_status
31
+ ),
32
+ { class: "form-select", data: { controller: "raif--tom-select" } } %>
33
+ </div>
34
+ </div>
35
+ <div class="col-md-3">
36
+ <div class="form-group">
37
+ <label for="llm_model_key"><%= t("raif.admin.common.model") %></label>
38
+ <%= select_tag :llm_model_key,
39
+ options_for_select(
40
+ [[t("raif.admin.common.all"), ""]] + @llm_model_keys.map { |key| [key, key] },
41
+ @selected_llm_model_key
42
+ ),
43
+ { class: "form-select", data: { controller: "raif--tom-select" } } %>
44
+ </div>
45
+ </div>
46
+ <div class="col-md-2">
47
+ <%= submit_tag t("raif.admin.common.filter"), class: "btn btn-primary" %>
48
+ </div>
49
+ </div>
50
+ <% end %>
51
+
5
52
  <% if @agents.any? %>
6
53
  <div class="table-responsive">
7
54
  <table class="table table-striped table-hover">
8
55
  <thead class="table-light">
9
56
  <tr>
10
57
  <th><%= t("raif.admin.common.id") %></th>
58
+ <th><%= t("raif.admin.common.type") %></th>
11
59
  <th><%= t("raif.admin.common.created_at") %></th>
12
60
  <th><%= t("raif.admin.common.task") %></th>
13
61
  <th><%= t("raif.admin.common.status") %></th>
62
+ <th><%= t("raif.admin.common.duration") %></th>
14
63
  <th><%= t("raif.admin.common.iterations") %></th>
64
+ <th><%= t("raif.admin.common.total_cost") %></th>
15
65
  <th><%= t("raif.admin.common.final_answer") %></th>
16
66
  </tr>
17
67
  </thead>
@@ -36,6 +36,10 @@
36
36
  <% end %>
37
37
  </div>
38
38
  </div>
39
+ <div class="row mb-3">
40
+ <div class="col-md-3"><strong><%= t("raif.admin.common.duration") %>:</strong></div>
41
+ <div class="col-md-9"><%= @agent.runtime_duration %></div>
42
+ </div>
39
43
  <div class="row mb-3">
40
44
  <div class="col-md-3"><strong><%= t("raif.admin.common.iterations") %>:</strong></div>
41
45
  <div class="col-md-9"><%= @agent.iteration_count %> / <%= @agent.max_iterations %></div>
@@ -50,9 +54,50 @@
50
54
  <div class="col-md-9"><%= @agent.requested_language_key %></div>
51
55
  </div>
52
56
  <% end %>
57
+ <div class="row mb-3">
58
+ <div class="col-md-3"><strong><%= t("raif.admin.common.prompt_tokens") %>:</strong></div>
59
+ <div class="col-md-9">
60
+ <% if @agent.total_prompt_tokens > 0 %>
61
+ <%= number_with_delimiter(@agent.total_prompt_tokens) %>
62
+ <% if @agent.total_prompt_token_cost > 0 %>
63
+ <small>(<%= t("raif.admin.common.est_cost") %>: <%= "$" %><%= number_with_precision(@agent.total_prompt_token_cost, precision: 6) %>)</small>
64
+ <% end %>
65
+ <% else %>
66
+ -
67
+ <% end %>
68
+ </div>
69
+ </div>
70
+ <div class="row mb-3">
71
+ <div class="col-md-3"><strong><%= t("raif.admin.common.completion_tokens") %>:</strong></div>
72
+ <div class="col-md-9">
73
+ <% if @agent.total_completion_tokens > 0 %>
74
+ <%= number_with_delimiter(@agent.total_completion_tokens) %>
75
+ <% if @agent.total_output_token_cost > 0 %>
76
+ <small>(<%= t("raif.admin.common.est_cost") %>: <%= "$" %><%= number_with_precision(@agent.total_output_token_cost, precision: 6) %>)</small>
77
+ <% end %>
78
+ <% else %>
79
+ -
80
+ <% end %>
81
+ </div>
82
+ </div>
83
+ <div class="row mb-3">
84
+ <div class="col-md-3"><strong><%= t("raif.admin.common.total_tokens") %>:</strong></div>
85
+ <div class="col-md-9">
86
+ <% if @agent.total_tokens_sum > 0 %>
87
+ <%= number_with_delimiter(@agent.total_tokens_sum) %>
88
+ <% if @agent.total_cost > 0 %>
89
+ <small>(<%= t("raif.admin.common.est_cost") %>: <%= "$" %><%= number_with_precision(@agent.total_cost, precision: 6) %>)</small>
90
+ <% end %>
91
+ <% else %>
92
+ -
93
+ <% end %>
94
+ </div>
95
+ </div>
53
96
  </div>
54
97
  </div>
55
98
 
99
+ <%= render "raif/admin/model_tools/list", model_tools: @agent.available_model_tools_map %>
100
+
56
101
  <div class="card mb-4">
57
102
  <div class="card-header">
58
103
  <h5 class="mb-0"><%= t("raif.admin.common.task") %></h5>
@@ -91,7 +136,11 @@
91
136
  <div class="conversation-history p-3">
92
137
  <% @agent.conversation_history.each_with_index do |message, index| %>
93
138
  <%= render partial: "raif/admin/agents/conversation_message",
94
- locals: { message: message, message_count: index + 1 } %>
139
+ locals: {
140
+ message: message,
141
+ message_count: index + 1,
142
+ is_final_answer: message["type"] == "tool_call" && message["name"] == "agent_final_answer"
143
+ } %>
95
144
  <% end %>
96
145
  </div>
97
146
  </div>