raif 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +8 -7
  3. data/app/assets/builds/raif.css +4 -1
  4. data/app/assets/builds/raif_admin.css +52 -2
  5. data/app/assets/builds/raif_admin_sprockets.js +2709 -0
  6. data/app/assets/javascript/raif/admin/copy_to_clipboard_controller.js +132 -0
  7. data/app/assets/javascript/raif/admin/cost_estimate_controller.js +80 -0
  8. data/app/assets/javascript/raif/admin/judge_config_controller.js +23 -0
  9. data/app/assets/javascript/raif/admin/select_all_checkboxes_controller.js +33 -0
  10. data/app/assets/javascript/raif/admin/sortable_table_controller.js +51 -0
  11. data/app/assets/javascript/raif/admin/table_search_controller.js +15 -0
  12. data/app/assets/javascript/raif/admin/tom_select_controller.js +33 -0
  13. data/app/assets/javascript/raif/controllers/conversations_controller.js +1 -1
  14. data/app/assets/javascript/raif_admin.js +23 -0
  15. data/app/assets/javascript/raif_admin_sprockets.js +24 -0
  16. data/app/assets/stylesheets/raif/admin/conversation.scss +16 -0
  17. data/app/assets/stylesheets/raif/conversations.scss +3 -0
  18. data/app/assets/stylesheets/raif.scss +2 -1
  19. data/app/assets/stylesheets/raif_admin.scss +50 -1
  20. data/app/controllers/raif/admin/agents_controller.rb +27 -1
  21. data/app/controllers/raif/admin/application_controller.rb +16 -0
  22. data/app/controllers/raif/admin/configs_controller.rb +95 -0
  23. data/app/controllers/raif/admin/llms_controller.rb +27 -0
  24. data/app/controllers/raif/admin/model_completions_controller.rb +24 -1
  25. data/app/controllers/raif/admin/model_tool_invocations_controller.rb +7 -1
  26. data/app/controllers/raif/admin/prompt_studio/agents_controller.rb +25 -0
  27. data/app/controllers/raif/admin/prompt_studio/base_controller.rb +32 -0
  28. data/app/controllers/raif/admin/prompt_studio/batch_runs_controller.rb +102 -0
  29. data/app/controllers/raif/admin/prompt_studio/conversations_controller.rb +25 -0
  30. data/app/controllers/raif/admin/prompt_studio/tasks_controller.rb +64 -0
  31. data/app/controllers/raif/admin/stats/model_tool_invocations_controller.rb +21 -0
  32. data/app/controllers/raif/admin/stats/tasks_controller.rb +15 -6
  33. data/app/controllers/raif/admin/stats_controller.rb +32 -3
  34. data/app/controllers/raif/admin/tasks_controller.rb +5 -0
  35. data/app/controllers/raif/conversation_entries_controller.rb +1 -0
  36. data/app/controllers/raif/conversations_controller.rb +10 -2
  37. data/app/helpers/raif/application_helper.rb +40 -0
  38. data/app/jobs/raif/conversation_entry_job.rb +8 -6
  39. data/app/jobs/raif/prompt_studio_batch_run_item_job.rb +11 -0
  40. data/app/jobs/raif/prompt_studio_batch_run_job.rb +15 -0
  41. data/app/jobs/raif/prompt_studio_task_run_job.rb +36 -0
  42. data/app/models/raif/admin/task_stat.rb +7 -0
  43. data/app/models/raif/agent.rb +98 -6
  44. data/app/models/raif/agents/native_tool_calling_agent.rb +179 -52
  45. data/app/models/raif/application_record.rb +18 -0
  46. data/app/models/raif/concerns/agent_inference_stats.rb +35 -0
  47. data/app/models/raif/concerns/has_prompt_templates.rb +88 -0
  48. data/app/models/raif/concerns/has_runtime_duration.rb +41 -0
  49. data/app/models/raif/concerns/json_schema_definition.rb +54 -6
  50. data/app/models/raif/concerns/llm_prompt_caching.rb +20 -0
  51. data/app/models/raif/concerns/llms/anthropic/message_formatting.rb +34 -0
  52. data/app/models/raif/concerns/llms/anthropic/response_tool_calls.rb +24 -0
  53. data/app/models/raif/concerns/llms/anthropic/tool_formatting.rb +8 -0
  54. data/app/models/raif/concerns/llms/bedrock/message_formatting.rb +43 -0
  55. data/app/models/raif/concerns/llms/bedrock/response_tool_calls.rb +26 -0
  56. data/app/models/raif/concerns/llms/bedrock/tool_formatting.rb +8 -0
  57. data/app/models/raif/concerns/llms/google/message_formatting.rb +112 -0
  58. data/app/models/raif/concerns/llms/google/response_tool_calls.rb +32 -0
  59. data/app/models/raif/concerns/llms/google/tool_formatting.rb +76 -0
  60. data/app/models/raif/concerns/llms/message_formatting.rb +41 -5
  61. data/app/models/raif/concerns/llms/open_ai/json_schema_validation.rb +3 -3
  62. data/app/models/raif/concerns/llms/open_ai_completions/message_formatting.rb +22 -0
  63. data/app/models/raif/concerns/llms/open_ai_completions/response_tool_calls.rb +22 -0
  64. data/app/models/raif/concerns/llms/open_ai_completions/tool_formatting.rb +8 -0
  65. data/app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb +17 -0
  66. data/app/models/raif/concerns/llms/open_ai_responses/response_tool_calls.rb +26 -0
  67. data/app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb +8 -0
  68. data/app/models/raif/concerns/provider_managed_tool_calls.rb +162 -0
  69. data/app/models/raif/concerns/run_with.rb +127 -0
  70. data/app/models/raif/conversation.rb +112 -8
  71. data/app/models/raif/conversation_entry.rb +38 -4
  72. data/app/models/raif/embedding_model.rb +2 -1
  73. data/app/models/raif/embedding_models/bedrock.rb +10 -1
  74. data/app/models/raif/embedding_models/google.rb +37 -0
  75. data/app/models/raif/embedding_models/open_ai.rb +1 -1
  76. data/app/models/raif/evals/llm_judge.rb +70 -0
  77. data/{lib → app/models}/raif/evals/llm_judges/binary.rb +41 -3
  78. data/{lib → app/models}/raif/evals/llm_judges/comparative.rb +41 -3
  79. data/{lib → app/models}/raif/evals/llm_judges/scored.rb +39 -1
  80. data/{lib → app/models}/raif/evals/llm_judges/summarization.rb +40 -2
  81. data/app/models/raif/llm.rb +104 -4
  82. data/app/models/raif/llms/anthropic.rb +32 -22
  83. data/app/models/raif/llms/bedrock.rb +64 -24
  84. data/app/models/raif/llms/google.rb +166 -0
  85. data/app/models/raif/llms/open_ai_base.rb +23 -5
  86. data/app/models/raif/llms/open_ai_completions.rb +14 -12
  87. data/app/models/raif/llms/open_ai_responses.rb +14 -17
  88. data/app/models/raif/llms/open_router.rb +16 -15
  89. data/app/models/raif/model_completion.rb +103 -1
  90. data/app/models/raif/model_tool.rb +55 -5
  91. data/app/models/raif/model_tool_invocation.rb +68 -6
  92. data/app/models/raif/model_tools/agent_final_answer.rb +2 -7
  93. data/app/models/raif/model_tools/provider_managed/code_execution.rb +4 -0
  94. data/app/models/raif/model_tools/provider_managed/image_generation.rb +4 -0
  95. data/app/models/raif/model_tools/provider_managed/web_search.rb +4 -0
  96. data/app/models/raif/prompt_studio_batch_run.rb +155 -0
  97. data/app/models/raif/prompt_studio_batch_run_item.rb +220 -0
  98. data/app/models/raif/streaming_responses/bedrock.rb +60 -1
  99. data/app/models/raif/streaming_responses/google.rb +71 -0
  100. data/app/models/raif/task.rb +85 -18
  101. data/app/models/raif/user_tool_invocation.rb +19 -0
  102. data/app/views/layouts/raif/admin.html.erb +43 -2
  103. data/app/views/raif/admin/agents/_agent.html.erb +9 -0
  104. data/app/views/raif/admin/agents/_conversation_message.html.erb +28 -6
  105. data/app/views/raif/admin/agents/index.html.erb +50 -0
  106. data/app/views/raif/admin/agents/show.html.erb +50 -1
  107. data/app/views/raif/admin/configs/show.html.erb +117 -0
  108. data/app/views/raif/admin/conversations/_conversation_entry.html.erb +29 -34
  109. data/app/views/raif/admin/conversations/show.html.erb +2 -0
  110. data/app/views/raif/admin/llms/index.html.erb +110 -0
  111. data/app/views/raif/admin/model_completions/_model_completion.html.erb +10 -5
  112. data/app/views/raif/admin/model_completions/index.html.erb +40 -1
  113. data/app/views/raif/admin/model_completions/show.html.erb +256 -84
  114. data/app/views/raif/admin/model_tool_invocations/index.html.erb +22 -1
  115. data/app/views/raif/admin/model_tool_invocations/show.html.erb +18 -0
  116. data/app/views/raif/admin/model_tools/_list.html.erb +16 -0
  117. data/app/views/raif/admin/model_tools/_model_tool.html.erb +36 -0
  118. data/app/views/raif/admin/prompt_studio/agents/index.html.erb +56 -0
  119. data/app/views/raif/admin/prompt_studio/agents/show.html.erb +57 -0
  120. data/app/views/raif/admin/prompt_studio/batch_runs/_batch_run_item.html.erb +54 -0
  121. data/app/views/raif/admin/prompt_studio/batch_runs/_judge_config_fields.html.erb +76 -0
  122. data/app/views/raif/admin/prompt_studio/batch_runs/_judge_detail_modal.html.erb +27 -0
  123. data/app/views/raif/admin/prompt_studio/batch_runs/_modal.html.erb +35 -0
  124. data/app/views/raif/admin/prompt_studio/batch_runs/_progress.html.erb +78 -0
  125. data/app/views/raif/admin/prompt_studio/batch_runs/show.html.erb +49 -0
  126. data/app/views/raif/admin/prompt_studio/conversations/index.html.erb +48 -0
  127. data/app/views/raif/admin/prompt_studio/conversations/show.html.erb +36 -0
  128. data/app/views/raif/admin/prompt_studio/shared/_nav_tabs.html.erb +17 -0
  129. data/app/views/raif/admin/prompt_studio/shared/_prompt_comparison.html.erb +87 -0
  130. data/app/views/raif/admin/prompt_studio/shared/_type_filter.html.erb +54 -0
  131. data/app/views/raif/admin/prompt_studio/tasks/_task_result.html.erb +145 -0
  132. data/app/views/raif/admin/prompt_studio/tasks/_task_row.html.erb +12 -0
  133. data/app/views/raif/admin/prompt_studio/tasks/_task_type_filter.html.erb +58 -0
  134. data/app/views/raif/admin/prompt_studio/tasks/_tasks_table.html.erb +22 -0
  135. data/app/views/raif/admin/prompt_studio/tasks/index.html.erb +35 -0
  136. data/app/views/raif/admin/prompt_studio/tasks/show.html.erb +19 -0
  137. data/app/views/raif/admin/stats/_stats_tile.html.erb +34 -0
  138. data/app/views/raif/admin/stats/index.html.erb +71 -88
  139. data/app/views/raif/admin/stats/model_tool_invocations/index.html.erb +43 -0
  140. data/app/views/raif/admin/stats/tasks/index.html.erb +20 -6
  141. data/app/views/raif/admin/tasks/_task.html.erb +1 -0
  142. data/app/views/raif/admin/tasks/index.html.erb +23 -6
  143. data/app/views/raif/admin/tasks/show.html.erb +56 -3
  144. data/app/views/raif/conversation_entries/_form.html.erb +3 -0
  145. data/app/views/raif/conversation_entries/_message.html.erb +10 -6
  146. data/app/views/raif/conversations/_conversation.html.erb +10 -0
  147. data/app/views/raif/conversations/_entry_processed.turbo_stream.erb +12 -0
  148. data/app/views/raif/conversations/index.html.erb +23 -0
  149. data/config/importmap.rb +8 -0
  150. data/config/locales/admin.en.yml +161 -1
  151. data/config/locales/en.yml +67 -4
  152. data/config/routes.rb +10 -0
  153. data/db/migrate/20250904194456_add_generating_entry_response_to_raif_conversations.rb +7 -0
  154. data/db/migrate/20250911125234_add_source_to_raif_tasks.rb +7 -0
  155. data/db/migrate/20251020005853_add_source_to_raif_agents.rb +7 -0
  156. data/db/migrate/20251020011346_rename_task_run_args_to_run_with.rb +7 -0
  157. data/db/migrate/20251020011405_add_run_with_to_raif_agents.rb +13 -0
  158. data/db/migrate/20251024160119_add_llm_messages_max_length_to_raif_conversations.rb +14 -0
  159. data/db/migrate/20251124185033_add_provider_tool_call_id_to_raif_model_tool_invocations.rb +7 -0
  160. data/db/migrate/20251128202941_add_tool_choice_to_raif_model_completions.rb +7 -0
  161. data/db/migrate/20260118144846_add_source_to_raif_conversations.rb +7 -0
  162. data/db/migrate/20260119000000_add_failure_tracking_to_raif_model_completions.rb +10 -0
  163. data/db/migrate/20260119000001_add_completed_at_to_raif_model_completions.rb +8 -0
  164. data/db/migrate/20260119000002_add_started_at_to_raif_model_completions.rb +8 -0
  165. data/db/migrate/20260307000000_add_prompt_studio_run_to_raif_tasks.rb +7 -0
  166. data/db/migrate/20260308000000_create_raif_prompt_studio_batch_runs.rb +27 -0
  167. data/db/migrate/20260308000001_create_raif_prompt_studio_batch_run_items.rb +24 -0
  168. data/db/migrate/20260407000000_add_cache_token_columns_to_raif_model_completions.rb +8 -0
  169. data/lib/generators/raif/agent/agent_generator.rb +18 -0
  170. data/lib/generators/raif/agent/templates/agent.rb.tt +7 -5
  171. data/lib/generators/raif/agent/templates/application_agent.rb.tt +1 -1
  172. data/lib/generators/raif/agent/templates/system_prompt.erb.tt +3 -0
  173. data/lib/generators/raif/conversation/conversation_generator.rb +19 -1
  174. data/lib/generators/raif/conversation/templates/conversation.rb.tt +6 -0
  175. data/lib/generators/raif/conversation/templates/system_prompt.erb.tt +4 -0
  176. data/lib/generators/raif/install/templates/initializer.rb +117 -8
  177. data/lib/generators/raif/task/task_generator.rb +18 -0
  178. data/lib/generators/raif/task/templates/prompt.erb.tt +4 -0
  179. data/lib/generators/raif/task/templates/task.rb.tt +10 -9
  180. data/lib/raif/configuration.rb +47 -2
  181. data/lib/raif/embedding_model_registry.rb +8 -0
  182. data/lib/raif/engine.rb +24 -1
  183. data/lib/raif/errors/blank_response_error.rb +8 -0
  184. data/lib/raif/errors/instance_dependent_schema_error.rb +8 -0
  185. data/lib/raif/errors/prompt_template_error.rb +15 -0
  186. data/lib/raif/errors/streaming_error.rb +6 -3
  187. data/lib/raif/errors.rb +3 -0
  188. data/lib/raif/evals/run.rb +1 -0
  189. data/lib/raif/evals.rb +0 -6
  190. data/lib/raif/json_schema_builder.rb +14 -0
  191. data/lib/raif/llm_registry.rb +433 -42
  192. data/lib/raif/messages.rb +180 -0
  193. data/lib/raif/prompt_studio_comparison_builder.rb +138 -0
  194. data/lib/raif/token_estimator.rb +28 -0
  195. data/lib/raif/version.rb +1 -1
  196. data/lib/raif.rb +11 -0
  197. data/lib/tasks/annotate_rb.rake +10 -0
  198. data/spec/support/rspec_helpers.rb +15 -9
  199. data/spec/support/test_task.rb +9 -0
  200. data/spec/support/test_template_task.rb +41 -0
  201. metadata +108 -15
  202. data/app/models/raif/agents/re_act_agent.rb +0 -127
  203. data/app/models/raif/agents/re_act_step.rb +0 -32
  204. data/app/models/raif/concerns/task_run_args.rb +0 -62
  205. data/lib/raif/evals/llm_judge.rb +0 -32
  206. /data/{lib → app/models}/raif/evals/scoring_rubric.rb +0 -0
@@ -0,0 +1,54 @@
1
+ <%= tag.tr id: dom_id(item) do %>
2
+ <td>
3
+ <%= link_to "##{item.source_task_id}", raif.admin_prompt_studio_task_path(item.source_task_id), class: "text-decoration-none" %>
4
+ </td>
5
+ <td>
6
+ <% case item.status %>
7
+ <% when "pending" %>
8
+ <span class="badge bg-secondary"><%= t("raif.admin.prompt_studio.batch_runs.status.pending") %></span>
9
+ <% when "running" %>
10
+ <span class="badge bg-warning text-dark">
11
+ <span class="spinner-border spinner-border-sm me-1" role="status" aria-hidden="true"></span>
12
+ <%= t("raif.admin.prompt_studio.batch_runs.status.running") %>
13
+ </span>
14
+ <% when "judging" %>
15
+ <span class="badge bg-info text-dark">
16
+ <span class="spinner-border spinner-border-sm me-1" role="status" aria-hidden="true"></span>
17
+ <%= t("raif.admin.prompt_studio.batch_runs.status.judging") %>
18
+ </span>
19
+ <% when "completed" %>
20
+ <span class="badge bg-success"><%= t("raif.admin.prompt_studio.batch_runs.status.completed") %></span>
21
+ <% when "failed" %>
22
+ <span class="badge bg-danger"><%= t("raif.admin.prompt_studio.batch_runs.status.failed") %></span>
23
+ <% end %>
24
+ </td>
25
+ <td>
26
+ <% if item.source_task&.raw_response.present? %>
27
+ <%= tag.span truncate(item.source_task.raw_response, length: 80), title: item.source_task.raw_response %>
28
+ <% end %>
29
+ </td>
30
+ <td>
31
+ <% if item.result_task&.raw_response.present? %>
32
+ <%= link_to truncate(item.result_task.raw_response, length: 80),
33
+ raif.admin_prompt_studio_task_path(item.result_task),
34
+ class: "text-decoration-none",
35
+ title: item.result_task.raw_response %>
36
+ <% end %>
37
+ </td>
38
+ <% if item.batch_run.has_judge? %>
39
+ <td>
40
+ <% if item.judge_summary.present? %>
41
+ <a href="#" class="text-decoration-none" data-bs-toggle="modal" data-bs-target="#judge-detail-modal-<%= item.id %>">
42
+ <strong><%= item.judge_summary %></strong>
43
+ </a>
44
+ <% end %>
45
+ </td>
46
+ <td>
47
+ <% if item.judge_reasoning.present? %>
48
+ <a href="#" class="text-decoration-none text-body" data-bs-toggle="modal" data-bs-target="#judge-detail-modal-<%= item.id %>">
49
+ <%= truncate(item.judge_reasoning, length: 80) %>
50
+ </a>
51
+ <% end %>
52
+ </td>
53
+ <% end %>
54
+ <% end %>
@@ -0,0 +1,76 @@
1
+ <div id="judge-config-section" data-controller="raif--judge-config">
2
+ <h6 class="mb-2"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.title") %></h6>
3
+ <p class="text-muted small"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.description") %></p>
4
+
5
+ <div class="mb-3">
6
+ <label for="judge_type" class="form-label"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.type_label") %></label>
7
+ <%= select_tag :judge_type,
8
+ options_for_select([
9
+ [t("raif.admin.prompt_studio.batch_runs.judge_config.type_none"), ""],
10
+ [t("raif.admin.prompt_studio.batch_runs.judge_config.type_binary"), "Raif::Evals::LlmJudges::Binary"],
11
+ [t("raif.admin.prompt_studio.batch_runs.judge_config.type_scored"), "Raif::Evals::LlmJudges::Scored"],
12
+ [t("raif.admin.prompt_studio.batch_runs.judge_config.type_comparative"), "Raif::Evals::LlmJudges::Comparative"],
13
+ [t("raif.admin.prompt_studio.batch_runs.judge_config.type_summarization"), "Raif::Evals::LlmJudges::Summarization"]
14
+ ]),
15
+ class: "form-select",
16
+ data: { controller: "raif--tom-select", action: "raif--judge-config#toggle raif--cost-estimate#calculate", raif__cost_estimate_target: "judgeTypeSelect" } %>
17
+ </div>
18
+
19
+ <div class="d-none" data-raif--judge-config-target="binary">
20
+ <div class="mb-3">
21
+ <label for="judge_criteria" class="form-label"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.criteria_label") %></label>
22
+ <%= text_area_tag :judge_criteria,
23
+ nil,
24
+ class: "form-control",
25
+ rows: 2,
26
+ placeholder: t("raif.admin.prompt_studio.batch_runs.judge_config.criteria_placeholder") %>
27
+ </div>
28
+ <div class="form-check mb-3">
29
+ <%= check_box_tag :judge_strict_mode, "1", false, class: "form-check-input" %>
30
+ <label for="judge_strict_mode" class="form-check-label"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.strict_mode_label") %></label>
31
+ </div>
32
+ </div>
33
+
34
+ <div class="d-none" data-raif--judge-config-target="scored">
35
+ <div class="mb-3">
36
+ <label for="judge_scoring_rubric" class="form-label"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.rubric_label") %></label>
37
+ <%= select_tag :judge_scoring_rubric,
38
+ options_for_select([
39
+ [t("raif.admin.prompt_studio.batch_runs.judge_config.rubric_accuracy"), "accuracy"],
40
+ [t("raif.admin.prompt_studio.batch_runs.judge_config.rubric_helpfulness"), "helpfulness"],
41
+ [t("raif.admin.prompt_studio.batch_runs.judge_config.rubric_clarity"), "clarity"]
42
+ ]),
43
+ class: "form-select" %>
44
+ </div>
45
+ </div>
46
+
47
+ <div class="d-none" data-raif--judge-config-target="comparative">
48
+ <div class="mb-3">
49
+ <label for="judge_comparison_criteria" class="form-label"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.comparison_criteria_label") %></label>
50
+ <%= text_area_tag :judge_comparison_criteria,
51
+ nil,
52
+ class: "form-control",
53
+ rows: 2,
54
+ placeholder: t("raif.admin.prompt_studio.batch_runs.judge_config.comparison_criteria_placeholder") %>
55
+ </div>
56
+ </div>
57
+
58
+ <div class="d-none" data-raif--judge-config-target="summarization">
59
+ <p class="text-muted small"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.summarization_description") %></p>
60
+ </div>
61
+
62
+ <div class="d-none" data-raif--judge-config-target="sharedOptions">
63
+ <div class="form-check mb-3">
64
+ <%= check_box_tag :judge_include_original_prompt_as_context, "1", true, class: "form-check-input" %>
65
+ <label for="judge_include_original_prompt_as_context" class="form-check-label"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.include_original_prompt_as_context_label") %></label>
66
+ <div class="form-text"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.include_original_prompt_as_context_description") %></div>
67
+ </div>
68
+ <div class="mb-3">
69
+ <label for="judge_llm_model_key" class="form-label"><%= t("raif.admin.prompt_studio.batch_runs.judge_config.judge_model_label") %></label>
70
+ <%= select_tag :judge_llm_model_key,
71
+ llm_model_options,
72
+ class: "form-select",
73
+ data: { controller: "raif--tom-select", raif__cost_estimate_target: "judgeModelSelect", action: "raif--cost-estimate#calculate" } %>
74
+ </div>
75
+ </div>
76
+ </div>
@@ -0,0 +1,27 @@
1
+ <div class="modal fade" id="judge-detail-modal-<%= item.id %>" tabindex="-1" aria-hidden="true">
2
+ <div class="modal-dialog modal-lg">
3
+ <div class="modal-content">
4
+ <div class="modal-header">
5
+ <h5 class="modal-title"><%= t("raif.admin.prompt_studio.batch_runs.show.judge_result") %> &mdash; <%= t("raif.admin.common.task") %> #<%= item.source_task_id %></h5>
6
+ <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
7
+ </div>
8
+ <div class="modal-body">
9
+ <% if item.judge_summary.present? %>
10
+ <div class="mb-3">
11
+ <h6><%= t("raif.admin.prompt_studio.batch_runs.show.judge_result") %></h6>
12
+ <span class="badge bg-primary fs-6"><%= item.judge_summary %></span>
13
+ </div>
14
+ <% end %>
15
+ <% if item.judge_reasoning.present? %>
16
+ <div>
17
+ <h6><%= t("raif.admin.prompt_studio.batch_runs.show.judge_reasoning") %></h6>
18
+ <pre class="pre-wrap border rounded p-3 bg-light"><%= item.judge_reasoning %></pre>
19
+ </div>
20
+ <% end %>
21
+ </div>
22
+ <div class="modal-footer">
23
+ <button type="button" class="btn btn-secondary" data-bs-dismiss="modal"><%= t("raif.admin.prompt_studio.common.back") %></button>
24
+ </div>
25
+ </div>
26
+ </div>
27
+ </div>
@@ -0,0 +1,35 @@
1
+ <div class="modal fade" id="batch-run-modal" tabindex="-1" aria-labelledby="batch-run-modal-label" aria-hidden="true">
2
+ <div class="modal-dialog modal-lg">
3
+ <div class="modal-content">
4
+ <div class="modal-header">
5
+ <h5 class="modal-title" id="batch-run-modal-label"><%= t("raif.admin.prompt_studio.batch_runs.create.title") %></h5>
6
+ <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
7
+ </div>
8
+ <div class="modal-body">
9
+ <div class="mb-3">
10
+ <label for="llm_model_key" class="form-label"><%= t("raif.admin.common.model") %></label>
11
+ <%= select_tag :llm_model_key,
12
+ llm_model_options,
13
+ class: "form-select",
14
+ data: { controller: "raif--tom-select", raif__cost_estimate_target: "modelSelect", action: "raif--cost-estimate#calculate" } %>
15
+ </div>
16
+
17
+ <%= render "raif/admin/prompt_studio/batch_runs/judge_config_fields" %>
18
+
19
+ <div class="alert alert-light border mb-0 mt-3 d-none" data-raif--cost-estimate-target="estimate"></div>
20
+ </div>
21
+ <div class="modal-footer d-flex justify-content-between">
22
+ <span class="text-muted">
23
+ <span data-raif--select-all-checkboxes-target="selectedCount">0</span>
24
+ <%= t("raif.admin.prompt_studio.batch_runs.create.tasks_selected") %>
25
+ </span>
26
+ <div class="d-flex gap-2">
27
+ <button type="button" class="btn btn-secondary" data-bs-dismiss="modal"><%= t("raif.admin.prompt_studio.common.back") %></button>
28
+ <%= submit_tag t("raif.admin.prompt_studio.batch_runs.create.submit"),
29
+ class: "btn btn-primary",
30
+ data: { confirm: t("raif.admin.prompt_studio.batch_runs.create.confirm") } %>
31
+ </div>
32
+ </div>
33
+ </div>
34
+ </div>
35
+ </div>
@@ -0,0 +1,78 @@
1
+ <%= tag.div id: dom_id(batch_run, :progress) do %>
2
+ <div class="card mb-4">
3
+ <div class="card-body">
4
+ <div class="d-flex justify-content-between align-items-center mb-2">
5
+ <div>
6
+ <strong><%= t("raif.admin.prompt_studio.batch_runs.show.progress") %></strong>
7
+ <% case batch_run.status %>
8
+ <% when :pending %>
9
+ <span class="badge bg-secondary ms-2"><%= t("raif.admin.prompt_studio.batch_runs.status.pending") %></span>
10
+ <% when :in_progress %>
11
+ <span class="badge bg-warning text-dark ms-2">
12
+ <span class="spinner-border spinner-border-sm me-1" role="status" aria-hidden="true"></span>
13
+ <%= t("raif.admin.prompt_studio.batch_runs.status.in_progress") %>
14
+ </span>
15
+ <% when :completed %>
16
+ <span class="badge bg-success ms-2"><%= t("raif.admin.prompt_studio.batch_runs.status.completed") %></span>
17
+ <% when :failed %>
18
+ <span class="badge bg-danger ms-2"><%= t("raif.admin.prompt_studio.batch_runs.status.failed") %></span>
19
+ <% end %>
20
+ </div>
21
+ <div class="text-muted">
22
+ <%= batch_run.completed_count + batch_run.failed_count %> / <%= batch_run.total_count %>
23
+ <% if batch_run.failed_count > 0 %>
24
+ (<span class="text-danger"><%= batch_run.failed_count %> <%= t("raif.admin.common.failed").downcase %></span>)
25
+ <% end %>
26
+ </div>
27
+ </div>
28
+ <div class="progress" style="height: 20px;">
29
+ <div class="progress-bar bg-success" role="progressbar"
30
+ style="width: <%= batch_run.progress_percentage %>%"
31
+ aria-valuenow="<%= batch_run.progress_percentage %>"
32
+ aria-valuemin="0"
33
+ aria-valuemax="100">
34
+ <%= batch_run.progress_percentage %>%
35
+ </div>
36
+ </div>
37
+
38
+ <% if batch_run.has_judge? && batch_run.completed? %>
39
+ <div class="mt-3">
40
+ <strong><%= t("raif.admin.prompt_studio.batch_runs.show.summary") %>:</strong>
41
+ <% case batch_run.judge_type %>
42
+ <% when "Raif::Evals::LlmJudges::Binary" %>
43
+ <% if batch_run.judge_pass_rate %>
44
+ <%= t("raif.admin.prompt_studio.batch_runs.show.pass_rate") %>: <%= batch_run.judge_pass_rate %>
45
+ <% end %>
46
+ <% when "Raif::Evals::LlmJudges::Scored" %>
47
+ <% if batch_run.judge_average_score %>
48
+ <%= t("raif.admin.prompt_studio.batch_runs.show.average_score") %>: <%= batch_run.judge_average_score %>
49
+ <% end %>
50
+ <% when "Raif::Evals::LlmJudges::Comparative" %>
51
+ <% summary = batch_run.judge_comparative_summary %>
52
+ <% if summary %>
53
+ <div class="d-flex gap-4 mt-2">
54
+ <div>
55
+ <span class="badge bg-success"><%= summary[:new_win_pct] %>%</span>
56
+ <%= t("raif.admin.prompt_studio.batch_runs.judge.new_response") %>
57
+ <span class="text-muted">(<%= summary[:new_wins] %>/<%= summary[:total] %>)</span>
58
+ </div>
59
+ <div>
60
+ <span class="badge bg-secondary"><%= summary[:original_win_pct] %>%</span>
61
+ <%= t("raif.admin.prompt_studio.batch_runs.judge.original_response") %>
62
+ <span class="text-muted">(<%= summary[:original_wins] %>/<%= summary[:total] %>)</span>
63
+ </div>
64
+ <% if summary[:ties] > 0 %>
65
+ <div>
66
+ <span class="badge bg-warning text-dark"><%= summary[:tie_pct] %>%</span>
67
+ <%= t("raif.admin.prompt_studio.batch_runs.judge.tie") %>
68
+ <span class="text-muted">(<%= summary[:ties] %>/<%= summary[:total] %>)</span>
69
+ </div>
70
+ <% end %>
71
+ </div>
72
+ <% end %>
73
+ <% end %>
74
+ </div>
75
+ <% end %>
76
+ </div>
77
+ </div>
78
+ <% end %>
@@ -0,0 +1,49 @@
1
+ <div class="d-flex justify-content-between align-items-center my-4">
2
+ <h1 class="mb-0"><%= t("raif.admin.prompt_studio.batch_runs.show.page_title", id: @batch_run.id) %></h1>
3
+ <%= link_to t("raif.admin.prompt_studio.common.back"),
4
+ raif.admin_prompt_studio_tasks_path(task_type: @batch_run.task_type),
5
+ class: "btn btn-outline-secondary" %>
6
+ </div>
7
+
8
+ <%= turbo_stream_from @batch_run %>
9
+
10
+ <%= render "raif/admin/prompt_studio/batch_runs/progress", batch_run: @batch_run %>
11
+
12
+ <div class="card mb-4">
13
+ <div class="card-header">
14
+ <h5 class="mb-0"><%= t("raif.admin.prompt_studio.batch_runs.show.results") %></h5>
15
+ </div>
16
+ <div class="card-body p-0">
17
+ <div class="table-responsive">
18
+ <table class="table table-striped mb-0">
19
+ <thead>
20
+ <tr>
21
+ <th><%= t("raif.admin.prompt_studio.batch_runs.show.source_task") %></th>
22
+ <th><%= t("raif.admin.common.status") %></th>
23
+ <th><%= t("raif.admin.prompt_studio.batch_runs.show.original_response") %></th>
24
+ <th><%= t("raif.admin.prompt_studio.batch_runs.show.new_response") %></th>
25
+ <% if @batch_run.has_judge? %>
26
+ <th><%= t("raif.admin.prompt_studio.batch_runs.show.judge_result") %></th>
27
+ <th><%= t("raif.admin.prompt_studio.batch_runs.show.judge_reasoning") %></th>
28
+ <% end %>
29
+ </tr>
30
+ </thead>
31
+ <tbody>
32
+ <% @items.each do |item| %>
33
+ <%= render "raif/admin/prompt_studio/batch_runs/batch_run_item", item: item %>
34
+ <% end %>
35
+ </tbody>
36
+ </table>
37
+ </div>
38
+ </div>
39
+ </div>
40
+
41
+ <div class="mt-3"><%== pagy_bootstrap_nav(@pagy) if @pagy.pages > 1 %></div>
42
+
43
+ <% if @batch_run.has_judge? %>
44
+ <% @items.each do |item| %>
45
+ <% if item.judge_summary.present? || item.judge_reasoning.present? %>
46
+ <%= render "raif/admin/prompt_studio/batch_runs/judge_detail_modal", item: item %>
47
+ <% end %>
48
+ <% end %>
49
+ <% end %>
@@ -0,0 +1,48 @@
1
+ <div class="d-flex justify-content-between align-items-center my-4">
2
+ <h1 class="mb-0"><%= t("raif.admin.prompt_studio.common.prompt_studio") %></h1>
3
+ </div>
4
+
5
+ <%= render "raif/admin/prompt_studio/shared/nav_tabs", active_tab: :conversations %>
6
+
7
+ <%= render "raif/admin/prompt_studio/shared/type_filter",
8
+ filter_path: raif.admin_prompt_studio_conversations_path,
9
+ param_name: :conversation_type,
10
+ types: @conversation_types,
11
+ selected_type: @selected_type,
12
+ llm_model_keys: @llm_model_keys %>
13
+
14
+ <% if @selected_type.present? && @conversations.present? %>
15
+ <div class="table-responsive">
16
+ <table class="table table-striped">
17
+ <thead>
18
+ <tr>
19
+ <th><%= t("raif.admin.common.id") %></th>
20
+ <th><%= t("raif.admin.common.type") %></th>
21
+ <th><%= t("raif.admin.common.created_at") %></th>
22
+ <th><%= t("raif.admin.common.model") %></th>
23
+ <th><%= t("raif.admin.common.entries_count") %></th>
24
+ <th></th>
25
+ </tr>
26
+ </thead>
27
+ <tbody>
28
+ <% @conversations.each do |conversation| %>
29
+ <tr>
30
+ <td><%= conversation.id %></td>
31
+ <td><%= conversation.type %></td>
32
+ <td><%= conversation.created_at.rfc822 %></td>
33
+ <td><%= conversation.llm_model_key %></td>
34
+ <td><%= conversation.conversation_entries_count %></td>
35
+ <td>
36
+ <%= link_to "View", raif.admin_prompt_studio_conversation_path(conversation), class: "btn btn-sm btn-outline-primary" %>
37
+ </td>
38
+ </tr>
39
+ <% end %>
40
+ </tbody>
41
+ </table>
42
+ </div>
43
+ <div class="mt-3"><%== pagy_bootstrap_nav(@pagy) if @pagy.pages > 1 %></div>
44
+ <% elsif @selected_type.present? %>
45
+ <div class="alert alert-info">
46
+ <%= t("raif.admin.prompt_studio.common.no_instances") %>
47
+ </div>
48
+ <% end %>
@@ -0,0 +1,36 @@
1
+ <div class="d-flex justify-content-between align-items-center my-4">
2
+ <h1 class="mb-0"><%= t("raif.admin.prompt_studio.conversations.show.page_title", id: @conversation.id) %></h1>
3
+ <%= link_to t("raif.admin.prompt_studio.common.back"),
4
+ raif.admin_prompt_studio_conversations_path(conversation_type: @conversation.type),
5
+ class: "btn btn-outline-secondary" %>
6
+ </div>
7
+
8
+ <div class="card mb-4">
9
+ <div class="card-header">
10
+ <h5 class="mb-0"><%= t("raif.admin.prompt_studio.conversations.show.conversation_details") %></h5>
11
+ </div>
12
+ <div class="card-body">
13
+ <div class="row mb-3">
14
+ <div class="col-md-2"><strong><%= t("raif.admin.common.id") %>:</strong></div>
15
+ <div class="col-md-10"><%= @conversation.id %></div>
16
+ </div>
17
+ <div class="row mb-3">
18
+ <div class="col-md-2"><strong><%= t("raif.admin.common.type") %>:</strong></div>
19
+ <div class="col-md-10"><%= @conversation.type %></div>
20
+ </div>
21
+ <div class="row mb-3">
22
+ <div class="col-md-2"><strong><%= t("raif.admin.common.model") %>:</strong></div>
23
+ <div class="col-md-10"><%= @conversation.llm_model_key %></div>
24
+ </div>
25
+ <div class="row mb-3">
26
+ <div class="col-md-2"><strong><%= t("raif.admin.common.created_at") %>:</strong></div>
27
+ <div class="col-md-10"><%= @conversation.created_at.rfc822 %></div>
28
+ </div>
29
+ <div class="row mb-3">
30
+ <div class="col-md-2"><strong><%= t("raif.admin.common.entries_count") %>:</strong></div>
31
+ <div class="col-md-10"><%= @conversation.conversation_entries_count %></div>
32
+ </div>
33
+ </div>
34
+ </div>
35
+
36
+ <%= render "raif/admin/prompt_studio/shared/prompt_comparison", comparison: @comparison %>
@@ -0,0 +1,17 @@
1
+ <ul class="nav nav-tabs mb-4">
2
+ <li class="nav-item">
3
+ <%= link_to t("raif.admin.common.tasks"),
4
+ raif.admin_prompt_studio_tasks_path,
5
+ class: "nav-link #{active_tab == :tasks ? "active" : ""}" %>
6
+ </li>
7
+ <li class="nav-item">
8
+ <%= link_to t("raif.admin.common.conversations"),
9
+ raif.admin_prompt_studio_conversations_path,
10
+ class: "nav-link #{active_tab == :conversations ? "active" : ""}" %>
11
+ </li>
12
+ <li class="nav-item">
13
+ <%= link_to t("raif.admin.common.agents"),
14
+ raif.admin_prompt_studio_agents_path,
15
+ class: "nav-link #{active_tab == :agents ? "active" : ""}" %>
16
+ </li>
17
+ </ul>
@@ -0,0 +1,87 @@
1
+ <% if comparison[:warnings].any? %>
2
+ <div class="alert <%= comparison[:has_stale_references] ? "alert-danger" : "alert-warning" %>">
3
+ <ul class="mb-0">
4
+ <% comparison[:warnings].each do |warning| %>
5
+ <li><%= warning %></li>
6
+ <% end %>
7
+ </ul>
8
+ </div>
9
+ <% end %>
10
+
11
+ <% if comparison[:original_system_prompt].present? || comparison[:current_system_prompt].present? %>
12
+ <div class="card mb-4">
13
+ <div class="card-header d-flex justify-content-between align-items-center">
14
+ <h5 class="mb-0"><%= t("raif.admin.common.system_prompt") %></h5>
15
+ <% if comparison[:system_prompt_changed] %>
16
+ <span class="badge bg-warning text-dark"><%= t("raif.admin.prompt_studio.common.system_prompt_changed") %></span>
17
+ <% elsif comparison[:current_system_prompt].present? %>
18
+ <span class="badge bg-success"><%= t("raif.admin.prompt_studio.common.system_prompt_unchanged") %></span>
19
+ <% end %>
20
+ </div>
21
+ <div class="card-body">
22
+ <% if comparison[:system_prompt_changed] %>
23
+ <div class="row">
24
+ <div class="col-md-6">
25
+ <h6><%= t("raif.admin.prompt_studio.common.original_system_prompt") %></h6>
26
+ <pre class="pre-wrap border rounded p-3 bg-light"><%= comparison[:original_system_prompt] %></pre>
27
+ </div>
28
+ <div class="col-md-6">
29
+ <h6><%= t("raif.admin.prompt_studio.common.current_system_prompt") %></h6>
30
+ <pre class="pre-wrap border rounded p-3 bg-light"><%= comparison[:current_system_prompt] %></pre>
31
+ </div>
32
+ </div>
33
+ <% else %>
34
+ <pre class="pre-wrap"><%= comparison[:current_system_prompt] || comparison[:original_system_prompt] %></pre>
35
+ <% end %>
36
+ </div>
37
+ </div>
38
+ <% end %>
39
+
40
+ <% if comparison[:original_prompt].present? || comparison[:current_prompt].present? %>
41
+ <div class="card mb-4">
42
+ <div class="card-header d-flex justify-content-between align-items-center">
43
+ <h5 class="mb-0"><%= t("raif.admin.common.prompt") %></h5>
44
+ <% if comparison[:prompt_changed] %>
45
+ <span class="badge bg-warning text-dark"><%= t("raif.admin.prompt_studio.common.prompt_changed") %></span>
46
+ <% elsif comparison[:current_prompt].present? %>
47
+ <span class="badge bg-success"><%= t("raif.admin.prompt_studio.common.prompt_unchanged") %></span>
48
+ <% end %>
49
+ </div>
50
+ <div class="card-body">
51
+ <% if comparison[:prompt_changed] %>
52
+ <div class="row">
53
+ <div class="col-md-6">
54
+ <h6 class="d-flex justify-content-between align-items-center">
55
+ <span><%= t("raif.admin.prompt_studio.common.original_prompt") %></span>
56
+ <span>
57
+ <% if comparison[:original_prompt_tokens].present? %>
58
+ <span class="badge bg-secondary fw-normal"><%= t("raif.admin.prompt_studio.common.prompt_tokens", count: number_with_delimiter(comparison[:original_prompt_tokens])) %></span>
59
+ <% end %>
60
+ <% if comparison[:original_prompt_token_cost].present? %>
61
+ <span class="badge bg-secondary fw-normal"><%= number_to_currency(comparison[:original_prompt_token_cost], precision: 4) %></span>
62
+ <% end %>
63
+ </span>
64
+ </h6>
65
+ <pre class="pre-wrap border rounded p-3 bg-light"><%= comparison[:original_prompt] %></pre>
66
+ </div>
67
+ <div class="col-md-6">
68
+ <h6 class="d-flex justify-content-between align-items-center">
69
+ <span><%= t("raif.admin.prompt_studio.common.current_prompt") %></span>
70
+ <span>
71
+ <% if comparison[:current_prompt_token_estimate].present? %>
72
+ <span class="badge bg-secondary fw-normal"><%= t("raif.admin.prompt_studio.common.estimated_tokens", count: number_with_delimiter(comparison[:current_prompt_token_estimate])) %></span>
73
+ <% end %>
74
+ <% if comparison[:current_prompt_cost_estimate].present? %>
75
+ <span class="badge bg-secondary fw-normal"><%= t("raif.admin.prompt_studio.common.estimated_cost", cost: number_to_currency(comparison[:current_prompt_cost_estimate], precision: 4)) %></span>
76
+ <% end %>
77
+ </span>
78
+ </h6>
79
+ <pre class="pre-wrap border rounded p-3 bg-light"><%= comparison[:current_prompt] %></pre>
80
+ </div>
81
+ </div>
82
+ <% else %>
83
+ <pre class="pre-wrap"><%= comparison[:current_prompt] || comparison[:original_prompt] %></pre>
84
+ <% end %>
85
+ </div>
86
+ </div>
87
+ <% end %>
@@ -0,0 +1,54 @@
1
+ <div class="row mb-4">
2
+ <div class="col-12">
3
+ <%= form_tag filter_path, method: :get, class: "mb-4" do %>
4
+ <div class="row align-items-end">
5
+ <div class="col-md-3">
6
+ <div class="form-group">
7
+ <label for="<%= param_name %>"><%= t("raif.admin.prompt_studio.common.type") %></label>
8
+ <%= select_tag param_name,
9
+ options_for_select(
10
+ types.map { |type| [type, type] },
11
+ selected_type
12
+ ),
13
+ include_blank: t("raif.admin.prompt_studio.common.select_type"),
14
+ class: "form-select",
15
+ data: { controller: "raif--tom-select" } %>
16
+ </div>
17
+ </div>
18
+ <% if local_assigns[:llm_model_keys]&.any? %>
19
+ <div class="col-md-2">
20
+ <div class="form-group">
21
+ <label for="llm_model_key"><%= t("raif.admin.common.model") %></label>
22
+ <%= select_tag :llm_model_key,
23
+ options_for_select(
24
+ llm_model_keys.map { |key| [key, key] },
25
+ params[:llm_model_key]
26
+ ),
27
+ include_blank: t("raif.admin.common.all"),
28
+ class: "form-select",
29
+ data: { controller: "raif--tom-select" } %>
30
+ </div>
31
+ </div>
32
+ <% end %>
33
+ <div class="col-md-2">
34
+ <div class="form-group">
35
+ <label for="created_after"><%= t("raif.admin.prompt_studio.common.created_after") %></label>
36
+ <%= date_field_tag :created_after, params[:created_after], class: "form-control", autocomplete: "off" %>
37
+ </div>
38
+ </div>
39
+ <div class="col-md-2">
40
+ <div class="form-group">
41
+ <label for="created_before"><%= t("raif.admin.prompt_studio.common.created_before") %></label>
42
+ <%= date_field_tag :created_before, params[:created_before], class: "form-control", autocomplete: "off" %>
43
+ </div>
44
+ </div>
45
+ <div class="col-md-auto">
46
+ <%= submit_tag t("raif.admin.common.filter"), class: "btn btn-primary" %>
47
+ </div>
48
+ <% if local_assigns[:extra_buttons] %>
49
+ <%= extra_buttons %>
50
+ <% end %>
51
+ </div>
52
+ <% end %>
53
+ </div>
54
+ </div>