raif 1.4.0 → 1.5.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: raif
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Ben Roesch
@@ -152,11 +152,21 @@ files:
 - Rakefile
 - app/assets/builds/raif.css
 - app/assets/builds/raif_admin.css
+- app/assets/builds/raif_admin_sprockets.js
 - app/assets/config/raif_manifest.js
 - app/assets/images/raif-logo-white.svg
 - app/assets/javascript/raif.js
+- app/assets/javascript/raif/admin/copy_to_clipboard_controller.js
+- app/assets/javascript/raif/admin/cost_estimate_controller.js
+- app/assets/javascript/raif/admin/judge_config_controller.js
+- app/assets/javascript/raif/admin/select_all_checkboxes_controller.js
+- app/assets/javascript/raif/admin/sortable_table_controller.js
+- app/assets/javascript/raif/admin/table_search_controller.js
+- app/assets/javascript/raif/admin/tom_select_controller.js
 - app/assets/javascript/raif/controllers/conversations_controller.js
 - app/assets/javascript/raif/stream_actions/raif_scroll_to_bottom.js
+- app/assets/javascript/raif_admin.js
+- app/assets/javascript/raif_admin_sprockets.js
 - app/assets/stylesheets/raif.scss
 - app/assets/stylesheets/raif/admin/conversation.scss
 - app/assets/stylesheets/raif/admin/stats.scss
@@ -167,8 +177,14 @@ files:
 - app/controllers/raif/admin/application_controller.rb
 - app/controllers/raif/admin/configs_controller.rb
 - app/controllers/raif/admin/conversations_controller.rb
+- app/controllers/raif/admin/llms_controller.rb
 - app/controllers/raif/admin/model_completions_controller.rb
 - app/controllers/raif/admin/model_tool_invocations_controller.rb
+- app/controllers/raif/admin/prompt_studio/agents_controller.rb
+- app/controllers/raif/admin/prompt_studio/base_controller.rb
+- app/controllers/raif/admin/prompt_studio/batch_runs_controller.rb
+- app/controllers/raif/admin/prompt_studio/conversations_controller.rb
+- app/controllers/raif/admin/prompt_studio/tasks_controller.rb
 - app/controllers/raif/admin/stats/model_tool_invocations_controller.rb
 - app/controllers/raif/admin/stats/tasks_controller.rb
 - app/controllers/raif/admin/stats_controller.rb
@@ -180,6 +196,9 @@ files:
 - app/helpers/raif/shared/conversations_helper.rb
 - app/jobs/raif/application_job.rb
 - app/jobs/raif/conversation_entry_job.rb
+- app/jobs/raif/prompt_studio_batch_run_item_job.rb
+- app/jobs/raif/prompt_studio_batch_run_job.rb
+- app/jobs/raif/prompt_studio_task_run_job.rb
 - app/models/raif/admin/task_stat.rb
 - app/models/raif/agent.rb
 - app/models/raif/agents/native_tool_calling_agent.rb
@@ -188,9 +207,12 @@ files:
 - app/models/raif/concerns/boolean_timestamp.rb
 - app/models/raif/concerns/has_available_model_tools.rb
 - app/models/raif/concerns/has_llm.rb
+- app/models/raif/concerns/has_prompt_templates.rb
 - app/models/raif/concerns/has_requested_language.rb
+- app/models/raif/concerns/has_runtime_duration.rb
 - app/models/raif/concerns/invokes_model_tools.rb
 - app/models/raif/concerns/json_schema_definition.rb
+- app/models/raif/concerns/llm_prompt_caching.rb
 - app/models/raif/concerns/llm_response_parsing.rb
 - app/models/raif/concerns/llm_temperature.rb
 - app/models/raif/concerns/llms/anthropic/message_formatting.rb
@@ -210,12 +232,20 @@ files:
 - app/models/raif/concerns/llms/open_ai_responses/message_formatting.rb
 - app/models/raif/concerns/llms/open_ai_responses/response_tool_calls.rb
 - app/models/raif/concerns/llms/open_ai_responses/tool_formatting.rb
+- app/models/raif/concerns/provider_managed_tool_calls.rb
 - app/models/raif/concerns/run_with.rb
 - app/models/raif/conversation.rb
 - app/models/raif/conversation_entry.rb
 - app/models/raif/embedding_model.rb
 - app/models/raif/embedding_models/bedrock.rb
+- app/models/raif/embedding_models/google.rb
 - app/models/raif/embedding_models/open_ai.rb
+- app/models/raif/evals/llm_judge.rb
+- app/models/raif/evals/llm_judges/binary.rb
+- app/models/raif/evals/llm_judges/comparative.rb
+- app/models/raif/evals/llm_judges/scored.rb
+- app/models/raif/evals/llm_judges/summarization.rb
+- app/models/raif/evals/scoring_rubric.rb
 - app/models/raif/llm.rb
 - app/models/raif/llms/anthropic.rb
 - app/models/raif/llms/bedrock.rb
@@ -236,6 +266,8 @@ files:
 - app/models/raif/model_tools/provider_managed/image_generation.rb
 - app/models/raif/model_tools/provider_managed/web_search.rb
 - app/models/raif/model_tools/wikipedia_search.rb
+- app/models/raif/prompt_studio_batch_run.rb
+- app/models/raif/prompt_studio_batch_run_item.rb
 - app/models/raif/streaming_responses/anthropic.rb
 - app/models/raif/streaming_responses/bedrock.rb
 - app/models/raif/streaming_responses/google.rb
@@ -253,6 +285,7 @@ files:
 - app/views/raif/admin/conversations/_conversation_entry.html.erb
 - app/views/raif/admin/conversations/index.html.erb
 - app/views/raif/admin/conversations/show.html.erb
+- app/views/raif/admin/llms/index.html.erb
 - app/views/raif/admin/model_completions/_model_completion.html.erb
 - app/views/raif/admin/model_completions/index.html.erb
 - app/views/raif/admin/model_completions/show.html.erb
@@ -261,6 +294,25 @@ files:
 - app/views/raif/admin/model_tool_invocations/show.html.erb
 - app/views/raif/admin/model_tools/_list.html.erb
 - app/views/raif/admin/model_tools/_model_tool.html.erb
+- app/views/raif/admin/prompt_studio/agents/index.html.erb
+- app/views/raif/admin/prompt_studio/agents/show.html.erb
+- app/views/raif/admin/prompt_studio/batch_runs/_batch_run_item.html.erb
+- app/views/raif/admin/prompt_studio/batch_runs/_judge_config_fields.html.erb
+- app/views/raif/admin/prompt_studio/batch_runs/_judge_detail_modal.html.erb
+- app/views/raif/admin/prompt_studio/batch_runs/_modal.html.erb
+- app/views/raif/admin/prompt_studio/batch_runs/_progress.html.erb
+- app/views/raif/admin/prompt_studio/batch_runs/show.html.erb
+- app/views/raif/admin/prompt_studio/conversations/index.html.erb
+- app/views/raif/admin/prompt_studio/conversations/show.html.erb
+- app/views/raif/admin/prompt_studio/shared/_nav_tabs.html.erb
+- app/views/raif/admin/prompt_studio/shared/_prompt_comparison.html.erb
+- app/views/raif/admin/prompt_studio/shared/_type_filter.html.erb
+- app/views/raif/admin/prompt_studio/tasks/_task_result.html.erb
+- app/views/raif/admin/prompt_studio/tasks/_task_row.html.erb
+- app/views/raif/admin/prompt_studio/tasks/_task_type_filter.html.erb
+- app/views/raif/admin/prompt_studio/tasks/_tasks_table.html.erb
+- app/views/raif/admin/prompt_studio/tasks/index.html.erb
+- app/views/raif/admin/prompt_studio/tasks/show.html.erb
 - app/views/raif/admin/stats/_stats_tile.html.erb
 - app/views/raif/admin/stats/index.html.erb
 - app/views/raif/admin/stats/model_tool_invocations/index.html.erb
@@ -314,16 +366,22 @@ files:
 - db/migrate/20260119000000_add_failure_tracking_to_raif_model_completions.rb
 - db/migrate/20260119000001_add_completed_at_to_raif_model_completions.rb
 - db/migrate/20260119000002_add_started_at_to_raif_model_completions.rb
+- db/migrate/20260307000000_add_prompt_studio_run_to_raif_tasks.rb
+- db/migrate/20260308000000_create_raif_prompt_studio_batch_runs.rb
+- db/migrate/20260308000001_create_raif_prompt_studio_batch_run_items.rb
+- db/migrate/20260407000000_add_cache_token_columns_to_raif_model_completions.rb
 - exe/raif
 - lib/generators/raif/agent/agent_generator.rb
 - lib/generators/raif/agent/templates/agent.rb.tt
 - lib/generators/raif/agent/templates/agent_eval_set.rb.tt
 - lib/generators/raif/agent/templates/application_agent.rb.tt
+- lib/generators/raif/agent/templates/system_prompt.erb.tt
 - lib/generators/raif/base_generator.rb
 - lib/generators/raif/conversation/conversation_generator.rb
 - lib/generators/raif/conversation/templates/application_conversation.rb.tt
 - lib/generators/raif/conversation/templates/conversation.rb.tt
 - lib/generators/raif/conversation/templates/conversation_eval_set.rb.tt
+- lib/generators/raif/conversation/templates/system_prompt.erb.tt
 - lib/generators/raif/eval_set/eval_set_generator.rb
 - lib/generators/raif/eval_set/templates/eval_set.rb.tt
 - lib/generators/raif/evals/setup/setup_generator.rb
@@ -334,6 +392,7 @@ files:
 - lib/generators/raif/model_tool/templates/model_tool_invocation_partial.html.erb.tt
 - lib/generators/raif/task/task_generator.rb
 - lib/generators/raif/task/templates/application_task.rb.tt
+- lib/generators/raif/task/templates/prompt.erb.tt
 - lib/generators/raif/task/templates/task.rb.tt
 - lib/generators/raif/task/templates/task_eval_set.rb.tt
 - lib/generators/raif/views_generator.rb
@@ -347,6 +406,7 @@ files:
 - lib/raif/engine.rb
 - lib/raif/errors.rb
 - lib/raif/errors/action_not_authorized_error.rb
+- lib/raif/errors/blank_response_error.rb
 - lib/raif/errors/instance_dependent_schema_error.rb
 - lib/raif/errors/invalid_config_error.rb
 - lib/raif/errors/invalid_conversation_type_error.rb
@@ -354,6 +414,7 @@ files:
 - lib/raif/errors/invalid_model_image_input_error.rb
 - lib/raif/errors/invalid_user_tool_type_error.rb
 - lib/raif/errors/open_ai/json_schema_error.rb
+- lib/raif/errors/prompt_template_error.rb
 - lib/raif/errors/streaming_error.rb
 - lib/raif/errors/unsupported_feature_error.rb
 - lib/raif/evals.rb
@@ -362,19 +423,15 @@ files:
 - lib/raif/evals/eval_sets/expectations.rb
 - lib/raif/evals/eval_sets/llm_judge_expectations.rb
 - lib/raif/evals/expectation_result.rb
-- lib/raif/evals/llm_judge.rb
-- lib/raif/evals/llm_judges/binary.rb
-- lib/raif/evals/llm_judges/comparative.rb
-- lib/raif/evals/llm_judges/scored.rb
-- lib/raif/evals/llm_judges/summarization.rb
 - lib/raif/evals/run.rb
-- lib/raif/evals/scoring_rubric.rb
 - lib/raif/json_schema_builder.rb
 - lib/raif/languages.rb
 - lib/raif/llm_registry.rb
 - lib/raif/messages.rb
 - lib/raif/migration_checker.rb
+- lib/raif/prompt_studio_comparison_builder.rb
 - lib/raif/rspec.rb
+- lib/raif/token_estimator.rb
 - lib/raif/utils.rb
 - lib/raif/utils/colors.rb
 - lib/raif/utils/html_fragment_processor.rb
@@ -391,6 +448,7 @@ files:
 - spec/support/test_llm.rb
 - spec/support/test_model_tool.rb
 - spec/support/test_task.rb
+- spec/support/test_template_task.rb
 homepage: https://github.com/cultivatelabs/raif
 licenses:
 - MIT

data/lib/raif/evals/llm_judge.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Raif
-  module Evals
-    class LlmJudge < Raif::Task
-      # Set default temperature for consistent judging
-      llm_temperature 0.0
-
-      # Default to JSON response format for structured output
-      llm_response_format :json
-
-      run_with :content_to_judge # the content to judge
-      run_with :additional_context # additional context to be provided to the judge
-
-      def default_llm_model_key
-        Raif.config.evals_default_llm_judge_model_key || super
-      end
-
-      def judgment_reasoning
-        parsed_response["reasoning"] if completed?
-      end
-
-      def judgment_confidence
-        parsed_response["confidence"] if completed?
-      end
-
-      def low_confidence?
-        judgment_confidence && judgment_confidence < 0.5
-      end
-    end
-  end
-end