raif 1.4.0 → 1.5.0

data/lib/raif/llm_registry.rb

@@ -41,6 +41,13 @@ module Raif
 
   def self.default_llms
     open_ai_models = [
+      {
+        key: :open_ai_gpt_5_4,
+        api_name: "gpt-5.4",
+        input_token_cost: 2.5 / 1_000_000,
+        output_token_cost: 15.0 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
       {
         key: :open_ai_gpt_5_2,
         api_name: "gpt-5.2",
@@ -48,6 +55,13 @@ module Raif
         output_token_cost: 14.0 / 1_000_000,
         model_provider_settings: { supports_temperature: false },
       },
+      {
+        key: :open_ai_gpt_5_3,
+        api_name: "gpt-5.3",
+        input_token_cost: 1.75 / 1_000_000,
+        output_token_cost: 14.0 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
       {
         key: :open_ai_gpt_5_1,
         api_name: "gpt-5.1",
@@ -62,6 +76,20 @@ module Raif
         output_token_cost: 10.0 / 1_000_000,
         model_provider_settings: { supports_temperature: false },
       },
+      {
+        key: :open_ai_gpt_5_4_mini,
+        api_name: "gpt-5.4-mini",
+        input_token_cost: 0.75 / 1_000_000,
+        output_token_cost: 4.5 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
+      {
+        key: :open_ai_gpt_5_4_nano,
+        api_name: "gpt-5.4-nano",
+        input_token_cost: 0.20 / 1_000_000,
+        output_token_cost: 1.25 / 1_000_000,
+        model_provider_settings: { supports_temperature: false },
+      },
       {
         key: :open_ai_gpt_5_mini,
         api_name: "gpt-5-mini",
@@ -123,8 +151,8 @@ module Raif
       {
         key: :open_ai_o1_mini,
         api_name: "o1-mini",
-        input_token_cost: 1.5 / 1_000_000,
-        output_token_cost: 6.0 / 1_000_000,
+        input_token_cost: 1.1 / 1_000_000,
+        output_token_cost: 4.4 / 1_000_000,
         model_provider_settings: { supports_temperature: false },
       },
       {
@@ -164,7 +192,7 @@ module Raif
     # o1-mini is not supported by the OpenAI Responses API.
     open_ai_responses_models.delete_if{|model| model[:key] == :open_ai_o1_mini }
 
-    # o1-pro and o3-pro are not supported by the OpenAI Completions API, but it is supported by the OpenAI Responses API.
+    # These models are not supported by the OpenAI Completions API, but are supported by the OpenAI Responses API.
     open_ai_responses_models << {
       key: :open_ai_responses_o1_pro,
       api_name: "o1-pro",
@@ -181,10 +209,79 @@ module Raif
       model_provider_settings: { supports_temperature: false },
     }
 
+    open_ai_responses_models << {
+      key: :open_ai_responses_gpt_5_pro,
+      api_name: "gpt-5-pro",
+      input_token_cost: 15.0 / 1_000_000,
+      output_token_cost: 120.0 / 1_000_000,
+      model_provider_settings: { supports_temperature: false },
+    }
+
+    open_ai_responses_models << {
+      key: :open_ai_responses_gpt_5_2_pro,
+      api_name: "gpt-5.2-pro",
+      input_token_cost: 21.0 / 1_000_000,
+      output_token_cost: 168.0 / 1_000_000,
+      model_provider_settings: { supports_temperature: false },
+    }
+
+    open_ai_responses_models << {
+      key: :open_ai_responses_gpt_5_4_pro,
+      api_name: "gpt-5.4-pro",
+      input_token_cost: 30.0 / 1_000_000,
+      output_token_cost: 180.0 / 1_000_000,
+      model_provider_settings: { supports_temperature: false, supports_structured_outputs: false },
+    }
+
     {
       Raif::Llms::OpenAiCompletions => open_ai_models,
       Raif::Llms::OpenAiResponses => open_ai_responses_models,
       Raif::Llms::Anthropic => [
+        {
+          key: :anthropic_claude_4_7_opus,
+          api_name: "claude-opus-4-7",
+          input_token_cost: 5.0 / 1_000_000,
+          output_token_cost: 25.0 / 1_000_000,
+          max_completion_tokens: 128_000,
+          model_provider_settings: { supports_temperature: false },
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :anthropic_claude_4_6_opus,
+          api_name: "claude-opus-4-6",
+          input_token_cost: 5.0 / 1_000_000,
+          output_token_cost: 25.0 / 1_000_000,
+          max_completion_tokens: 128_000,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :anthropic_claude_4_6_sonnet,
+          api_name: "claude-sonnet-4-6",
+          input_token_cost: 3.0 / 1_000_000,
+          output_token_cost: 15.0 / 1_000_000,
+          max_completion_tokens: 64_000,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :anthropic_claude_4_5_opus,
+          api_name: "claude-opus-4-5",
+          input_token_cost: 5.0 / 1_000_000,
+          output_token_cost: 25.0 / 1_000_000,
+          max_completion_tokens: 64_000,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
         {
           key: :anthropic_claude_4_5_sonnet,
           api_name: "claude-sonnet-4-5",
@@ -223,7 +320,7 @@ module Raif
           api_name: "claude-opus-4-20250514",
           input_token_cost: 15.0 / 1_000_000,
           output_token_cost: 75.0 / 1_000_000,
-          max_completion_tokens: 8192,
+          max_completion_tokens: 32_000,
           supported_provider_managed_tools: [
             Raif::ModelTools::ProviderManaged::WebSearch,
             Raif::ModelTools::ProviderManaged::CodeExecution
@@ -234,7 +331,7 @@ module Raif
           api_name: "claude-sonnet-4-20250514",
           input_token_cost: 3.0 / 1_000_000,
           output_token_cost: 15.0 / 1_000_000,
-          max_completion_tokens: 8192,
+          max_completion_tokens: 64_000,
           supported_provider_managed_tools: [
             Raif::ModelTools::ProviderManaged::WebSearch,
             Raif::ModelTools::ProviderManaged::CodeExecution
@@ -282,6 +379,34 @@ module Raif
         },
       ],
       Raif::Llms::Bedrock => [
+        {
+          key: :bedrock_claude_4_7_opus,
+          api_name: "anthropic.claude-opus-4-7",
+          input_token_cost: 0.005 / 1000,
+          output_token_cost: 0.025 / 1000,
+          max_completion_tokens: 128_000
+        },
+        {
+          key: :bedrock_claude_4_6_opus,
+          api_name: "anthropic.claude-opus-4-6-v1",
+          input_token_cost: 0.005 / 1000,
+          output_token_cost: 0.025 / 1000,
+          max_completion_tokens: 128_000
+        },
+        {
+          key: :bedrock_claude_4_6_sonnet,
+          api_name: "anthropic.claude-sonnet-4-6",
+          input_token_cost: 0.003 / 1000,
+          output_token_cost: 0.015 / 1000,
+          max_completion_tokens: 64_000
+        },
+        {
+          key: :bedrock_claude_4_5_opus,
+          api_name: "anthropic.claude-opus-4-5-20251101-v1:0",
+          input_token_cost: 0.005 / 1000,
+          output_token_cost: 0.025 / 1000,
+          max_completion_tokens: 64_000
+        },
         {
           key: :bedrock_claude_4_5_sonnet,
           api_name: "anthropic.claude-sonnet-4-5-20250929-v1:0",
@@ -308,14 +433,14 @@ module Raif
           api_name: "anthropic.claude-sonnet-4-20250514-v1:0",
           input_token_cost: 0.003 / 1000,
           output_token_cost: 0.015 / 1000,
-          max_completion_tokens: 8192
+          max_completion_tokens: 64_000
         },
         {
           key: :bedrock_claude_4_opus,
           api_name: "anthropic.claude-opus-4-20250514-v1:0",
           input_token_cost: 0.015 / 1000,
           output_token_cost: 0.075 / 1000,
-          max_completion_tokens: 8192
+          max_completion_tokens: 32_000
         },
         {
           key: :bedrock_claude_3_7_sonnet,
@@ -365,6 +490,34 @@ module Raif
           input_token_cost: 0.0002625 / 1000,
           output_token_cost: 0.0042 / 1000,
           max_completion_tokens: 4096
+        },
+        {
+          key: :bedrock_deepseek_v3_2,
+          api_name: "deepseek.v3.2",
+          input_token_cost: 0.62 / 1_000_000,
+          output_token_cost: 1.85 / 1_000_000,
+          max_completion_tokens: 32_768
+        },
+        {
+          key: :bedrock_deepseek_r1,
+          api_name: "deepseek.r1-v1:0",
+          input_token_cost: 0.00135 / 1_000,
+          output_token_cost: 0.0054 / 1_000,
+          max_completion_tokens: 32_768
+        },
+        {
+          key: :bedrock_gpt_oss_120b,
+          api_name: "openai.gpt-oss-120b-1:0",
+          input_token_cost: 0.15 / 1_000_000,
+          output_token_cost: 0.60 / 1_000_000,
+          max_completion_tokens: 32_768
+        },
+        {
+          key: :bedrock_gpt_oss_20b,
+          api_name: "openai.gpt-oss-20b-1:0",
+          input_token_cost: 0.07 / 1_000_000,
+          output_token_cost: 0.30 / 1_000_000,
+          max_completion_tokens: 32_768
         }
       ],
       Raif::Llms::OpenRouter => [
@@ -377,8 +530,8 @@ module Raif
         {
           key: :open_router_deepseek_chat_v3,
           api_name: "deepseek/deepseek-chat-v3-0324",
-          input_token_cost: 0.27 / 1_000_000,
-          output_token_cost: 1.1 / 1_000_000,
+          input_token_cost: 0.20 / 1_000_000,
+          output_token_cost: 0.77 / 1_000_000,
         },
         {
           key: :open_router_deepseek_v3_1,
@@ -386,6 +539,12 @@ module Raif
           input_token_cost: 0.25 / 1_000_000,
           output_token_cost: 1.0 / 1_000_000,
         },
+        {
+          key: :open_router_deepseek_v3_2,
+          api_name: "deepseek/deepseek-v3.2",
+          input_token_cost: 0.26 / 1_000_000,
+          output_token_cost: 0.38 / 1_000_000,
+        },
         {
           key: :open_router_gemini_2_0_flash,
           api_name: "google/gemini-2.0-flash-001",
@@ -410,6 +569,18 @@ module Raif
           input_token_cost: 2.0 / 1_000_000,
           output_token_cost: 12.0 / 1_000_000,
         },
+        {
+          key: :open_router_gemini_3_1_pro_preview,
+          api_name: "google/gemini-3.1-pro-preview",
+          input_token_cost: 2.0 / 1_000_000,
+          output_token_cost: 12.0 / 1_000_000,
+        },
+        {
+          key: :open_router_gemini_3_1_flash_lite_preview,
+          api_name: "google/gemini-3.1-flash-lite-preview",
+          input_token_cost: 0.25 / 1_000_000,
+          output_token_cost: 1.5 / 1_000_000,
+        },
         {
           key: :open_router_grok_4,
           api_name: "x-ai/grok-4",
@@ -428,6 +599,12 @@ module Raif
           input_token_cost: 0.45 / 1_000_000,
           output_token_cost: 2.35 / 1_000_000,
         },
+        {
+          key: :open_router_kimi_k2_5,
+          api_name: "moonshotai/kimi-k2.5",
+          input_token_cost: 0.45 / 1_000_000,
+          output_token_cost: 2.20 / 1_000_000,
+        },
         {
           key: :open_router_llama_3_3_70b_instruct,
           api_name: "meta-llama/llama-3.3-70b-instruct",
@@ -458,6 +635,18 @@ module Raif
           input_token_cost: 0.255 / 1_000_000,
           output_token_cost: 1.02 / 1_000_000,
         },
+        {
+          key: :open_router_minimax_m2_1,
+          api_name: "minimax/minimax-m2.1",
+          input_token_cost: 0.27 / 1_000_000,
+          output_token_cost: 0.95 / 1_000_000,
+        },
+        {
+          key: :open_router_minimax_m2_5,
+          api_name: "minimax/minimax-m2.5",
+          input_token_cost: 0.27 / 1_000_000,
+          output_token_cost: 0.95 / 1_000_000,
+        },
         {
           key: :open_router_mistral_large_3_2512,
           api_name: "mistralai/mistral-large-2512",
@@ -482,8 +671,40 @@ module Raif
           input_token_cost: 0.05 / 1_000_000,
           output_token_cost: 0.2 / 1_000_000,
         },
+        {
+          key: :open_router_grok_4_20,
+          api_name: "x-ai/grok-4.20",
+          input_token_cost: 2.0 / 1_000_000,
+          output_token_cost: 6.0 / 1_000_000,
+        },
+        {
+          key: :open_router_google_gemma_4_31b_it,
+          api_name: "google/gemma-4-31b-it",
+          input_token_cost: 0.14 / 1_000_000,
+          output_token_cost: 0.40 / 1_000_000,
+        },
       ],
       Raif::Llms::Google => [
+        {
+          key: :google_gemini_3_1_pro,
+          api_name: "gemini-3.1-pro-preview",
+          input_token_cost: 2.0 / 1_000_000,
+          output_token_cost: 12.0 / 1_000_000,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
+        {
+          key: :google_gemini_3_1_flash_lite,
+          api_name: "gemini-3.1-flash-lite-preview",
+          input_token_cost: 0.25 / 1_000_000,
+          output_token_cost: 1.5 / 1_000_000,
+          supported_provider_managed_tools: [
+            Raif::ModelTools::ProviderManaged::WebSearch,
+            Raif::ModelTools::ProviderManaged::CodeExecution
+          ]
+        },
         {
           key: :google_gemini_3_0_pro,
           api_name: "gemini-3-pro-preview",

data/lib/raif/prompt_studio_comparison_builder.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Raif
+  class PromptStudioComparisonBuilder
+    # Attempts to rebuild the prompt from current code for a given record.
+    # Returns a hash with the rendered prompts and any warnings.
+    def self.build(record)
+      new(record).build
+    end
+
+    def initialize(record)
+      @record = record
+    end
+
+    def build
+      {
+        original_prompt: original_prompt,
+        original_system_prompt: original_system_prompt,
+        current_prompt: current_prompt,
+        current_system_prompt: current_system_prompt,
+        prompt_changed: changed?(original_prompt, current_prompt),
+        system_prompt_changed: changed?(original_system_prompt, current_system_prompt),
+        has_stale_references: has_stale_references?,
+        warnings: warnings,
+        original_prompt_tokens: original_prompt_tokens,
+        original_prompt_token_cost: original_prompt_token_cost,
+        current_prompt_token_estimate: current_prompt_token_estimate,
+        current_prompt_cost_estimate: current_prompt_cost_estimate
+      }
+    end
+
+  private
+
+    def original_prompt
+      @original_prompt ||= @record.respond_to?(:prompt) ? @record.prompt : nil
+    end
+
+    def original_system_prompt
+      @original_system_prompt ||= @record.system_prompt
+    end
+
+    def current_prompt
+      return @current_prompt if defined?(@current_prompt)
+
+      @current_prompt = begin
+        @record.build_prompt
+      rescue NotImplementedError
+        nil
+      rescue => e
+        warnings << "Error rendering current prompt: #{e.message}"
+        nil
+      end
+    end
+
+    def current_system_prompt
+      return @current_system_prompt if defined?(@current_system_prompt)
+
+      @current_system_prompt = begin
+        @record.build_system_prompt
+      rescue NotImplementedError
+        nil
+      rescue => e
+        warnings << "Error rendering current system prompt: #{e.message}"
+        nil
+      end
+    end
+
+    def warnings
+      @warnings ||= [].tap do |w|
+        w << I18n.t("raif.admin.prompt_studio.common.warning_stale_reference") if has_stale_references?
+      end
+    end
+
+    def has_stale_references?
+      return @has_stale_references if defined?(@has_stale_references)
+
+      @has_stale_references = detect_stale_references
+    end
+
+    def detect_stale_references
+      return false unless @record.respond_to?(:run_with) && @record.run_with.present?
+
+      @record.run_with.each do |_key, value|
+        if value.is_a?(String) && value.start_with?("gid://")
+          begin
+            return true if GlobalID::Locator.locate(value).nil?
+          rescue StandardError
+            return true
+          end
+        end
+      end
+
+      false
+    end
+
+    def changed?(original, current)
+      original.present? && current.present? && original.strip != current.strip
+    end
+
+    def original_prompt_tokens
+      return unless @record.respond_to?(:raif_model_completion)
+
+      @record.raif_model_completion&.prompt_tokens
+    end
+
+    def original_prompt_token_cost
+      return unless @record.respond_to?(:raif_model_completion)
+
+      @record.raif_model_completion&.prompt_token_cost
+    end
+
+    def current_prompt_token_estimate
+      return unless prompt_changed? || system_prompt_changed?
+
+      Raif::TokenEstimator.estimate_tokens(current_system_prompt, current_prompt)
+    end
+
+    def current_prompt_cost_estimate
+      token_estimate = current_prompt_token_estimate
+      return unless token_estimate
+
+      return unless @record.llm_model_key.present?
+
+      llm_config = Raif.llm_config(@record.llm_model_key.to_sym)
+      return unless llm_config&.dig(:input_token_cost)
+
+      llm_config[:input_token_cost] * token_estimate
+    end
+
+    def prompt_changed?
+      changed?(original_prompt, current_prompt)
+    end
+
+    def system_prompt_changed?
+      changed?(original_system_prompt, current_system_prompt)
+    end
+  end
+end

data/lib/raif/token_estimator.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Raif
+  class TokenEstimator
+    def self.available?
+      return true if defined?(::Tiktoken)
+
+      require "tiktoken_ruby"
+      !!defined?(::Tiktoken)
+    rescue LoadError
+      false
+    end
+
+    # Estimates the total token count for a prompt + system prompt combination.
+    # Returns nil if tiktoken_ruby is not installed.
+    def self.estimate_tokens(*texts)
+      return unless available?
+
+      encoder = encoder_for_model("gpt-4")
+      texts.compact.sum { |text| encoder.encode(text).length }
+    end
+
+    def self.encoder_for_model(model)
+      @encoders ||= {}
+      @encoders[model] ||= ::Tiktoken.encoding_for_model(model)
+    end
+  end
+end

data/lib/raif/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Raif
-  VERSION = "1.4.0"
+  VERSION = "1.5.0"
 end

data/lib/raif.rb

@@ -11,6 +11,8 @@ require "raif/embedding_model_registry"
 require "raif/json_schema_builder"
 require "raif/migration_checker"
 require "raif/messages"
+require "raif/prompt_studio_comparison_builder"
+require "raif/token_estimator"
 
 require "faraday"
 require "event_stream_parser"

data/spec/support/rspec_helpers.rb

@@ -6,7 +6,7 @@ module Raif
     def stubbed_llm(llm_model_key, source_instance, &block)
       test_llm = Raif.llm(llm_model_key.to_sym)
 
-      allow(test_llm).to receive(:perform_model_completion!) do |model_completion|
+      allow(test_llm).to receive(:perform_model_completion!) do |model_completion, &streaming_block|
         result = block.call(model_completion.messages, model_completion, source_instance)
         model_completion.raw_response = result if result.is_a?(String)
         model_completion.completion_tokens = rand(100..2000)
@@ -14,6 +14,12 @@ module Raif
         model_completion.total_tokens = model_completion.completion_tokens + model_completion.prompt_tokens
         model_completion.save!
 
+        if streaming_block && result.is_a?(String)
+          result.chars.each_slice(25) do |chunk|
+            streaming_block.call(model_completion, chunk.join, nil)
+          end
+        end
+
         model_completion
       end
 

data/spec/support/test_task.rb

@@ -30,6 +30,15 @@ class Raif::TestJsonTask < Raif::Task
   end
 end
 
+class Raif::TestCachedTask < Raif::Task
+  enable_anthropic_prompt_caching
+  enable_bedrock_prompt_caching
+
+  def build_prompt
+    "Tell me a joke"
+  end
+end
+
 class Raif::TestHtmlTask < Raif::Task
   llm_response_format :html
   llm_response_allowed_tags %w[p b i u s a]

data/spec/support/test_template_task.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+class Raif::TestTemplateTask < Raif::Task
+  run_with :topic
+
+  after_initialize -> { self.topic ||= "pirates" }
+
+  def topic_description
+    "the topic of #{topic}"
+  end
+end
+
+class Raif::TestTemplateSystemPromptTask < Raif::Task
+  run_with :persona
+
+  after_initialize -> { self.persona ||= "comedian" }
+
+  def build_prompt
+    "Tell me a joke"
+  end
+end
+
+class Raif::TestTemplateConversation < Raif::Conversation
+  attr_writer :persona
+
+  def persona
+    @persona || "helpful assistant"
+  end
+end
+
+class Raif::TestTemplateWithPartialTask < Raif::Task
+  run_with :topic
+
+  after_initialize -> { self.topic ||= "dogs" }
+end
+
+class Raif::TestTemplateAgent < Raif::Agent
+  run_with :agent_role
+
+  after_initialize -> { self.agent_role ||= "researcher" }
+end