completion-kit 0.28.34 → 0.28.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b098378de17e0d51976041f5376929f0fd220bc23e86f1044bf89133b3b7210e
-  data.tar.gz: 65742e5a42a70e82fe3f4216652e8a80e4b08552eb54cc4a7e8621969a8dd26f
+  metadata.gz: f506b4c95279cd20d1085f9f3a2eb3eed2d30b03694d3232977a410044a36f8d
+  data.tar.gz: 4a5ea855e4fbb830f0df97dd1f957b86894710464bf011e3c17878cfaad6c416
 SHA512:
-  metadata.gz: 3ac30e089d1ec6597556b0f4744315d07f1bf94207e174055a6e7ae51ee09f88b180a681ee79cd671cf1887cfee7cd1e99584adda5e95411d8a09ad4c96d5071
-  data.tar.gz: ba7069754a2f1048da264f66a29e630f220225be24a7078bc4bc6b5a5004d328b02e42046b6b58c5a53a294d293fd386b81eccfbbd7f3b7cfd59d5b545062741
+  metadata.gz: 561278c7bf724b085e2cc05e3204d89b839e16803de24f9accfa59630edba15e4bdf85a9edbd745e03c9385807295749b95fda1b757b62c781265ebc705ca795
+  data.tar.gz: d2744c24f3eff96c4290bfa333145fa45941c6bf3d26cf7768968e452c6bba4d85c1668be97a8054ef1548aec9e55f782af081b82974c3c6d7a12dae6122ee41

data/app/controllers/completion_kit/runs_controller.rb

@@ -222,7 +222,10 @@ module CompletionKit
     end
 
     def run_params
-      params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :max_tokens, :judge_temperature, :output_column, :expected_column, metric_ids: [], tag_names: [])
+      permitted = params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :max_tokens, :judge_temperature, :output_column, :expected_column, :omit_temperature, metric_ids: [], tag_names: [])
+      omit = permitted.delete(:omit_temperature)
+      permitted[:temperature] = nil if omit == "1"
+      permitted
     end
 
     # Editing a run that already has results forks a new run — but only when a

data/app/jobs/completion_kit/judge_review_job.rb

@@ -75,6 +75,10 @@ module CompletionKit
         record_terminal_failure!(e)
         enqueue_completion_check
         return
+      ensure
+        if judge.temperature_dropped? && !run.judge_temperature_ignored?
+          run.update_columns(judge_temperature_ignored: true)
+        end
       end
 
       review = response.reviews.find_or_initialize_by(metric_id: metric.id)

data/app/models/completion_kit/run.rb

@@ -337,7 +337,9 @@ module CompletionKit
             progress_current: 0,
             progress_total: 0,
             failure_summary: nil,
-            error_message: nil
+            error_message: nil,
+            temperature_ignored: false,
+            judge_temperature_ignored: false
           )
         end
       rescue ActiveRecord::RecordInvalid => e
@@ -436,7 +438,7 @@ module CompletionKit
           error_status: nil,
           error_message: nil
         )
-        update!(status: "running", failure_summary: nil, error_message: nil)
+        update!(status: "running", failure_summary: nil, error_message: nil, judge_temperature_ignored: false)
 
         response_ids.each do |rid|
           llm_metrics.each { |m| JudgeReviewJob.perform_later(rid, m.id, id) } if llm_judge_configured?
@@ -467,7 +469,7 @@ module CompletionKit
     end
 
     def nondeterministic_judge?
-      judge_temperature.to_f > 0
+      judge_temperature.to_f > 0 || judge_temperature_ignored?
     end
 
     def rerun!
@@ -559,6 +561,7 @@ module CompletionKit
         expected_column: expected_column,
         created_at: created_at, updated_at: updated_at,
         max_tokens: max_tokens, judge_temperature: judge_temperature,
+        temperature_ignored: temperature_ignored, judge_temperature_ignored: judge_temperature_ignored,
         responses_count: responses.count, avg_score: avg_score,
         check_pass_rate: check_pass_rate,
         metric_averages: metric_averages,

data/app/services/completion_kit/anthropic_client.rb

@@ -5,9 +5,6 @@ module CompletionKit
       { id: "claude-3-5-haiku-latest", name: "Claude 3.5 Haiku" }
     ].freeze
 
-    def temperature_dropped?
-      @temperature_dropped == true
-    end
 
     def generate_completion(prompt, options = {})
       @temperature_dropped = false
@@ -15,11 +12,11 @@ module CompletionKit
 
       model = options[:model] || "claude-3-7-sonnet-latest"
       max_tokens = options[:max_tokens] || 1000
-      temperature = options[:temperature] || 0.7
+      temperature = resolve_temperature(options)
 
       response = post_messages(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
 
-      if response.status == 400 && temperature_unsupported?(response.body)
+      if response.status == 400 && !temperature.nil? && temperature_unsupported?(response.body)
         @temperature_dropped = true
         response = post_messages(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
@@ -97,9 +94,5 @@ module CompletionKit
       end
     end
 
-    def temperature_unsupported?(body)
-      s = body.to_s
-      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported"))
-    end
   end
 end

data/app/services/completion_kit/azure_foundry_client.rb

@@ -1,8 +1,5 @@
 module CompletionKit
   class AzureFoundryClient < LlmClient
-    def temperature_dropped?
-      @temperature_dropped == true
-    end
 
     def generate_completion(prompt, options = {})
       @temperature_dropped = false
@@ -11,7 +8,7 @@ module CompletionKit
 
       model = options[:model]
       max_tokens = options[:max_tokens] || 1000
-      temperature = options[:temperature] || 0.7
+      temperature = resolve_temperature(options)
       max_completion = false
 
       response = post_chat(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature, max_completion: max_completion)
@@ -129,10 +126,6 @@ module CompletionKit
       end
     end
 
-    def temperature_unsupported?(body)
-      s = body.to_s
-      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported") || s.include?("Unsupported parameter"))
-    end
 
     def max_tokens_unsupported?(body)
       s = body.to_s

data/app/services/completion_kit/judge_service.rb

@@ -13,6 +13,10 @@ module CompletionKit
       @judge_client = LlmClient.for_model(@judge_model, ApiConfig.for_model(@judge_model))
     end
 
+    def temperature_dropped?
+      @judge_client.temperature_dropped?
+    end
+
     def evaluate(output, expected_output = nil, prompt = nil, criteria: nil, rubric_text: nil, input_data: nil, human_examples: nil, **_extras)
       raise CompletionKit::ConfigurationError, "Judge not configured" unless @judge_client.configured?
 

data/app/services/completion_kit/llm_client.rb

@@ -4,10 +4,39 @@ require "json"
 
 module CompletionKit
   class LlmClient
+    DEFAULT_TEMPERATURE = 0.7
+
+    # Providers phrase a temperature refusal every which way. OpenAI's current
+    # reasoning models say "Unsupported value: 'temperature' does not support
+    # 0.7 with this model. Only the default (1) value is supported.", which
+    # shares no wording with Azure's "Unsupported parameter" or Anthropic's
+    # "not supported". Matching on any one phrasing silently turns a
+    # recoverable refusal into a failed row.
+    #
+    # The wording can sit on either side of the parameter name, so both orders
+    # carry the same phrase set. The gap cannot cross a brace, which keeps a
+    # refusal aimed at some other parameter from matching a temperature echoed
+    # back in a neighbouring object of the same error body. That would
+    # otherwise strip a temperature the model was perfectly happy with and
+    # flag the run as having had it ignored.
+    REFUSAL_PHRASE = /
+      deprecated | not\s+supported | does\s+not\s+support |
+      only\s+the\s+default | unsupported\s+(?:parameter|value)
+    /xi
+    TEMPERATURE_REFUSAL = /
+      temperature [^{}]{0,80}? #{REFUSAL_PHRASE}
+      |
+      #{REFUSAL_PHRASE} [^{}]{0,80}? temperature
+    /xi
+
     def initialize(config = {})
       @config = config
     end
 
+    def temperature_dropped?
+      @temperature_dropped == true
+    end
+
     def generate_completion(prompt, options = {})
       raise NotImplementedError, "Subclasses must implement generate_completion"
     end
@@ -50,6 +79,17 @@ module CompletionKit
 
     protected
 
+    # An explicit nil means the caller wants no temperature sent at all, which
+    # is the only request many current models accept. An absent key still gets
+    # the historical default.
+    def resolve_temperature(options)
+      options.key?(:temperature) ? options[:temperature] : DEFAULT_TEMPERATURE
+    end
+
+    def temperature_unsupported?(body)
+      TEMPERATURE_REFUSAL.match?(body.to_s)
+    end
+
     def build_connection(url, timeout: nil, open_timeout: nil)
       Faraday.new(url: url) do |f|
         f.options.timeout = timeout if timeout

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -181,7 +181,9 @@ module CompletionKit
         if run.metric_ids.empty?
           warnings << "No metrics are attached, so this run judges nothing. Attach metric_ids or a metric_group_id before generating."
         end
-        if run.nondeterministic_judge?
+        if run.judge_temperature_ignored?
+          warnings << "The judge model refused the temperature parameter, so it was re-sent without one and the provider applied its own default. judge_temperature reads #{run.judge_temperature} but was never applied, and these scores are not reproducible. Pick a judge model that accepts temperature if you need reproducible scoring."
+        elsif run.nondeterministic_judge?
           warnings << "Judge temperature is #{run.judge_temperature}. Judging above 0 makes scores irreproducible: the same output can get a different score on a re-judge. Set judge_temperature to 0 unless you are deliberately measuring judge variance."
         end
         return json if warnings.empty?

data/app/services/completion_kit/ollama_client.rb

@@ -1,8 +1,5 @@
 module CompletionKit
   class OllamaClient < LlmClient
-    def temperature_dropped?
-      @temperature_dropped == true
-    end
 
     def generate_completion(prompt, options = {})
       @temperature_dropped = false
@@ -11,11 +8,11 @@ module CompletionKit
 
       model = options[:model]
       max_tokens = options[:max_tokens] || 1000
-      temperature = options[:temperature] || 0.7
+      temperature = resolve_temperature(options)
 
       response = post_completion(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
 
-      if response.status == 400 && temperature_unsupported?(response.body)
+      if response.status == 400 && !temperature.nil? && temperature_unsupported?(response.body)
         @temperature_dropped = true
         response = post_completion(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
@@ -97,9 +94,5 @@ module CompletionKit
       end
     end
 
-    def temperature_unsupported?(body)
-      s = body.to_s
-      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported") || s.include?("Unsupported parameter"))
-    end
   end
 end

data/app/services/completion_kit/open_ai_client.rb

@@ -6,9 +6,6 @@ module CompletionKit
       { id: "gpt-4o-mini", name: "GPT-4o Mini" }
     ].freeze
 
-    def temperature_dropped?
-      @temperature_dropped == true
-    end
 
     def generate_completion(prompt, options = {})
       @temperature_dropped = false
@@ -16,11 +13,11 @@ module CompletionKit
 
       model = options[:model] || "gpt-4.1-mini"
       max_tokens = options[:max_tokens] || 8192
-      temperature = options[:temperature] || 0.7
+      temperature = resolve_temperature(options)
 
       response = post_responses(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
 
-      if response.status == 400 && temperature_unsupported?(response.body)
+      if response.status == 400 && !temperature.nil? && temperature_unsupported?(response.body)
         @temperature_dropped = true
         response = post_responses(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
@@ -38,7 +35,7 @@ module CompletionKit
         data = JSON.parse(response.body)
         if data["status"] == "incomplete"
           reason = data.dig("incomplete_details", "reason") || "unknown"
-          return "Error: response incomplete (#{reason}) — increase max_tokens=#{max_tokens} or pick a non-reasoning judge model"
+          return "Error: response incomplete (#{reason}). Increase max_tokens=#{max_tokens} or pick a non-reasoning judge model"
         end
         message = Array(data["output"]).find { |o| o["type"] == "message" }
         content = message&.dig("content", 0, "text").to_s.strip
@@ -93,9 +90,5 @@ module CompletionKit
       end
     end
 
-    def temperature_unsupported?(body)
-      s = body.to_s
-      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported") || s.include?("Unsupported parameter"))
-    end
   end
 end

data/app/services/completion_kit/open_router_client.rb

@@ -4,9 +4,6 @@ module CompletionKit
     REFERER = "https://completionkit.com".freeze
     APP_TITLE = "CompletionKit".freeze
 
-    def temperature_dropped?
-      @temperature_dropped == true
-    end
 
     def generate_completion(prompt, options = {})
       @temperature_dropped = false
@@ -14,11 +11,11 @@ module CompletionKit
 
       model = options[:model] || "openai/gpt-4o-mini"
       max_tokens = options[:max_tokens] || 8192
-      temperature = options[:temperature] || 0.7
+      temperature = resolve_temperature(options)
 
       response = post_chat(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
 
-      if response.status == 400 && temperature_unsupported?(response.body)
+      if response.status == 400 && !temperature.nil? && temperature_unsupported?(response.body)
         @temperature_dropped = true
         response = post_chat(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
@@ -90,9 +87,5 @@ module CompletionKit
       end
     end
 
-    def temperature_unsupported?(body)
-      s = body.to_s
-      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported") || s.include?("Unsupported parameter"))
-    end
   end
 end

data/app/views/completion_kit/runs/_form.html.erb

@@ -85,12 +85,21 @@
 
     <div class="ck-field">
       <label class="ck-label" for="run_temperature" style="position: relative;">
-        Temperature<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">Controls how random the model's output is. Lower values are more focused and deterministic — the model picks the most likely words. Higher values are more varied and creative, with more risk of odd phrasing. Most LLMs default to 1.0; for evaluation, try a few values and see how your prompt holds up. Newer reasoning models (Claude Opus 4.7, GPT-5 family, etc.) ignore temperature entirely — CompletionKit detects this and re-sends without the parameter.</span>
+        Temperature<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">Controls how random the model's output is. Lower values are more focused and deterministic, so the model picks the most likely words. Higher values are more varied and creative, with more risk of odd phrasing. Most LLMs default to 1.0; for evaluation, try a few values and see how your prompt holds up. Newer reasoning models (Claude Opus 4.7, GPT-5 family, etc.) refuse temperature outright. CompletionKit detects that and re-sends without the parameter, and flags the run so you know the value was never applied. Tick the box below to skip sending it in the first place.</span>
       </label>
       <div class="ck-slider-row">
-        <%= form.range_field :temperature, min: 0, max: 1, step: 0.1, class: "ck-slider", id: "run_temperature", oninput: "document.getElementById('temp-value').textContent = this.value" %>
-        <span class="ck-slider-value" id="temp-value"><%= run.temperature || 1.0 %></span>
+        <%= form.range_field :temperature, min: 0, max: 1, step: 0.1, class: "ck-slider", id: "run_temperature", disabled: run.temperature.nil?, oninput: "document.getElementById('temp-value').textContent = this.value" %>
+        <span class="ck-slider-value" id="temp-value"><%= run.temperature.nil? ? "off" : run.temperature %></span>
       </div>
+      <label class="ck-checkbox-label">
+        <%= check_box_tag "run[omit_temperature]", "1", run.temperature.nil?, id: "run_omit_temperature", class: "ck-checkbox",
+              onchange: "document.getElementById('run_temperature').disabled = this.checked; document.getElementById('temp-value').textContent = this.checked ? 'off' : document.getElementById('run_temperature').value;" %>
+        <span class="ck-checkbox-label__box" aria-hidden="true"></span>
+        <span class="ck-checkbox-label__body">
+          <span class="ck-checkbox-label__text">Send no temperature</span>
+          <span class="ck-checkbox-label__hint">The model applies its own default. Required by models that refuse the parameter, which most current frontier models do.</span>
+        </span>
+      </label>
     </div>
 
     <div class="ck-field">

data/app/views/completion_kit/runs/show.html.erb

@@ -96,7 +96,9 @@
   <div class="ck-run-config__row">
     <span class="ck-run-config__key">Judge temperature</span>
     <span><%= @run.judge_temperature %></span>
-    <% if @run.nondeterministic_judge? %>
+    <% if @run.judge_temperature_ignored? %>
+      <span class="ck-run-config__warn" style="color: var(--ck-warning);" title="The judge model refused the temperature parameter, so CompletionKit re-sent the request without it and the provider applied its own default. The scores are not reproducible despite the setting above.">refused by judge, not reproducible</span>
+    <% elsif @run.nondeterministic_judge? %>
       <span class="ck-run-config__warn" style="color: var(--ck-warning);" title="Judging above temperature 0 makes scores irreproducible. The same output can get a different score on a re-judge.">scores not reproducible</span>
     <% end %>
   </div>

data/db/migrate/20260804000001_add_judge_temperature_ignored_to_runs.rb

@@ -0,0 +1,5 @@
+class AddJudgeTemperatureIgnoredToRuns < ActiveRecord::Migration[8.0]
+  def change
+    add_column :completion_kit_runs, :judge_temperature_ignored, :boolean, default: false, null: false
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.34"
+  VERSION = "0.28.35"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.34
+  version: 0.28.35
 platform: ruby
 authors:
 - Damien Bastin
@@ -447,6 +447,7 @@ files:
 - db/migrate/20260728000001_add_max_tokens_to_completion_kit_runs.rb
 - db/migrate/20260730000001_add_judge_temperature_to_completion_kit_runs.rb
 - db/migrate/20260730000002_create_completion_kit_prompt_serves.rb
+- db/migrate/20260804000001_add_judge_temperature_ignored_to_runs.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb