completion-kit 0.5.13 → 0.5.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ec878fca14222f69bc34d85967b065b298ea669be11f60c49b52af13aefdfe5
-  data.tar.gz: 8211fc882175c1e69c52f2cba5a5e14668af4ddfc5784362252e8c051a59bee1
+  metadata.gz: c9c349e12b1fb11eccb4b691dfac14e86bceda2cac6718523e9d3a9e0516698d
+  data.tar.gz: 43374b6f2a44ad5cfc44ac307ad6ba933c61262f79cfc1dd8797cdabef52fd87
 SHA512:
-  metadata.gz: 2afc081fde8f6722aadee50f973bc1512be15e5cbd19913881f0f38f06df37fee4f1cdbda54cfcb230892e73df7b27f6fdf537ea71df7da49ddfba39d2c9f644
-  data.tar.gz: 02cfae669de0b9a1f7a10bbedd7eecda8d708802ff85c1d8c6a50368fb49385e43f19b388f2ea45313b90e14a3d53464688b897ad23378964cfb55103bd3a8c2
+  metadata.gz: 33ed9038a4416e3d3929b7a18757779e5c406f4b4b3cdb1d8bf0ce945f6a8ceac4606d467068b4710a53f86c544a6fbe5fd6b56473ff9d4c7da2ac0f11cb2a13
+  data.tar.gz: 0c907dae8acd807f60061c79213022cdb12bc33c627080d2e85be2954345fea1335eb38fed99ccfe95c9d3f470be1d90e9e3f5a636180ab43648328fd30749b1

data/app/helpers/completion_kit/application_helper.rb

@@ -179,6 +179,18 @@ module CompletionKit
       ["tag", "tag-#{tag.color}", ("tag-outline" if outline)].compact.join(" ")
     end
 
+    def ck_run_path(run)
+      CompletionKit::Engine.routes.url_helpers.run_path(run)
+    end
+
+    def ck_prompt_path(prompt)
+      CompletionKit::Engine.routes.url_helpers.prompt_path(prompt)
+    end
+
+    def ck_dataset_path(dataset)
+      CompletionKit::Engine.routes.url_helpers.dataset_path(dataset)
+    end
+
     def ck_format_maybe_json(text)
       s = text.to_s
       return s if s.strip.empty?

data/app/services/completion_kit/judge_service.rb

@@ -1,6 +1,8 @@
 require "faraday"
 
 module CompletionKit
+  class JudgeParseError < StandardError; end
+
   class JudgeService
     def initialize(config = {})
       @config = config
@@ -9,7 +11,7 @@ module CompletionKit
     end
 
     def evaluate(output, expected_output = nil, prompt = nil, criteria: nil, rubric_text: nil, human_examples: nil, input_data: nil, **_extras)
-      return { score: 1, feedback: "Judge not configured" } unless @judge_client.configured?
+      raise CompletionKit::ConfigurationError, "Judge not configured" unless @judge_client.configured?
 
       judge_prompt = build_judge_prompt(output, expected_output, prompt,
         criteria: criteria,
@@ -19,10 +21,6 @@ module CompletionKit
       response = @judge_client.generate_completion(judge_prompt, model: @judge_model)
       raise StandardError, response if response.start_with?("Error:")
       parse_judge_response(response)
-    rescue Faraday::Error
-      raise
-    rescue => e
-      { score: 1, feedback: "Error during evaluation: #{e.message}" }
     end
 
     private
@@ -66,16 +64,13 @@ module CompletionKit
       score_match = response.match(/\*{0,2}Score:?\*{0,2}\s*(\d+(?:\.\d+)?)/i)
       feedback_match = response.match(/\*{0,2}Feedback:?\*{0,2}\s*(.+)/mi)
 
-      score = score_match ? score_match[1].to_f : 1
-      feedback = if feedback_match
-                   feedback_match[1].strip
-                 elsif score_match
-                   "No feedback provided"
-                 else
-                   "Could not parse judge response: #{response.truncate(500)}"
-                 end
+      unless score_match
+        raise CompletionKit::JudgeParseError,
+              "Could not parse judge response: #{response.truncate(500)}"
+      end
 
-      score = [[score, 1].max, 5].min
+      score = [[score_match[1].to_f, 1].max, 5].min
+      feedback = feedback_match ? feedback_match[1].strip : "No feedback provided"
 
       { score: score, feedback: feedback }
     end

data/app/services/completion_kit/open_ai_client.rb

@@ -15,7 +15,7 @@ module CompletionKit
       return "Error: API key not configured" unless configured?
 
       model = options[:model] || "gpt-4.1-mini"
-      max_tokens = options[:max_tokens] || 1000
+      max_tokens = options[:max_tokens] || 8192
       temperature = options[:temperature] || 0.7
 
       response = post_responses(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
@@ -36,8 +36,14 @@ module CompletionKit
 
       if response.success?
         data = JSON.parse(response.body)
+        if data["status"] == "incomplete"
+          reason = data.dig("incomplete_details", "reason") || "unknown"
+          return "Error: response incomplete (#{reason}) — increase max_tokens=#{max_tokens} or pick a non-reasoning judge model"
+        end
         message = Array(data["output"]).find { |o| o["type"] == "message" }
-        message&.dig("content", 0, "text").to_s.strip
+        content = message&.dig("content", 0, "text").to_s.strip
+        return "Error: model returned empty content" if content.empty?
+        content
       else
         "Error: #{response.status} - #{response.body}"
       end

data/app/services/completion_kit/open_router_client.rb

@@ -13,7 +13,7 @@ module CompletionKit
       return "Error: API key not configured" unless configured?
 
       model = options[:model] || "openai/gpt-4o-mini"
-      max_tokens = options[:max_tokens] || 1000
+      max_tokens = options[:max_tokens] || 8192
       temperature = options[:temperature] || 0.7
 
       response = post_chat(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
@@ -34,7 +34,13 @@ module CompletionKit
 
       if response.success?
         data = JSON.parse(response.body)
-        data.dig("choices", 0, "message", "content").to_s.strip
+        choice = data.dig("choices", 0) || {}
+        if choice["finish_reason"] == "length"
+          return "Error: response truncated by max_tokens=#{max_tokens} before visible content was emitted (reasoning model burned through the budget)"
+        end
+        content = choice.dig("message", "content").to_s.strip
+        return "Error: model returned empty content" if content.empty?
+        content
       else
         "Error: #{response.status} - #{response.body}"
       end

data/app/views/completion_kit/runs/_row.html.erb

@@ -1,4 +1,4 @@
-<tr onclick="window.location='<%= run_path(run) %>'" style="cursor: pointer;">
+<tr onclick="window.location='<%= ck_run_path(run) %>'" style="cursor: pointer;">
   <td>
     <div class="ck-runs-table__identity">
       <span class="ck-run-name">
@@ -7,14 +7,14 @@
       </span>
       <div class="ck-runs-table__config">
         <% if run.prompt %>
-          <%= link_to run.prompt.name, prompt_path(run.prompt), class: "ck-runs-table__config-link", onclick: "event.stopPropagation();" %>
+          <%= link_to run.prompt.name, ck_prompt_path(run.prompt), class: "ck-runs-table__config-link", onclick: "event.stopPropagation();" %>
           <span class="ck-runs-table__version">v<%= run.prompt.version_number %></span>
         <% else %>
           <span class="ck-runs-table__version">Judge-only</span>
         <% end %>
         <% if run.dataset %>
           <span class="ck-runs-table__sep">·</span>
-          <%= link_to run.dataset.name, dataset_path(run.dataset), class: "ck-runs-table__config-link", onclick: "event.stopPropagation();" %>
+          <%= link_to run.dataset.name, ck_dataset_path(run.dataset), class: "ck-runs-table__config-link", onclick: "event.stopPropagation();" %>
         <% end %>
       </div>
       <% if run.tags.any? %>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.13"
+  VERSION = "0.5.15"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.13
+  version: 0.5.15
 platform: ruby
 authors:
 - Damien Bastin