completion-kit 0.27.3 → 0.27.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce7146633041f98e640bc4178adab7bcd061332d91ede886e925ddf57f7db99e
-  data.tar.gz: 4ebd7d9c9590f4e0c5d84e92785afc852c0c583f895d2a38a77515b8e62e3d0f
+  metadata.gz: f36bc9d9d0e9aa1e0e6ed29dec74bef0021e112c1c2198f737313ccabd02c827
+  data.tar.gz: a3dd4f7b50e7bf96389b390fa985954fed6644b0cd33d1eda1edfa92613b6a04
 SHA512:
-  metadata.gz: b6ca1e8722e58788cfd684c33c70998a3c730ed2520cc8944ab0f18e8f228cbad4fff888c693091ea0e9f308414d7cdf9e90351ff45b932826d89303c422a0a9
-  data.tar.gz: 793f163f371634c7fe1c30bd02988d918e35f023e048da655878c4e111ca5df437affba18f2d2f122934f29d8a1d04af14f7fda36a60b24bcbe7ca7cb1b56e2f
+  metadata.gz: 74ca0f3de22ea9af05a5afc86771bad4e12e018d2342d76110fc699ad6630345cd7664c2e2169b98bf2c4120cfbd96369b98be42c425ea6206344e2348052819
+  data.tar.gz: 42ce6279ed741e065dbeb4e51dba9342f08e7059690852787fbd83a1e0b279435b867d92825f4714e9594c39e5fe117d92d1445a4d651db7f72896d86965501c

data/app/jobs/completion_kit/generate_row_job.rb

@@ -12,20 +12,30 @@ module CompletionKit
       30 * executions
     end
 
+    rescue_from(StandardError) do |error|
+      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, response_id: @response_id })
+      record_terminal_failure!(error)
+      enqueue_completion_check
+    end
+
     retry_on Faraday::TimeoutError,
              Faraday::ConnectionFailed,
-             wait: :polynomially_longer, attempts: 5
+             wait: :polynomially_longer, attempts: 5 do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
+    end
 
     retry_on CompletionKit::RateLimitError,
-             wait: method(:rate_limit_wait), attempts: 5
+             wait: ->(executions) { rate_limit_wait(executions) }, attempts: 5 do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
+    end
 
     discard_on ActiveJob::DeserializationError
-    discard_on CompletionKit::ConfigurationError
 
-    rescue_from(StandardError) do |error|
-      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, response_id: @response_id })
-      record_terminal_failure!(error)
-      enqueue_completion_check
+    discard_on CompletionKit::ConfigurationError do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
     end
 
     before_perform do |job|
@@ -49,7 +59,12 @@ module CompletionKit
       raise ConfigurationError, client.configuration_errors.join(", ") unless client.configured?
 
       text = client.generate_completion(rendered, model: prompt.llm_model, temperature: run.temperature)
-      raise StandardError, text.to_s.sub(/\AError:\s*/, "") if text.to_s.start_with?("Error:")
+
+      if text.to_s.start_with?("Error:")
+        record_terminal_failure!(ProviderError.from_client_error(text))
+        enqueue_completion_check
+        return
+      end
 
       if client.respond_to?(:temperature_dropped?) && client.temperature_dropped? && !run.temperature_ignored?
         run.update_columns(temperature_ignored: true)

data/app/jobs/completion_kit/judge_review_job.rb

@@ -14,20 +14,30 @@ module CompletionKit
       30 * executions
     end
 
+    rescue_from(StandardError) do |error|
+      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, review_id: @review_id })
+      record_terminal_failure!(error)
+      enqueue_completion_check
+    end
+
     retry_on Faraday::TimeoutError,
              Faraday::ConnectionFailed,
-             wait: :polynomially_longer, attempts: 5
+             wait: :polynomially_longer, attempts: 5 do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
+    end
 
     retry_on CompletionKit::RateLimitError,
-             wait: method(:rate_limit_wait), attempts: 5
+             wait: ->(executions) { rate_limit_wait(executions) }, attempts: 5 do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
+    end
 
     discard_on ActiveJob::DeserializationError
-    discard_on CompletionKit::ConfigurationError
 
-    rescue_from(StandardError) do |error|
-      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, review_id: @review_id })
-      record_terminal_failure!(error)
-      enqueue_completion_check
+    discard_on CompletionKit::ConfigurationError do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
     end
 
     before_perform do |job|
@@ -52,15 +62,21 @@ module CompletionKit
       config = ApiConfig.for_model(run.judge_model).merge(judge_model: run.judge_model)
       judge = JudgeService.new(config)
 
-      evaluation = judge.evaluate(
-        response.response_text,
-        response.expected_output,
-        run.prompt&.template,
-        criteria: metric.instruction.to_s,
-        rubric_text: metric.display_rubric_text,
-        input_data: response.input_data,
-        human_examples: review_examples_for(metric, response)
-      )
+      begin
+        evaluation = judge.evaluate(
+          response.response_text,
+          response.expected_output,
+          run.prompt&.template,
+          criteria: metric.instruction.to_s,
+          rubric_text: metric.display_rubric_text,
+          input_data: response.input_data,
+          human_examples: review_examples_for(metric, response)
+        )
+      rescue CompletionKit::ProviderError => e
+        record_terminal_failure!(e)
+        enqueue_completion_check
+        return
+      end
 
       review = response.reviews.find_or_initialize_by(metric_id: metric.id)
       current_metric_version = MetricVersion.ensure_current_for(metric)

data/app/models/completion_kit/metric_version.rb

@@ -30,6 +30,8 @@ module CompletionKit
         state: "published",
         published_at: Time.current
       )
+    rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
+      current.find_by(metric_id: metric.id) || raise
     end
 
     def check?

data/app/services/completion_kit/judge_service.rb

@@ -20,7 +20,7 @@ module CompletionKit
         human_examples: human_examples)
 
       response = @judge_client.generate_completion(judge_prompt, model: @judge_model)
-      raise StandardError, response if response.start_with?("Error:")
+      raise CompletionKit::ProviderError.from_client_error(response) if response.start_with?("Error:")
       parse_judge_response(response)
     end
 

data/app/views/completion_kit/responses/show.html.erb

@@ -125,6 +125,8 @@
                 <% end %>
               <% elsif review.check? && !review.passed.nil? %>
                 <%= ck_check_badge(review.passed) %>
+              <% elsif review.status == "failed" %>
+                <span class="<%= ck_badge_classes(:failed) %>" title="<%= review.error_message.presence || "Judge failed" %>">Failed</span>
               <% else %>
                 <span class="<%= ck_badge_classes(:pending) %>">Pending</span>
               <% end %>

data/db/migrate/20260715000001_dedupe_and_unique_index_metric_versions.rb

@@ -0,0 +1,56 @@
+class DedupeAndUniqueIndexMetricVersions < ActiveRecord::Migration[8.1]
+  class MetricVersion < ActiveRecord::Base
+    self.table_name = "completion_kit_metric_versions"
+  end
+
+  class Review < ActiveRecord::Base
+    self.table_name = "completion_kit_reviews"
+  end
+
+  class Agreement < ActiveRecord::Base
+    self.table_name = "completion_kit_agreements"
+  end
+
+  def up
+    collapse_duplicate_versions
+    collapse_extra_current
+
+    remove_index :completion_kit_metric_versions, name: "index_ck_metric_versions_on_metric_vnum", if_exists: true
+    add_index :completion_kit_metric_versions, [:metric_id, :version_number],
+      unique: true, name: "index_ck_metric_versions_on_metric_vnum"
+  end
+
+  def down
+    remove_index :completion_kit_metric_versions, name: "index_ck_metric_versions_on_metric_vnum", if_exists: true
+    add_index :completion_kit_metric_versions, [:metric_id, :version_number],
+      name: "index_ck_metric_versions_on_metric_vnum"
+  end
+
+  private
+
+  def collapse_duplicate_versions
+    MetricVersion.order(:id).pluck(:id, :metric_id, :version_number)
+                 .group_by { |(_, metric_id, version_number)| [metric_id, version_number] }
+                 .each_value do |group|
+      next if group.size < 2
+
+      canonical = group.first.first
+      losers = group.drop(1).map(&:first)
+      Review.where(metric_version_id: losers).update_all(metric_version_id: canonical)
+      Agreement.where(metric_version_id: losers).update_all(metric_version_id: canonical)
+      MetricVersion.where(id: losers).delete_all
+    end
+  end
+
+  def collapse_extra_current
+    MetricVersion.where(current: true)
+                 .order(metric_id: :asc, version_number: :desc, id: :desc)
+                 .pluck(:id, :metric_id)
+                 .group_by { |(_, metric_id)| metric_id }
+                 .each_value do |rows|
+      next if rows.size < 2
+
+      MetricVersion.where(id: rows.drop(1).map(&:first)).update_all(current: false)
+    end
+  end
+end

data/lib/completion_kit/errors.rb

@@ -13,4 +13,22 @@ module CompletionKit
       @retry_after = retry_after
     end
   end
+
+  class ProviderError < Error
+    attr_reader :status
+
+    def initialize(message = nil, status: nil)
+      super(message)
+      @status = status
+    end
+
+    def self.from_client_error(text)
+      detail = text.to_s.sub(/\AError:\s*/, "")
+      if (match = detail.match(/\A(\d{3})\s*-\s*(.*)/m))
+        new(match[2], status: match[1].to_i)
+      else
+        new(detail)
+      end
+    end
+  end
 end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.27.3"
+  VERSION = "0.27.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.27.3
+  version: 0.27.5
 platform: ruby
 authors:
 - Damien Bastin
@@ -480,6 +480,7 @@ files:
 - db/migrate/20260706000001_add_expected_column_to_completion_kit_runs.rb
 - db/migrate/20260708000001_add_api_version_to_completion_kit_provider_credentials.rb
 - db/migrate/20260713000001_add_catalog_model_count_to_completion_kit_provider_credentials.rb
+- db/migrate/20260715000001_dedupe_and_unique_index_metric_versions.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb