completion-kit 0.27.2 → 0.27.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f8fdeef847d7243a665b86e111343d0255ac2eb56ba6538c56c21871de04d1c
-  data.tar.gz: 58911d7d679a943284eacaaca586c6ad375efdf674aa1604f9895ca971f80c2e
+  metadata.gz: a2dbe2221c832f769b697c57e5268d40f3d6474786f78da6abc53288e150376c
+  data.tar.gz: b851f95fd737b829e0bf0b82d83f37902e5e18a84b4de9a6dbe38bc9154e5317
 SHA512:
-  metadata.gz: fd44b864ef07239759b9211cf24dac1aad601d30bd1b92068e5e2952571e338decb85be400464c1c3a947f40040c14aeea74384512594290f79dbfddbed19555
-  data.tar.gz: ad879e8ebc9f77dab47592f23b3bf0a6ae7eb101e11dc07e43a1ed7b67caa19fde109b3dcc20ab445c3a0c4daebb843b442b1de69ec9e433a3fe5da9d7d5c661
+  metadata.gz: 477ad7b3ee34ced678bd03c21c972c3033f617dfdcd5e44a1e9d5596280d8878995c28fcab3cd8f7f2ddcf1c19b8e10c0db9c5330595d350c038373e2e1a7942
+  data.tar.gz: 13ca8b2b7544bb1d9e7e171faa167ea392c8b13546fc67023485f0b5dde671d1852d69447e494ebb6760b3ce6a063f598b18daca8dbd79541322153f8fa8aa2b

data/app/jobs/completion_kit/generate_row_job.rb

@@ -12,20 +12,30 @@ module CompletionKit
       30 * executions
     end
 
+    rescue_from(StandardError) do |error|
+      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, response_id: @response_id })
+      record_terminal_failure!(error)
+      enqueue_completion_check
+    end
+
     retry_on Faraday::TimeoutError,
              Faraday::ConnectionFailed,
-             wait: :polynomially_longer, attempts: 5
+             wait: :polynomially_longer, attempts: 5 do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
+    end
 
     retry_on CompletionKit::RateLimitError,
-             wait: method(:rate_limit_wait), attempts: 5
+             wait: ->(executions) { rate_limit_wait(executions) }, attempts: 5 do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
+    end
 
     discard_on ActiveJob::DeserializationError
-    discard_on CompletionKit::ConfigurationError
 
-    rescue_from(StandardError) do |error|
-      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, response_id: @response_id })
-      record_terminal_failure!(error)
-      enqueue_completion_check
+    discard_on CompletionKit::ConfigurationError do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
     end
 
     before_perform do |job|
@@ -49,7 +59,12 @@ module CompletionKit
       raise ConfigurationError, client.configuration_errors.join(", ") unless client.configured?
 
       text = client.generate_completion(rendered, model: prompt.llm_model, temperature: run.temperature)
-      raise StandardError, text.to_s.sub(/\AError:\s*/, "") if text.to_s.start_with?("Error:")
+
+      if text.to_s.start_with?("Error:")
+        record_terminal_failure!(ProviderError.from_client_error(text))
+        enqueue_completion_check
+        return
+      end
 
       if client.respond_to?(:temperature_dropped?) && client.temperature_dropped? && !run.temperature_ignored?
         run.update_columns(temperature_ignored: true)

data/app/jobs/completion_kit/judge_review_job.rb

@@ -14,20 +14,30 @@ module CompletionKit
       30 * executions
     end
 
+    rescue_from(StandardError) do |error|
+      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, review_id: @review_id })
+      record_terminal_failure!(error)
+      enqueue_completion_check
+    end
+
     retry_on Faraday::TimeoutError,
              Faraday::ConnectionFailed,
-             wait: :polynomially_longer, attempts: 5
+             wait: :polynomially_longer, attempts: 5 do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
+    end
 
     retry_on CompletionKit::RateLimitError,
-             wait: method(:rate_limit_wait), attempts: 5
+             wait: ->(executions) { rate_limit_wait(executions) }, attempts: 5 do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
+    end
 
     discard_on ActiveJob::DeserializationError
-    discard_on CompletionKit::ConfigurationError
 
-    rescue_from(StandardError) do |error|
-      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, review_id: @review_id })
-      record_terminal_failure!(error)
-      enqueue_completion_check
+    discard_on CompletionKit::ConfigurationError do |job, error|
+      job.send(:record_terminal_failure!, error)
+      job.send(:enqueue_completion_check)
     end
 
     before_perform do |job|
@@ -52,15 +62,21 @@ module CompletionKit
       config = ApiConfig.for_model(run.judge_model).merge(judge_model: run.judge_model)
       judge = JudgeService.new(config)
 
-      evaluation = judge.evaluate(
-        response.response_text,
-        response.expected_output,
-        run.prompt&.template,
-        criteria: metric.instruction.to_s,
-        rubric_text: metric.display_rubric_text,
-        input_data: response.input_data,
-        human_examples: review_examples_for(metric, response)
-      )
+      begin
+        evaluation = judge.evaluate(
+          response.response_text,
+          response.expected_output,
+          run.prompt&.template,
+          criteria: metric.instruction.to_s,
+          rubric_text: metric.display_rubric_text,
+          input_data: response.input_data,
+          human_examples: review_examples_for(metric, response)
+        )
+      rescue CompletionKit::ProviderError => e
+        record_terminal_failure!(e)
+        enqueue_completion_check
+        return
+      end
 
       review = response.reviews.find_or_initialize_by(metric_id: metric.id)
       current_metric_version = MetricVersion.ensure_current_for(metric)

data/app/models/completion_kit/metric.rb

@@ -13,6 +13,7 @@ module CompletionKit
     has_many :metric_group_memberships, dependent: :destroy
     has_many :metric_groups, through: :metric_group_memberships, source: :metric_group
     has_many :metric_versions, dependent: :destroy
+    has_many :run_metrics, dependent: :destroy
     has_many :reviews, dependent: :nullify
     has_many :dashboard_dismissals, as: :dismissable, dependent: :destroy
 

data/app/services/completion_kit/judge_service.rb

@@ -20,7 +20,7 @@ module CompletionKit
         human_examples: human_examples)
 
       response = @judge_client.generate_completion(judge_prompt, model: @judge_model)
-      raise StandardError, response if response.start_with?("Error:")
+      raise CompletionKit::ProviderError.from_client_error(response) if response.start_with?("Error:")
       parse_judge_response(response)
     end
 

data/app/views/completion_kit/responses/show.html.erb

@@ -125,6 +125,8 @@
                 <% end %>
               <% elsif review.check? && !review.passed.nil? %>
                 <%= ck_check_badge(review.passed) %>
+              <% elsif review.status == "failed" %>
+                <span class="<%= ck_badge_classes(:failed) %>" title="<%= review.error_message.presence || "Judge failed" %>">Failed</span>
               <% else %>
                 <span class="<%= ck_badge_classes(:pending) %>">Pending</span>
               <% end %>

data/lib/completion_kit/errors.rb

@@ -13,4 +13,22 @@ module CompletionKit
       @retry_after = retry_after
     end
   end
+
+  class ProviderError < Error
+    attr_reader :status
+
+    def initialize(message = nil, status: nil)
+      super(message)
+      @status = status
+    end
+
+    def self.from_client_error(text)
+      detail = text.to_s.sub(/\AError:\s*/, "")
+      if (match = detail.match(/\A(\d{3})\s*-\s*(.*)/m))
+        new(match[2], status: match[1].to_i)
+      else
+        new(detail)
+      end
+    end
+  end
 end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.27.2"
+  VERSION = "0.27.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.27.2
+  version: 0.27.4
 platform: ruby
 authors:
 - Damien Bastin