completion-kit 0.28.38 → 0.28.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d1ac95215866b0de18d662ad0c690780a31b77cf163edd099a3a4b98d6e2f80
-  data.tar.gz: e016b642179a75138333d5d9bb5d1781e296349ed3952df828e8b154d4412518
+  metadata.gz: c3f438ee1d22fb6ef2b93af5db0802335c162bd61209ef1cd7ebdad82e3956e1
+  data.tar.gz: b893015507fe05d66a3274191e559d0983943e3293d97d2f99c098ef04a08231
 SHA512:
-  metadata.gz: 43f8b0575d21007a494f37dfc49e95dadfc140cc4d33e20137baba7deecbc6b81bdf03cee4ebbac80e1122d01a0b9b4c68d37e8e91f4aaf7e9c0437cc6fa20a0
-  data.tar.gz: f115a337c2b5b621755a421e1f2ddd1caacfd068db491872b728aa5f181beed9029e5d10b839169b822836025408854acbe20903ef869b33d15410f8f1da83bd
+  metadata.gz: 32f2758f6a521c7121df79319c870b80145b6bef889a4d0eb29a0cd1516af02db478faf3f78a70d2f8bd08dc320dba6ccd303a19b790afa2ab1a1b52044a8ed0
+  data.tar.gz: db71150f73c7294b531398dc4bcfb392c6a5afb6ba736d37bfd8c1cfd3c94509013492376afc9d0ec17ee766d998a8cf99466aacb51eb87ff0d8261f3024e6cc

data/app/models/completion_kit/response.rb

@@ -57,10 +57,14 @@ module CompletionKit
     def fully_reviewed?
       metric_ids = run.metric_ids
       return true if metric_ids.empty?
-      reviewed_metric_ids = reviews.select { |review| HasJobStatus::TERMINAL_STATUSES.include?(review.status) }.map(&:metric_id).uniq
+      reviewed_metric_ids = reviews.select { |review| review.status == "succeeded" }.map(&:metric_id).uniq
       (metric_ids - reviewed_metric_ids).empty?
     end
 
+    def scoring_failed?
+      reviews.any? { |review| review.status == "failed" }
+    end
+
     private
 
     def requires_response_text?

data/app/models/completion_kit/run.rb

@@ -5,6 +5,12 @@ module CompletionKit
 
     STATUSES = %w[pending running completed failed].freeze
     INSERT_BATCH_SIZE = 1000
+    REVIEW_RETRY_RESET = {
+      status: "pending",
+      attempts: 0,
+      error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+      ai_score: nil, passed: nil, ai_feedback: nil
+    }.freeze
 
     belongs_to :prompt, optional: true
     belongs_to :dataset, optional: true
@@ -492,15 +498,21 @@ module CompletionKit
     def retry_failures!(only: nil)
       scope = responses.where(status: "failed")
       scope = scope.where(id: only) if only.present?
+      failed_response_ids = scope.pluck(:id)
+
+      # A response can generate fine and still have a review blow up, so those
+      # reviews are retryable on their own without regenerating the response.
+      unscored = Review.where(status: "failed")
+                       .where.not(metric_id: nil)
+                       .where(response_id: responses.where(status: "succeeded").select(:id))
+      unscored = unscored.where(response_id: only) if only.present?
+      unscored_pairs = unscored.pluck(:response_id, :metric_id)
+
+      return self if failed_response_ids.empty? && unscored_pairs.empty?
 
       transaction do
-        failed_response_ids = scope.pluck(:id)
-        Review.where(response_id: failed_response_ids, status: "failed").update_all(
-          status: "pending",
-          attempts: 0,
-          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-          ai_score: nil, passed: nil, ai_feedback: nil
-        )
+        Review.where(response_id: failed_response_ids, status: "failed").update_all(REVIEW_RETRY_RESET)
+        unscored.update_all(REVIEW_RETRY_RESET)
         scope.update_all(
           status: "pending",
           attempts: 0,
@@ -509,10 +521,25 @@ module CompletionKit
         )
         update!(status: "running")
         failed_response_ids.each { |rid| GenerateRowJob.perform_later(id, rid) }
+        enqueue_review_retries(unscored_pairs)
       end
       self
     end
 
+    def enqueue_review_retries(pairs)
+      return if pairs.empty?
+
+      kinds = Metric.where(id: pairs.map(&:last).uniq).pluck(:id, :metric_type).to_h
+      pairs.each do |response_id, metric_id|
+        if kinds[metric_id] == "check"
+          CheckReviewJob.perform_later(response_id, metric_id, id)
+        else
+          JudgeReviewJob.perform_later(response_id, metric_id, id)
+        end
+      end
+      RunCompletionCheckJob.perform_later(id)
+    end
+
     def progress_snapshot
       generated_done = responses.where(status: "succeeded").count
       generated_failed = responses.where(status: "failed").count

data/app/views/completion_kit/runs/_response_row.html.erb

@@ -54,7 +54,9 @@
         <span class="ck-chip ck-chip--warning">Retrying <%= response.attempts %>/5</span>
       <% end %>
     <% when "succeeded" %>
-      <% if response.fully_reviewed? %>
+      <% if response.scoring_failed? && run.status != "running" %>
+        <span class="ck-chip ck-chip--danger">Failed</span>
+      <% elsif response.fully_reviewed? %>
         <span class="ck-chip ck-chip--done">Done</span>
       <% elsif run.status == "running" %>
         <span class="ck-chip">Judging</span>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.38"
+  VERSION = "0.28.39"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.38
+  version: 0.28.39
 platform: ruby
 authors:
 - Damien Bastin