completion-kit 0.28.37 → 0.28.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28e080ca8be590088a2a99f16dc5e0f5e6c3f292d29bbc708e6fd8ef14b9b7d3
-  data.tar.gz: f5ff52512c81735d06941634973277915ab89d95a94a0640a47b3436f961d185
+  metadata.gz: c3f438ee1d22fb6ef2b93af5db0802335c162bd61209ef1cd7ebdad82e3956e1
+  data.tar.gz: b893015507fe05d66a3274191e559d0983943e3293d97d2f99c098ef04a08231
 SHA512:
-  metadata.gz: 2a8140334350dc31588e0b6f29c8f44fc2b6b4b17a8251d422a9d1d7cc7a605e8bf05e913afcb7fbbd1d682273ce9dd339786050b492e1111ae752285b271518
-  data.tar.gz: 93bc707699f8081213f5344ba2077be80a004adc24445b173a9d815bf4cffd1a68301b53bd1a34a17f9190dbfb5e0acc3790cfe316cdb26c7dcb291c984ed287
+  metadata.gz: 32f2758f6a521c7121df79319c870b80145b6bef889a4d0eb29a0cd1516af02db478faf3f78a70d2f8bd08dc320dba6ccd303a19b790afa2ab1a1b52044a8ed0
+  data.tar.gz: db71150f73c7294b531398dc4bcfb392c6a5afb6ba736d37bfd8c1cfd3c94509013492376afc9d0ec17ee766d998a8cf99466aacb51eb87ff0d8261f3024e6cc

data/app/assets/stylesheets/completion_kit/application.css

@@ -3247,11 +3247,15 @@ select.ck-input {
   flex-wrap: wrap;
   align-items: flex-start;
 }
+/* The column is sized by the button, which never wraps, so min-content
+   resolves to the button's own width and the note wraps underneath it. A
+   fixed cap sized the column for the note instead, which let the wider
+   button overflow and swallow the gap between the two actions. */
 .ck-banner-action {
   display: flex;
   flex-direction: column;
-  gap: 0.35rem;
-  max-width: 16rem;
+  gap: 0.4rem;
+  width: min-content;
 }
 .ck-banner-action__note {
   margin: 0;

data/app/models/completion_kit/response.rb

@@ -57,10 +57,14 @@ module CompletionKit
     def fully_reviewed?
       metric_ids = run.metric_ids
       return true if metric_ids.empty?
-      reviewed_metric_ids = reviews.select { |review| HasJobStatus::TERMINAL_STATUSES.include?(review.status) }.map(&:metric_id).uniq
+      reviewed_metric_ids = reviews.select { |review| review.status == "succeeded" }.map(&:metric_id).uniq
       (metric_ids - reviewed_metric_ids).empty?
     end
 
+    def scoring_failed?
+      reviews.any? { |review| review.status == "failed" }
+    end
+
     private
 
     def requires_response_text?

data/app/models/completion_kit/run.rb

@@ -5,6 +5,12 @@ module CompletionKit
 
     STATUSES = %w[pending running completed failed].freeze
     INSERT_BATCH_SIZE = 1000
+    REVIEW_RETRY_RESET = {
+      status: "pending",
+      attempts: 0,
+      error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+      ai_score: nil, passed: nil, ai_feedback: nil
+    }.freeze
 
     belongs_to :prompt, optional: true
     belongs_to :dataset, optional: true
@@ -492,15 +498,21 @@ module CompletionKit
     def retry_failures!(only: nil)
       scope = responses.where(status: "failed")
       scope = scope.where(id: only) if only.present?
+      failed_response_ids = scope.pluck(:id)
+
+      # A response can generate fine and still have a review blow up, so those
+      # reviews are retryable on their own without regenerating the response.
+      unscored = Review.where(status: "failed")
+                       .where.not(metric_id: nil)
+                       .where(response_id: responses.where(status: "succeeded").select(:id))
+      unscored = unscored.where(response_id: only) if only.present?
+      unscored_pairs = unscored.pluck(:response_id, :metric_id)
+
+      return self if failed_response_ids.empty? && unscored_pairs.empty?
 
       transaction do
-        failed_response_ids = scope.pluck(:id)
-        Review.where(response_id: failed_response_ids, status: "failed").update_all(
-          status: "pending",
-          attempts: 0,
-          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-          ai_score: nil, passed: nil, ai_feedback: nil
-        )
+        Review.where(response_id: failed_response_ids, status: "failed").update_all(REVIEW_RETRY_RESET)
+        unscored.update_all(REVIEW_RETRY_RESET)
         scope.update_all(
           status: "pending",
           attempts: 0,
@@ -509,10 +521,25 @@ module CompletionKit
         )
         update!(status: "running")
         failed_response_ids.each { |rid| GenerateRowJob.perform_later(id, rid) }
+        enqueue_review_retries(unscored_pairs)
       end
       self
     end
 
+    def enqueue_review_retries(pairs)
+      return if pairs.empty?
+
+      kinds = Metric.where(id: pairs.map(&:last).uniq).pluck(:id, :metric_type).to_h
+      pairs.each do |response_id, metric_id|
+        if kinds[metric_id] == "check"
+          CheckReviewJob.perform_later(response_id, metric_id, id)
+        else
+          JudgeReviewJob.perform_later(response_id, metric_id, id)
+        end
+      end
+      RunCompletionCheckJob.perform_later(id)
+    end
+
     def progress_snapshot
       generated_done = responses.where(status: "succeeded").count
       generated_failed = responses.where(status: "failed").count

data/app/views/completion_kit/runs/_response_row.html.erb

@@ -54,7 +54,9 @@
         <span class="ck-chip ck-chip--warning">Retrying <%= response.attempts %>/5</span>
       <% end %>
     <% when "succeeded" %>
-      <% if response.fully_reviewed? %>
+      <% if response.scoring_failed? && run.status != "running" %>
+        <span class="ck-chip ck-chip--danger">Failed</span>
+      <% elsif response.fully_reviewed? %>
         <span class="ck-chip ck-chip--done">Done</span>
       <% elsif run.status == "running" %>
         <span class="ck-chip">Judging</span>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.37"
+  VERSION = "0.28.39"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.37
+  version: 0.28.39
 platform: ruby
 authors:
 - Damien Bastin