completion-kit 0.28.38 → 0.28.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d1ac95215866b0de18d662ad0c690780a31b77cf163edd099a3a4b98d6e2f80
-  data.tar.gz: e016b642179a75138333d5d9bb5d1781e296349ed3952df828e8b154d4412518
+  metadata.gz: ecdbb9977e07a943af3309aa8e1353ebaa16e7556a7bd362c5d2b7a63812d15e
+  data.tar.gz: 9b3ff099f006237a04d26040c3cfeab2fe331ac7b07e4f5aa3fb016839182800
 SHA512:
-  metadata.gz: 43f8b0575d21007a494f37dfc49e95dadfc140cc4d33e20137baba7deecbc6b81bdf03cee4ebbac80e1122d01a0b9b4c68d37e8e91f4aaf7e9c0437cc6fa20a0
-  data.tar.gz: f115a337c2b5b621755a421e1f2ddd1caacfd068db491872b728aa5f181beed9029e5d10b839169b822836025408854acbe20903ef869b33d15410f8f1da83bd
+  metadata.gz: 549b39f898a3951b4426bc71f8b2665fa88ccea18bd21e290fd91ee60ca80cec64deff6de6654d0037ccf20b4a3e929b44fde79a74965bf3e5bb859721307f9d
+  data.tar.gz: efa593027d5439bd970f6cbb74479bd2d59063ebff99a5cc0b767788661fdf9605e55f7fdc1a652eef101f845d3367031fc089a168c8583a4f4ca556de525366

data/app/assets/stylesheets/completion_kit/application.css

@@ -1214,27 +1214,6 @@ tr:hover .ck-chip--publish {
   border-top: 1px dashed var(--ck-line);
 }
 
-.ck-run-status__retry {
-  appearance: none;
-  font-family: var(--ck-mono);
-  font-size: 0.74rem;
-  font-weight: 600;
-  letter-spacing: 0.12em;
-  text-transform: uppercase;
-  padding: 0.7rem 1.1rem;
-  border: 1px solid rgba(248, 113, 113, 0.5);
-  background: rgba(248, 113, 113, 0.08);
-  color: var(--ck-danger);
-  border-radius: var(--ck-radius);
-  cursor: pointer;
-  transition: background 0.15s, border-color 0.15s;
-}
-
-.ck-run-status__retry:hover {
-  background: rgba(248, 113, 113, 0.16);
-  border-color: var(--ck-danger);
-}
-
 .ck-run-status__cells {
   display: flex;
   align-items: stretch;

data/app/helpers/completion_kit/application_helper.rb

@@ -25,6 +25,14 @@ module CompletionKit
       render partial, runs: runs
     end
 
+    def ck_retry_failures_label(generated_failed, judged_failed)
+      if generated_failed.zero?
+        "Retry scoring on #{pluralize(judged_failed, "response")}"
+      else
+        "Retry #{pluralize(generated_failed + judged_failed, "failed response")}"
+      end
+    end
+
     def ck_button_classes(tone = :dark, variant: :solid)
       base = "ck-button"
 

data/app/models/completion_kit/response.rb

@@ -57,10 +57,14 @@ module CompletionKit
     def fully_reviewed?
       metric_ids = run.metric_ids
       return true if metric_ids.empty?
-      reviewed_metric_ids = reviews.select { |review| HasJobStatus::TERMINAL_STATUSES.include?(review.status) }.map(&:metric_id).uniq
+      reviewed_metric_ids = reviews.select { |review| review.status == "succeeded" }.map(&:metric_id).uniq
       (metric_ids - reviewed_metric_ids).empty?
     end
 
+    def scoring_failed?
+      reviews.any? { |review| review.status == "failed" }
+    end
+
     private
 
     def requires_response_text?

data/app/models/completion_kit/run.rb

@@ -5,6 +5,12 @@ module CompletionKit
 
     STATUSES = %w[pending running completed failed].freeze
     INSERT_BATCH_SIZE = 1000
+    REVIEW_RETRY_RESET = {
+      status: "pending",
+      attempts: 0,
+      error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+      ai_score: nil, passed: nil, ai_feedback: nil
+    }.freeze
 
     belongs_to :prompt, optional: true
     belongs_to :dataset, optional: true
@@ -492,15 +498,21 @@ module CompletionKit
     def retry_failures!(only: nil)
       scope = responses.where(status: "failed")
       scope = scope.where(id: only) if only.present?
+      failed_response_ids = scope.pluck(:id)
+
+      # A response can generate fine and still have a review blow up, so those
+      # reviews are retryable on their own without regenerating the response.
+      unscored = Review.where(status: "failed")
+                       .where.not(metric_id: nil)
+                       .where(response_id: responses.where(status: "succeeded").select(:id))
+      unscored = unscored.where(response_id: only) if only.present?
+      unscored_pairs = unscored.pluck(:response_id, :metric_id)
+
+      return self if failed_response_ids.empty? && unscored_pairs.empty?
 
       transaction do
-        failed_response_ids = scope.pluck(:id)
-        Review.where(response_id: failed_response_ids, status: "failed").update_all(
-          status: "pending",
-          attempts: 0,
-          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-          ai_score: nil, passed: nil, ai_feedback: nil
-        )
+        Review.where(response_id: failed_response_ids, status: "failed").update_all(REVIEW_RETRY_RESET)
+        unscored.update_all(REVIEW_RETRY_RESET)
         scope.update_all(
           status: "pending",
           attempts: 0,
@@ -509,10 +521,25 @@ module CompletionKit
         )
         update!(status: "running")
         failed_response_ids.each { |rid| GenerateRowJob.perform_later(id, rid) }
+        enqueue_review_retries(unscored_pairs)
       end
       self
     end
 
+    def enqueue_review_retries(pairs)
+      return if pairs.empty?
+
+      kinds = Metric.where(id: pairs.map(&:last).uniq).pluck(:id, :metric_type).to_h
+      pairs.each do |response_id, metric_id|
+        if kinds[metric_id] == "check"
+          CheckReviewJob.perform_later(response_id, metric_id, id)
+        else
+          JudgeReviewJob.perform_later(response_id, metric_id, id)
+        end
+      end
+      RunCompletionCheckJob.perform_later(id)
+    end
+
     def progress_snapshot
       generated_done = responses.where(status: "succeeded").count
       generated_failed = responses.where(status: "failed").count

data/app/views/completion_kit/runs/_response_row.html.erb

@@ -54,7 +54,9 @@
         <span class="ck-chip ck-chip--warning">Retrying <%= response.attempts %>/5</span>
       <% end %>
     <% when "succeeded" %>
-      <% if response.fully_reviewed? %>
+      <% if response.scoring_failed? && run.status != "running" %>
+        <span class="ck-chip ck-chip--danger">Failed</span>
+      <% elsif response.fully_reviewed? %>
         <span class="ck-chip ck-chip--done">Done</span>
       <% elsif run.status == "running" %>
         <span class="ck-chip">Judging</span>

data/app/views/completion_kit/runs/_status_panel.html.erb

@@ -111,9 +111,10 @@
       </div>
 
       <% if failed_count > 0 %>
-        <%= button_to retry_failures_run_path(run), method: :post, class: "ck-run-status__retry", form_class: "ck-run-status__action" do %>
-          Retry <%= failed_count %> failed <%= "case".pluralize(failed_count) %>
-        <% end %>
+        <%= button_to ck_retry_failures_label(snap[:generated_failed], snap[:judged_failed]),
+              retry_failures_run_path(run), method: :post,
+              class: ck_button_classes(:light, variant: :outline),
+              form_class: "ck-run-status__action" %>
       <% end %>
     </section>
   <% end %>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.38"
+  VERSION = "0.28.40"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.38
+  version: 0.28.40
 platform: ruby
 authors:
 - Damien Bastin