completion-kit 0.18.1 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 620084f7c112f139684433d2d6f2d8ee407bfd5c7aaf1612ac4a5334c404e490
-  data.tar.gz: 7e93349d581b10eaa993250d678f80550492a0075c426b8970a97c498b277ce6
+  metadata.gz: 38312b44903cb1fa31fc64cb00d76713e4bc7c85eb51c9b17ebae6c988778509
+  data.tar.gz: 3bef91185c33760cbb9be5711e59a591115d53dde31b606f3eb92b334979d7ac
 SHA512:
-  metadata.gz: e94ead519660a768f88f7981128f338f39448837ad12cf45ca26409a7b574bc9d6a2e6980f2fef839c5fbd44d17ad465b4cce21937068a98db8312e06f437dde
-  data.tar.gz: 4f31db3df58c983066fbd19b71f8f978215755f592f8637e0116a1b3f0fb33c213079517503077cda2622c52895531c0226f0407f0c32ba33d0788c134bae64f
+  metadata.gz: 718185f7ad5d9644f32f068f1172abd4cb11b4de0a3c28099d3549d4678a437c9a98048c480e50005308e242671fe6d4746828ddee2b8a95e0efb561756d88c6
+  data.tar.gz: 54eccc031ee3b7d02bf543f06a86d9db6e00883925c61a5a85afb261734c999382a2c77076cbdf42bb9481a89cf4074b871f5cb347e60f21cb65e06772f17b8e

data/Rakefile

@@ -7,3 +7,10 @@ RSpec::Core::RakeTask.new(:spec) do |t|
 end
 
 task default: :spec
+
+desc "Run the full suite with judge API keys cleared, matching CI's keyless environment"
+task :release_guard do
+  sh "OPENAI_API_KEY= ANTHROPIC_API_KEY= OLLAMA_API_KEY= bundle exec rspec"
+end
+
+Rake::Task["release:guard_clean"].enhance([:release_guard])

data/app/assets/stylesheets/completion_kit/application.css

@@ -413,6 +413,10 @@ form.button_to {
   grid-template-columns: repeat(3, minmax(0, 1fr));
 }
 
+.ck-grid--cards-4 {
+  grid-template-columns: repeat(4, minmax(0, 1fr));
+}
+
 /* ── Dashboard: workspace stat ribbon ─────────────────────────────────
    Replaces the old oversized count cards. One thin instrument strip,
    four navigable segments split by hairline dividers. */
@@ -3698,7 +3702,7 @@ table.ck-runs-table {
 }
 
 .ck-runs-table td {
-  vertical-align: middle;
+  vertical-align: top;
   padding-top: 0.7rem;
   padding-bottom: 0.7rem;
 }
@@ -5163,6 +5167,7 @@ a.tag-mark {
 .ck-failure-list__surface--run { color: var(--ck-warning); }
 .ck-failure-list__surface--generation { color: var(--ck-danger); }
 .ck-failure-list__surface--judge { color: var(--ck-info); }
+.ck-failure-list__surface--check { color: var(--ck-danger); }
 .ck-failure-list__cause {
   overflow: hidden;
   text-overflow: ellipsis;

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -134,11 +134,15 @@ module CompletionKit
                 {
                   metric_id: mid,
                   metric_name: anchor.metric_name,
+                  kind: anchor.check? ? "check" : "llm_judge",
                   left_score: l_review ? l_review.ai_score : nil,
                   right_score: r_review ? r_review.ai_score : nil,
+                  left_passed: l_review&.passed,
+                  right_passed: r_review&.passed,
                   left_metric_version_id: l_review&.metric_version_id,
                   right_metric_version_id: r_review&.metric_version_id,
-                  delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil
+                  delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil,
+                  result_change: CompletionKit::RunComparison.result_change(l_review&.passed, r_review&.passed)
                 }
               end.compact
             }

data/app/controllers/completion_kit/dashboard_controller.rb

@@ -14,6 +14,7 @@ module CompletionKit
       @activity = DashboardStats.activity
       @worst_metric = DashboardStats.worst_metric(since: 7.days.ago)
       @failures = DashboardStats.failures(since: 7.days.ago)
+      @failing_checks = DashboardStats.failing_checks(since: 7.days.ago)
       @ignored_metrics = DashboardDismissal.metrics
       @ignored_failures = DashboardDismissal.failures
       @prompt_changes = DashboardStats.prompt_changes

data/app/controllers/completion_kit/dashboard_dismissals_controller.rb

@@ -27,6 +27,7 @@ module CompletionKit
 
     def baseline_for(record)
       return nil unless record.is_a?(Metric)
+      return DashboardStats.metric_pass_rate(record.id, since: WINDOW.ago) if record.check?
       DashboardStats.metric_average(record.id, since: WINDOW.ago)
     end
 

data/app/controllers/completion_kit/responses_controller.rb

@@ -1,5 +1,6 @@
 module CompletionKit
   class ResponsesController < ApplicationController
+    include CompletionKit::ResponseOrdering
     before_action :set_run
     before_action :set_response
 
@@ -24,21 +25,7 @@ module CompletionKit
     end
 
     def ordered_response_ids
-      if @run.judge_configured? && @sort == "score_asc"
-        @run.responses
-          .left_joins(:reviews)
-          .group("completion_kit_responses.id")
-          .order(Arel.sql("AVG(completion_kit_reviews.ai_score) ASC NULLS LAST"))
-          .pluck(:id)
-      elsif @run.judge_configured? && @sort != "none"
-        @run.responses
-          .left_joins(:reviews)
-          .group("completion_kit_responses.id")
-          .order(Arel.sql("AVG(completion_kit_reviews.ai_score) DESC NULLS LAST"))
-          .pluck(:id)
-      else
-        @run.responses.order(:id).pluck(:id)
-      end
+      ordered_responses_relation(@run, @sort).pluck(:id)
     end
   end
 end

data/app/controllers/completion_kit/runs_controller.rb

@@ -1,6 +1,7 @@
 module CompletionKit
   class RunsController < ApplicationController
     include CompletionKit::TagFiltering
+    include CompletionKit::ResponseOrdering
     before_action :set_run, only: [:show, :edit, :update, :destroy, :generate, :suggest, :retry_failures, :rerun, :regrade, :refresh_status, :compare]
     before_action :load_form_collections, only: [:new, :edit, :create, :update]
 
@@ -10,21 +11,7 @@ module CompletionKit
     end
 
     def show
-      @responses = if @run.judge_configured? && params[:sort] == "score_asc"
-                     @run.responses
-                       .left_joins(:reviews)
-                       .includes(:reviews)
-                       .group("completion_kit_responses.id")
-                       .order(Arel.sql("AVG(completion_kit_reviews.ai_score) ASC NULLS LAST"))
-                   elsif @run.judge_configured?
-                     @run.responses
-                       .left_joins(:reviews)
-                       .includes(:reviews)
-                       .group("completion_kit_responses.id")
-                       .order(Arel.sql("AVG(completion_kit_reviews.ai_score) DESC NULLS LAST"))
-                   else
-                     @run.responses.includes(:reviews).order(:id)
-                   end
+      @responses = ordered_responses_relation(@run, params[:sort]).includes(:reviews)
     end
 
     def new
@@ -208,11 +195,15 @@ module CompletionKit
             {
               metric_id: mid,
               metric_name: anchor.metric_name,
+              kind: anchor.check? ? "check" : "llm_judge",
               left_score: l_review ? l_review.ai_score : nil,
               right_score: r_review ? r_review.ai_score : nil,
+              left_passed: l_review&.passed,
+              right_passed: r_review&.passed,
               left_version_label: version_label_for(l_review, metric_versions),
               right_version_label: version_label_for(r_review, metric_versions),
-              delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil
+              delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil,
+              result_change: RunComparison.result_change(l_review&.passed, r_review&.passed)
             }
           end.compact
         }

data/app/controllers/concerns/completion_kit/response_ordering.rb

@@ -0,0 +1,25 @@
+module CompletionKit
+  module ResponseOrdering
+    extend ActiveSupport::Concern
+
+    private
+
+    FAILED_CHECKS_SQL = "SUM(CASE WHEN completion_kit_reviews.passed IS FALSE THEN 1 ELSE 0 END)".freeze
+    RUBRIC_AVG_SQL = "AVG(completion_kit_reviews.ai_score)".freeze
+
+    def ordered_responses_relation(run, sort)
+      return run.responses.order(:id) unless run.gradable?
+
+      composite = if sort == "score_asc"
+                    "#{FAILED_CHECKS_SQL} DESC, #{RUBRIC_AVG_SQL} ASC NULLS LAST"
+                  else
+                    "#{FAILED_CHECKS_SQL} ASC, #{RUBRIC_AVG_SQL} DESC NULLS LAST"
+                  end
+
+      run.responses
+         .left_joins(:reviews)
+         .group("completion_kit_responses.id")
+         .order(Arel.sql("#{composite}, completion_kit_responses.id ASC"))
+    end
+  end
+end

data/app/helpers/completion_kit/application_helper.rb

@@ -159,6 +159,34 @@ module CompletionKit
       :low
     end
 
+    def ck_check_badge(passed)
+      if passed == true
+        content_tag(:span, "Pass", class: ck_badge_classes(:high))
+      elsif passed == false
+        content_tag(:span, "Fail", class: ck_badge_classes(:low))
+      else
+        content_tag(:span, "Pending", class: ck_badge_classes(:pending))
+      end
+    end
+
+    def ck_result_change_badge(change)
+      case change
+      when "broke"
+        content_tag(:span, "Broke", class: "ck-delta ck-delta--negative")
+      when "fixed"
+        content_tag(:span, "Fixed", class: "ck-delta ck-delta--positive")
+      when "same"
+        content_tag(:span, "Same", class: "ck-delta ck-delta--zero")
+      end
+    end
+
+    def ck_pass_rate_kind(rate)
+      return :high if rate >= 0.9
+      return :medium if rate >= 0.7
+
+      :low
+    end
+
     def ck_word_diff_old(old_text, new_text)
       diff_tokens(old_text, new_text, :old)
     end

data/app/services/completion_kit/dashboard_stats.rb

@@ -58,6 +58,28 @@ module CompletionKit
       scored_reviews_since(since).where(metric_id: metric_id).average(:ai_score)&.to_f&.round(2)
     end
 
+    def self.metric_pass_rate(metric_id, since:)
+      resolved = Review.joins(:response)
+                       .where(metric_id: metric_id)
+                       .where("completion_kit_reviews.created_at >= ?", since)
+                       .where.not(passed: nil)
+      total = resolved.count
+      return nil if total.zero?
+
+      (resolved.where(passed: true).count.to_f / total).round(2)
+    end
+
+    def self.failing_checks(since:)
+      reviews = Review.where(passed: false)
+                      .where("completion_kit_reviews.created_at >= ?", since)
+                      .includes(response: :run)
+                      .order(updated_at: :desc)
+      items = reviews.map do |review|
+        { metric_name: review.metric_name, response: review.response, run: review.response.run }
+      end
+      { count: items.size, items: items }
+    end
+
     # Everything that terminally failed in the window across all three
     # surfaces — failed runs, failed generations, failed judge reviews —
     # excluding any the user has dismissed. Returns a count and an items list

data/app/services/completion_kit/run_comparison.rb

@@ -0,0 +1,13 @@
+module CompletionKit
+  module RunComparison
+    module_function
+
+    def result_change(left_passed, right_passed)
+      return nil if left_passed.nil? || right_passed.nil?
+      return "broke" if left_passed && !right_passed
+      return "fixed" if !left_passed && right_passed
+
+      "same"
+    end
+  end
+end

data/app/views/completion_kit/dashboard/_failing_checks_card.html.erb

@@ -0,0 +1,19 @@
+<div class="ck-card ck-stat-card ck-rise" id="ck-failing-checks-card" style="--rise-delay: 200ms;">
+  <p class="ck-kicker">Failing checks · last 7 days</p>
+  <div class="ck-stat-card__body">
+    <span class="ck-stat-card__count<%= failing_checks[:count].positive? ? ' is-danger' : ' is-clean' %>"><%= failing_checks[:count] %></span>
+  </div>
+
+  <% if failing_checks[:items].any? %>
+    <ul class="ck-failure-list">
+      <% failing_checks[:items].first(5).each do |item| %>
+        <li class="ck-failure-list__item">
+          <span class="ck-failure-list__surface ck-failure-list__surface--check">check</span>
+          <%= link_to item[:metric_name], completion_kit.run_path(item[:run]), class: "ck-link ck-failure-list__cause" %>
+        </li>
+      <% end %>
+    </ul>
+  <% else %>
+    <div class="ck-stat-card__foot"><span>No failing checks this week.</span></div>
+  <% end %>
+</div>

data/app/views/completion_kit/dashboard/show.html.erb

@@ -28,7 +28,7 @@
 </nav>
 
 <% if @activity %>
-  <div class="ck-grid ck-grid--cards ck-grid--cards-3 ck-pulse-grid">
+  <div class="ck-grid ck-grid--cards ck-grid--cards-4 ck-pulse-grid">
     <div class="ck-card ck-stat-card ck-rise" style="--rise-delay: 60ms;">
       <p class="ck-kicker">Activity · last 14 days</p>
       <% activity_max = @activity.map { |d| d[:count] }.max %>
@@ -49,6 +49,9 @@
 
     <%= render "completion_kit/dashboard/failures_card",
                failures: @failures, ignored_failures: @ignored_failures %>
+
+    <%= render "completion_kit/dashboard/failing_checks_card",
+               failing_checks: @failing_checks %>
   </div>
 
   <div class="ck-card ck-card--spaced ck-rise" style="--rise-delay: 240ms;">

data/app/views/completion_kit/prompts/index.html.erb

@@ -54,10 +54,15 @@
           <% family_runs = CompletionKit::Run.where(prompt_id: prompt.family_versions.select(:id)).display_scoped %>
           <% current_version_runs = prompt.runs.display_scoped.includes(responses: :reviews) %>
           <% best_score = current_version_runs.map(&:avg_score).compact.max %>
+          <% best_pass_rate = current_version_runs.map(&:check_pass_rate).compact.max %>
           <td>
             <% if best_score %>
               <span class="<%= ck_badge_classes(ck_score_kind(best_score)) %>"><%= best_score %></span>
-            <% else %>
+            <% end %>
+            <% if best_pass_rate %>
+              <span class="<%= ck_badge_classes(ck_pass_rate_kind(best_pass_rate)) %>"><%= (best_pass_rate * 100).round %>%</span>
+            <% end %>
+            <% unless best_score || best_pass_rate %>
               <span class="ck-prompts-table__dim">—</span>
             <% end %>
           </td>

data/app/views/completion_kit/prompts/show.html.erb

@@ -64,7 +64,9 @@
       </thead>
       <tbody>
         <% versions.each do |v| %>
-          <% best_score = v.runs.display_scoped.map(&:avg_score).compact.max %>
+          <% scoped_runs = v.runs.display_scoped %>
+          <% best_score = scoped_runs.map(&:avg_score).compact.max %>
+          <% best_pass_rate = scoped_runs.map(&:check_pass_rate).compact.max %>
           <% pred = predecessor_of[v] %>
           <tr class="<%= "ck-results-table__row--active" if v.id == @prompt.id %>" onclick="window.location='<%= prompt_path(v) %>'" style="cursor: pointer;">
             <td>
@@ -86,7 +88,11 @@
             <td>
               <% if best_score %>
                 <span class="<%= ck_badge_classes(ck_score_kind(best_score)) %>"><%= best_score %></span>
-              <% else %>
+              <% end %>
+              <% if best_pass_rate %>
+                <span class="<%= ck_badge_classes(ck_pass_rate_kind(best_pass_rate)) %>"><%= (best_pass_rate * 100).round %>%</span>
+              <% end %>
+              <% unless best_score || best_pass_rate %>
                 <span class="ck-prompts-table__dim">—</span>
               <% end %>
             </td>

data/app/views/completion_kit/responses/show.html.erb

@@ -116,6 +116,8 @@
                 <% 5.times do |i| %>
                   <svg viewBox="0 0 24 24" width="16" height="16" stroke-width="1.75" class="ck-star <%= i < review.ai_score.to_i ? "ck-star--filled" : "ck-star--empty" %>"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
                 <% end %>
+              <% elsif review.check? && !review.passed.nil? %>
+                <%= ck_check_badge(review.passed) %>
               <% else %>
                 <span class="<%= ck_badge_classes(:pending) %>">Pending</span>
               <% end %>

data/app/views/completion_kit/runs/_response_row.html.erb

@@ -10,14 +10,21 @@
     <% end %>
   </td>
   <td data-label="Metrics">
-    <% scored_reviews = response.reviews.select { |r| r.ai_score.present? }.sort_by { |r| r.metric_name.to_s.downcase } %>
-    <% if scored_reviews.any? %>
+    <% pip_reviews = response.reviews.select { |r| r.ai_score.present? || !r.passed.nil? }.sort_by { |r| r.metric_name.to_s.downcase } %>
+    <% if pip_reviews.any? %>
       <span class="ck-metric-bar ck-metric-bar--compact">
-        <% scored_reviews.each do |r| %>
-          <span class="ck-metric-pip ck-metric-pip--<%= ck_score_kind(r.ai_score.to_f) %>">
-            <span class="ck-metric-pip__bar"></span>
-            <span class="ck-metric-pip__label"><%= r.metric_name %> <strong><%= r.ai_score %></strong></span>
-          </span>
+        <% pip_reviews.each do |r| %>
+          <% if r.ai_score.present? %>
+            <span class="ck-metric-pip ck-metric-pip--<%= ck_score_kind(r.ai_score.to_f) %>">
+              <span class="ck-metric-pip__bar"></span>
+              <span class="ck-metric-pip__label"><%= r.metric_name %> <strong><%= r.ai_score %></strong></span>
+            </span>
+          <% else %>
+            <span class="ck-metric-pip ck-metric-pip--<%= ck_pass_rate_kind(r.passed ? 1.0 : 0.0) %>">
+              <span class="ck-metric-pip__bar"></span>
+              <span class="ck-metric-pip__label"><%= r.metric_name %> <strong><%= r.passed ? "Pass" : "Fail" %></strong></span>
+            </span>
+          <% end %>
         <% end %>
       </span>
     <% else %>
@@ -26,7 +33,12 @@
   </td>
   <td data-label="Avg score">
     <% if response.reviewed? %>
-      <span class="<%= ck_badge_classes(ck_score_kind(response.score.to_f)) %>"><%= response.score %></span>
+      <% if response.score %>
+        <span class="<%= ck_badge_classes(ck_score_kind(response.score.to_f)) %>"><%= response.score %></span>
+      <% end %>
+      <% if response.checks_total.positive? %>
+        <span class="<%= ck_badge_classes(ck_pass_rate_kind(response.checks_passed.to_f / response.checks_total)) %>"><%= response.checks_passed %>/<%= response.checks_total %></span>
+      <% end %>
     <% else %>
       <span class="ck-response-cell__dim">—</span>
     <% end %>

data/app/views/completion_kit/runs/_row.html.erb

@@ -34,10 +34,17 @@
     <% if metrics.any? %>
       <div class="ck-metric-bar ck-metric-bar--compact">
         <% metrics.each do |m| %>
-          <span class="ck-metric-pip ck-metric-pip--<%= ck_score_kind(m[:avg]) %>">
-            <span class="ck-metric-pip__bar"></span>
-            <span class="ck-metric-pip__label"><%= m[:name] %> <strong><%= m[:avg] %></strong></span>
-          </span>
+          <% if m[:kind] == "check" %>
+            <span class="ck-metric-pip ck-metric-pip--<%= ck_pass_rate_kind(m[:pass_rate]) %>">
+              <span class="ck-metric-pip__bar"></span>
+              <span class="ck-metric-pip__label"><%= m[:name] %> <strong><%= (m[:pass_rate] * 100).round %>%</strong></span>
+            </span>
+          <% else %>
+            <span class="ck-metric-pip ck-metric-pip--<%= ck_score_kind(m[:avg]) %>">
+              <span class="ck-metric-pip__bar"></span>
+              <span class="ck-metric-pip__label"><%= m[:name] %> <strong><%= m[:avg] %></strong></span>
+            </span>
+          <% end %>
         <% end %>
       </div>
     <% else %>
@@ -46,8 +53,14 @@
   </td>
   <td>
     <% avg = run.avg_score %>
-    <% if avg %>
-      <span class="<%= ck_badge_classes(ck_score_kind(avg)) %>"><%= avg %></span>
+    <% pass_rate = run.check_pass_rate %>
+    <% if avg || pass_rate %>
+      <% if avg %>
+        <span class="<%= ck_badge_classes(ck_score_kind(avg)) %>"><%= avg %></span>
+      <% end %>
+      <% if pass_rate %>
+        <span class="<%= ck_badge_classes(ck_pass_rate_kind(pass_rate)) %>"><%= (pass_rate * 100).round %>%</span>
+      <% end %>
     <% else %>
       <span class="ck-runs-table__dim">—</span>
     <% end %>

data/app/views/completion_kit/runs/_sort_toolbar.html.erb

@@ -1,5 +1,5 @@
 <div id="run_sort_toolbar">
-  <% if run.judge_configured? %>
+  <% if run.gradable? %>
     <% active = run.status == "completed" && run.responses.joins(:reviews).exists? %>
     <div class="ck-toolbar" style="margin-top: 1.5rem;<%= ' visibility: hidden;' unless active %>" aria-hidden="<%= !active %>">
       <%= link_to "Best first", run_path(run, sort: "score_desc"), class: params[:sort].blank? || params[:sort] == "score_desc" ? ck_button_classes(:dark) : ck_button_classes(:light, variant: :outline), tabindex: active ? nil : -1 %>

data/app/views/completion_kit/runs/_status_panel.html.erb

@@ -2,7 +2,9 @@
 <div id="run_status_panel" aria-live="polite" aria-atomic="true">
   <% if run.status.in?(%w[running completed]) && snap[:generated_total] > 0 %>
     <% failed_count = snap[:generated_failed] + snap[:judged_failed] %>
-    <% has_judge = snap[:judged_total] > 0 || run.judge_configured? %>
+    <% show_grading = snap[:judged_total] > 0 || run.gradable? %>
+    <% has_llm_metric = run.llm_metrics.any? %>
+    <% has_checks = run.check_metrics.any? %>
     <% metric_avgs = run.metric_averages.sort_by { |m| m[:name].to_s.downcase } %>
     <% metric_lookup = metric_avgs.index_by { |m| m[:name].to_s.downcase } %>
     <section class="ck-run-status ck-run-status--<%= run.status %>">
@@ -18,7 +20,7 @@
           <p class="ck-run-status__cell-value ck-run-status__summary-line">
             <span class="ck-run-status__summary-num"><%= snap[:generated_done] %></span>
             <span class="ck-run-status__summary-text">of <%= snap[:generated_total] %> responses</span>
-            <% if has_judge %>
+            <% if show_grading %>
               <span class="ck-run-status__summary-sep">·</span>
               <span class="ck-run-status__summary-num"><%= snap[:judged_done] %></span>
               <span class="ck-run-status__summary-text">of <%= snap[:judged_total] %> judged</span>
@@ -30,7 +32,7 @@
           </p>
         </div>
 
-        <% if has_judge %>
+        <% if show_grading %>
           <div class="ck-run-status__cell">
             <p class="ck-run-status__metric-label">Metrics</p>
             <div class="ck-run-status__cell-value">
@@ -38,7 +40,12 @@
                 <div class="ck-metric-bar ck-metric-bar--compact">
                   <% run.metrics.order(:name).each do |metric| %>
                     <% avg_for_metric = metric_lookup[metric.name.to_s.downcase] %>
-                    <% if avg_for_metric %>
+                    <% if avg_for_metric && avg_for_metric[:kind] == "check" %>
+                      <span class="ck-metric-pip ck-metric-pip--<%= ck_pass_rate_kind(avg_for_metric[:pass_rate]) %>">
+                        <span class="ck-metric-pip__bar"></span>
+                        <span class="ck-metric-pip__label"><%= metric.name %> <strong><%= (avg_for_metric[:pass_rate] * 100).round %>%</strong></span>
+                      </span>
+                    <% elsif avg_for_metric %>
                       <span class="ck-metric-pip ck-metric-pip--<%= ck_score_kind(avg_for_metric[:avg]) %>">
                         <span class="ck-metric-pip__bar"></span>
                         <span class="ck-metric-pip__label"><%= metric.name %> <strong><%= avg_for_metric[:avg] %></strong></span>
@@ -57,16 +64,31 @@
             </div>
           </div>
 
-          <div class="ck-run-status__cell">
-            <p class="ck-run-status__metric-label">Avg score</p>
-            <div class="ck-run-status__cell-value">
-              <% if run.avg_score %>
-                <span class="<%= ck_badge_classes(ck_score_kind(run.avg_score)) %> ck-badge--lg"><%= run.avg_score %></span>
-              <% else %>
-                <span class="ck-run-status__cell-empty">—</span>
-              <% end %>
+          <% if has_llm_metric %>
+            <div class="ck-run-status__cell">
+              <p class="ck-run-status__metric-label">Avg score</p>
+              <div class="ck-run-status__cell-value">
+                <% if run.avg_score %>
+                  <span class="<%= ck_badge_classes(ck_score_kind(run.avg_score)) %> ck-badge--lg"><%= run.avg_score %></span>
+                <% else %>
+                  <span class="ck-run-status__cell-empty">—</span>
+                <% end %>
+              </div>
             </div>
-          </div>
+          <% end %>
+
+          <% if has_checks %>
+            <div class="ck-run-status__cell">
+              <p class="ck-run-status__metric-label">Checks passed</p>
+              <div class="ck-run-status__cell-value">
+                <% if run.check_pass_rate %>
+                  <span class="<%= ck_badge_classes(ck_pass_rate_kind(run.check_pass_rate)) %> ck-badge--lg"><%= (run.check_pass_rate * 100).round %>%</span>
+                <% else %>
+                  <span class="ck-run-status__cell-empty">—</span>
+                <% end %>
+              </div>
+            </div>
+          <% end %>
         <% end %>
       </div>
 

data/app/views/completion_kit/runs/compare.html.erb

@@ -46,21 +46,35 @@
             <% end %>
             <td><%= pm[:metric_name] %></td>
             <td>
-              <% if pm[:left_score] %>
+              <% if pm[:kind] == "check" %>
+                <% if pm[:left_passed].nil? %>
+                  <span class="ck-meta-copy">—</span>
+                <% else %>
+                  <%= ck_check_badge(pm[:left_passed]) %>
+                <% end %>
+              <% elsif pm[:left_score] %>
                 <span class="<%= ck_badge_classes(ck_score_kind(pm[:left_score].to_f)) %>"><%= pm[:left_score] %></span>
               <% else %>
                 <span class="ck-meta-copy">—</span>
               <% end %>
             </td>
             <td>
-              <% if pm[:right_score] %>
+              <% if pm[:kind] == "check" %>
+                <% if pm[:right_passed].nil? %>
+                  <span class="ck-meta-copy">—</span>
+                <% else %>
+                  <%= ck_check_badge(pm[:right_passed]) %>
+                <% end %>
+              <% elsif pm[:right_score] %>
                 <span class="<%= ck_badge_classes(ck_score_kind(pm[:right_score].to_f)) %>"><%= pm[:right_score] %></span>
               <% else %>
                 <span class="ck-meta-copy">—</span>
               <% end %>
             </td>
             <td>
-              <% if pm[:delta] %>
+              <% if pm[:kind] == "check" %>
+                <%= ck_result_change_badge(pm[:result_change]) || content_tag(:span, "—", class: "ck-meta-copy") %>
+              <% elsif pm[:delta] %>
                 <% delta_class = pm[:delta] > 0 ? "ck-delta--positive" : pm[:delta] < 0 ? "ck-delta--negative" : "ck-delta--zero" %>
                 <span class="ck-delta <%= delta_class %>"><%= pm[:delta].positive? ? "+#{pm[:delta]}" : pm[:delta].to_s %></span>
               <% else %>

data/app/views/completion_kit/suggestions/show.html.erb

@@ -20,6 +20,9 @@
       <% if @run.avg_score %>
         <span class="<%= ck_badge_classes(ck_score_kind(@run.avg_score)) %>"><%= @run.avg_score %></span>
       <% end %>
+      <% if @run.check_pass_rate %>
+        <span class="<%= ck_badge_classes(ck_pass_rate_kind(@run.check_pass_rate)) %>"><%= (@run.check_pass_rate * 100).round %>%</span>
+      <% end %>
     </p>
   </div>
   <div class="ck-actions">

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.18.1"
+  VERSION = "0.19.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.18.1
+  version: 0.19.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -261,6 +261,7 @@ files:
 - app/controllers/completion_kit/runs_controller.rb
 - app/controllers/completion_kit/suggestions_controller.rb
 - app/controllers/completion_kit/tags_controller.rb
+- app/controllers/concerns/completion_kit/response_ordering.rb
 - app/controllers/concerns/completion_kit/tag_filtering.rb
 - app/helpers/completion_kit/application_helper.rb
 - app/jobs/completion_kit/application_job.rb
@@ -339,6 +340,7 @@ files:
 - app/services/completion_kit/prompt_improvement_service.rb
 - app/services/completion_kit/prompt_improvement_validator.rb
 - app/services/completion_kit/provider_endpoint.rb
+- app/services/completion_kit/run_comparison.rb
 - app/services/completion_kit/starter_metrics.rb
 - app/services/completion_kit/worker_health.rb
 - app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb
@@ -352,6 +354,7 @@ files:
 - app/views/completion_kit/api_reference/index.html.erb
 - app/views/completion_kit/dashboard/_eye_icon.html.erb
 - app/views/completion_kit/dashboard/_eye_off_icon.html.erb
+- app/views/completion_kit/dashboard/_failing_checks_card.html.erb
 - app/views/completion_kit/dashboard/_failures_card.html.erb
 - app/views/completion_kit/dashboard/_worst_metric_card.html.erb
 - app/views/completion_kit/dashboard/show.html.erb