completion-kit 0.7.0 → 0.9.0

data/app/views/completion_kit/calibrations/_trust_panel.html.erb

@@ -1,9 +1,9 @@
 <% stats = local_assigns[:stats] %>
 <% metric = local_assigns[:metric] %>
 <% anchor = metric&.name&.parameterize %>
+<% current_metric_version = metric && CompletionKit::MetricVersion.current.find_by(metric_id: metric.id) %>
 <% target_response = if (stats.sample_size.zero? || stats.counter_only?) && metric
      created_by = CompletionKit.config.username.presence || "operator"
-     current_metric_version = CompletionKit::MetricVersion.current.find_by(metric_id: metric.id)
      verdicted_ids = if current_metric_version
        CompletionKit::Calibration.where(metric_id: metric.id, created_by: created_by, metric_version_id: current_metric_version.id).pluck(:response_id)
      else
@@ -22,19 +22,26 @@
    end %>
 
 <p class="ck-trust-line ck-trust-line--<%= stats.gate %>">
-  <span class="ck-trust-line__label"><%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true" %>Calibration</span>
   <% if stats.sample_size.zero? %>
-    <span class="ck-trust-line__state">Not measured yet.</span>
-    <span class="ck-trust-line__hint">Needs <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts on the judge's scores.<% if prior_version_verdicts > 0 %> (<%= pluralize(prior_version_verdicts, "verdict") %> on prior versions, tied to that version's history.)<% end %><% if target_response %>
-      <%= link_to "Give a verdict →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-trust-line__link" %>
-    <% else %> Run this metric on a dataset, then give a verdict.<% end %></span>
+    <span class="ck-trust-line__lead">Not measured yet.</span>
+    <span class="ck-trust-line__hint">Needs <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> human reviews of the judge's scores.<% if prior_version_verdicts > 0 %> (<%= pluralize(prior_version_verdicts, "earlier-version review") %> kept on file.)<% end %></span>
+    <% if target_response %>
+      <%= link_to "Review a judge's score →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-cal-link" %>
+    <% end %>
   <% elsif stats.counter_only? %>
-    <span class="ck-trust-line__counter"><%= stats.sample_size %>/<%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %></span>
-    <span class="ck-trust-line__hint"><%= pluralize(stats.sample_size, "verdict") %><% if stats.short_to_target > 0 %> · <%= stats.short_to_target %> more before this can be measured<% end %><% if target_response %> · <%= link_to "Give another verdict →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-trust-line__link" %><% end %></span>
+    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Human reviews</span> <strong><%= stats.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %></strong></span>
+    <% if stats.short_to_target > 0 %><span class="ck-trust-line__hint"><%= stats.short_to_target %> more to report a rate</span><% end %>
+    <% if target_response %>
+      <%= link_to "Review another score →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-cal-link" %>
+    <% end %>
   <% else %>
-    <span class="ck-trust-line__score" title="Roughly how often the judge and the humans agreed.">~<%= (stats.agreement_point * 100).round %>%</span>
-    <span class="ck-trust-line__margin" title="The range we're confident the true rate sits in.">±<%= (stats.margin * 100).round %> pt</span>
-    <span class="ck-trust-line__gate" title="<%= stats.firm? ? 'Enough verdicts for a reliable read.' : 'Early read. Keep giving verdicts.' %>"><%= stats.firm? ? "settled" : "early" %></span>
-    <span class="ck-trust-line__hint"><%= pluralize(stats.sample_size, "verdict") %><% if stats.borderline_rate && stats.borderline_rate > 0 %><% level = stats.borderline_rate > 0.30 ? "danger" : stats.borderline_rate > 0.15 ? "warning" : "ok" %> · <span class="ck-trust-line__borderline ck-trust-line__borderline--<%= level %>" title="<%= level == 'ok' ? '' : 'Reviewers said the rubric was unclear here.' %>"><%= (stats.borderline_rate * 100).round %>% unclear</span><% end %></span>
+    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Agreement</span> <strong class="ck-trust-line__figure">~<%= (stats.agreement_point * 100).round %>%</strong></span>
+    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Margin</span> ±<%= (stats.margin * 100).round %> pt</span>
+    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Read</span> <%= stats.firm? ? "settled" : "early" %></span>
+    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Sample</span> <%= stats.sample_size %></span>
+    <% if stats.borderline_rate && stats.borderline_rate > 0 %>
+      <% level = stats.borderline_rate > 0.30 ? "danger" : stats.borderline_rate > 0.15 ? "warning" : "ok" %>
+      <span class="ck-cal-stat"><span class="ck-cal-stat__label">Unclear</span> <span class="ck-trust-line__borderline ck-trust-line__borderline--<%= level %>"><%= (stats.borderline_rate * 100).round %>%</span></span>
+    <% end %>
   <% end %>
 </p>

data/app/views/completion_kit/metrics/_form.html.erb

@@ -40,20 +40,19 @@
   <% if suggestion %>
     <div class="ck-suggestion-banner" role="status">
       <div class="ck-suggestion-banner__body">
-        <p class="ck-kicker">Proposed improvements</p>
-        <p class="ck-meta-copy">Based on your disagreements, the model proposed these changes to the instruction and rubric. Apply pieces inline below, take everything at once, try again, or discard.</p>
+        <p class="ck-kicker ck-kicker--icon"><%= heroicon_tag "sparkles", variant: :outline, class: "ck-magic-icon", "aria-hidden": "true" %>Proposed changes</p>
+        <p class="ck-meta-copy">Based on human reviews, here are some proposed changes to the metric.</p>
       </div>
       <div class="ck-suggestion-banner__actions">
-        <%= button_to "Try again", suggest_variants_metric_path(metric, back_to: "edit"),
-              method: :post, form_class: "inline-block",
-              class: ck_button_classes(:light, variant: :outline),
-              title: "Discard these improvements and ask the model for fresh ones.",
-              data: { turbo_confirm: "Replace these improvements with fresh ones from the model?" } %>
-        <%= button_to "Discard", dismiss_suggestion_metric_path(metric, draft_id: suggestion.id, back_to: "edit"),
-              method: :delete, form_class: "inline-block",
-              class: ck_button_classes(:light, variant: :outline),
-              data: { turbo_confirm: "Drop these improvements?" } %>
-        <%= button_to "Take everything", publish_draft_metric_path(metric, draft_id: suggestion.id),
+        <%= button_to suggest_variants_metric_path(metric, back_to: "edit"),
+              method: :post, form_class: "inline-block", class: "ck-icon-btn",
+              title: "Try again", "aria-label": "Try again",
+              data: { turbo_confirm: "Replace these changes with fresh ones from the model?" } do %><%= heroicon_tag "arrow-path", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
+        <%= button_to dismiss_suggestion_metric_path(metric, draft_id: suggestion.id, back_to: "edit"),
+              method: :delete, form_class: "inline-block", class: "ck-icon-btn",
+              title: "Discard these changes", "aria-label": "Discard",
+              data: { turbo_confirm: "Drop these changes?" } do %><%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
+        <%= button_to "Apply all", publish_draft_metric_path(metric, draft_id: suggestion.id),
               method: :post, form_class: "inline-block",
               class: ck_button_classes(:dark) %>
       </div>

data/app/views/completion_kit/metrics/edit.html.erb

@@ -10,6 +10,24 @@
   </div>
 </section>
 
+<% if CompletionKit.config.judge_calibration_enabled && @suggestion_draft.nil? && @edit_draft.nil? && @improve_disagreement_count.to_i.positive? %>
+  <div class="ck-suggestion-banner" role="status">
+    <div class="ck-suggestion-banner__body">
+      <p class="ck-kicker">Improve from reviews</p>
+      <p class="ck-meta-copy">Based on human reviews, the model can propose changes to this metric.</p>
+    </div>
+    <div class="ck-suggestion-banner__actions">
+      <%= button_to suggest_variants_metric_path(@metric, back_to: "edit"),
+            method: :post, form_class: "inline-block",
+            class: ck_button_classes(:light, variant: :outline) + " ck-button--sm",
+            data: { turbo_confirm: "Draft improvements to this metric from your human reviews? You can edit or apply them here before publishing." } do %>
+        <%= heroicon_tag "sparkles", variant: :outline, class: "ck-magic-icon", "aria-hidden": "true" %>
+        Suggest improvements
+      <% end %>
+    </div>
+  </div>
+<% end %>
+
 <%= render "form",
       metric: @metric,
       suggestion_draft: @suggestion_draft,

data/app/views/completion_kit/metrics/index.html.erb

@@ -28,23 +28,6 @@
         <tr onclick="window.location='<%= metric_path(metric) %>'" style="cursor: pointer;">
           <td>
             <%= link_to metric_path(metric), class: "ck-record-name" do %><strong><%= metric.name %></strong><% end %>
-            <% if CompletionKit.config.judge_calibration_enabled %>
-              <% s = CompletionKit::MetricCalibrationStats.for(metric) %>
-              <p class="ck-metrics-table__trust" title="Calibration: how often this metric's scores match the humans who reviewed them.">
-                <%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true", class: "ck-trust-icon" %>
-                <span class="ck-metrics-table__trust-label">Calibration</span>
-                <% if s.counter_only? %>
-                  <% if s.sample_size.zero? %>
-                    <span class="ck-metrics-table__trust-state">Not measured yet</span>
-                  <% else %>
-                    <%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts
-                  <% end %>
-                <% else %>
-                  <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read. Keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
-                  ±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %>
-                <% end %>
-              </p>
-            <% end %>
             <% if metric.tags.any? %>
               <div class="tag-marks-row">
                 <%= render "completion_kit/tags/marks", tags: metric.tags %>

data/app/views/completion_kit/metrics/show.html.erb

@@ -6,11 +6,6 @@
 <section class="ck-page-header">
   <div>
     <h1 class="ck-title"><%= @metric.name %></h1>
-    <% if CompletionKit.config.judge_calibration_enabled %>
-      <%= render "completion_kit/calibrations/trust_panel",
-            stats: CompletionKit::MetricCalibrationStats.for(@metric),
-            metric: @metric %>
-    <% end %>
     <% if @metric.tags.any? %>
       <div class="tag-marks-row tag-marks-row--header">
         <%= render "completion_kit/tags/marks", tags: @metric.tags %>
@@ -18,23 +13,6 @@
     <% end %>
   </div>
   <div class="ck-actions">
-    <% if CompletionKit.config.judge_calibration_enabled %>
-      <% if @suggestion_draft || @edit_draft %>
-        <% review_title = @suggestion_draft ? "The model proposed improvements based on your disagreements. Review and apply what you want." : "An unpublished draft of this metric is saved. Review and publish, or keep editing." %>
-        <%= link_to "Review changes →", edit_metric_path(@metric),
-              class: ck_button_classes(:dark),
-              title: review_title %>
-      <% elsif @improve_disagreement_count.positive? %>
-        <%= button_to "Improve the metric", suggest_variants_metric_path(@metric),
-              method: :post, form_class: "inline-block",
-              class: ck_button_classes(:light, variant: :outline),
-              title: "Ask the model to suggest improvements to this metric's instruction and rubric based on the disagreements collected so far.",
-              data: { turbo_confirm: "Ask the model for suggested improvements based on the disagreements collected so far?" } %>
-      <% else %>
-        <button type="button" class="<%= ck_button_classes(:light, variant: :outline) %>" disabled
-                title="Mark at least one case as Disagree before the model can suggest a change.">Improve the metric</button>
-      <% end %>
-    <% end %>
     <%= link_to "Edit", edit_metric_path(@metric), class: ck_button_classes(:light, variant: :outline) %>
   </div>
 </section>
@@ -64,15 +42,15 @@
   </div>
 </section>
 
-<% if CompletionKit.config.judge_calibration_enabled && @versions.size > 1 %>
+<% if CompletionKit.config.judge_calibration_enabled && @versions.any? %>
   <% predecessor_of = @versions.index_with { |v| @versions.detect { |o| o.version_number < v.version_number } } %>
-  <% version_changed = ->(v, pred) { pred && (pred.instruction.to_s != v.instruction.to_s || pred.rubric_bands != v.rubric_bands) } %>
   <section class="ck-card ck-card--spaced">
     <p class="ck-kicker">Versions</p>
     <table class="ck-results-table ck-metric-versions-table">
       <thead>
         <tr>
           <th scope="col">Version</th>
+          <th scope="col">&Delta; Change</th>
           <th scope="col">Source</th>
           <th scope="col">Created</th>
         </tr>
@@ -86,28 +64,35 @@
                 <div class="ck-version-cell__label">
                   <strong><%= v.version_label %></strong>
                   <% if v.current? %>
-                    <span class="ck-chip">Published</span>
+                    <span class="ck-version-state ck-version-state--live">Published</span>
                   <% elsif v.draft? %>
+                    <span class="ck-version-state">Draft</span>
                     <%= button_to "Publish", publish_draft_metric_path(@metric, draft_id: v.id),
                           method: :post, form_class: "inline-block",
-                          class: "ck-chip ck-chip--publish" %>
+                          class: "ck-chip ck-chip--cta" %>
                   <% else %>
+                    <span class="ck-version-state">Past</span>
                     <%= button_to "Make current", publish_draft_metric_path(@metric, draft_id: v.id),
                           method: :post, form_class: "inline-block",
                           class: "ck-chip ck-chip--publish",
-                          data: { turbo_confirm: "Roll the live judge back to #{v.version_label}? Calibration verdicts collected against the current version stay tied to it." } %>
+                          data: { turbo_confirm: "Make #{v.version_label} the version to use? #{v.version_label} will be used in test runs using this metric now. Reviews you have already given stay with the version they were made against." } %>
                   <% end %>
                 </div>
-                <% if version_changed.call(v, pred) %>
-                  <button type="button" class="ck-cell-link ck-cell-link--delta"
-                          title="What changed from #{pred.version_label}"
-                          onclick="document.getElementById('ck-mvdiff-<%= v.id %>').showModal()">&Delta;</button>
-                <% end %>
               </div>
             </td>
+            <td>
+              <% summary = v.change_summary_against(pred) %>
+              <% if summary %>
+                <button type="button" class="ck-change-link ck-change-link--<%= summary[:magnitude] %>"
+                        title="Compare with <%= pred.version_label %>"
+                        onclick="document.getElementById('ck-mvdiff-<%= v.id %>').showModal()"><%= summary[:label] %></button>
+              <% else %>
+                <span class="ck-meta-copy">—</span>
+              <% end %>
+            </td>
             <td>
               <% source_label, source_class = case v.source
-                                              when "suggestion" then ["AI suggestion", "ck-source-chip ck-source-chip--ai"]
+                                              when "suggestion" then ["AI calibration", "ck-source-chip ck-source-chip--ai"]
                                               when "edit" then ["Manual edit", "ck-source-chip ck-source-chip--manual"]
                                               when "revert" then ["Reverted", "ck-source-chip ck-source-chip--revert"]
                                               else ["Original", "ck-source-chip ck-source-chip--initial"]
@@ -115,7 +100,15 @@
               <span class="<%= source_class %>"><%= source_label %></span>
             </td>
             <td class="ck-meta-copy">
-              <time datetime="<%= v.created_at.utc.iso8601 %>" data-relative-time><%= time_ago_in_words(v.created_at) %> ago</time>
+              <div class="ck-version-created">
+                <time datetime="<%= v.created_at.utc.iso8601 %>" data-relative-time><%= time_ago_in_words(v.created_at) %> ago</time>
+                <% if v.draft? %>
+                  <%= button_to dismiss_suggestion_metric_path(@metric, draft_id: v.id),
+                        method: :delete, form_class: "inline-block", class: "ck-icon-btn",
+                        title: "Discard draft #{v.version_label}", "aria-label": "Discard draft #{v.version_label}",
+                        data: { turbo_confirm: "Discard draft #{v.version_label}? This can't be undone." } do %><%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
+                <% end %>
+              </div>
             </td>
           </tr>
         <% end %>
@@ -125,7 +118,7 @@
 
   <% @versions.each do |v| %>
     <% pred = predecessor_of[v] %>
-    <% next unless version_changed.call(v, pred) %>
+    <% next unless v.change_summary_against(pred) %>
     <dialog id="ck-mvdiff-<%= v.id %>" class="ck-modal" onclick="if(event.target===this)this.close()">
       <article class="ck-modal__panel" tabindex="-1" onclick="event.stopPropagation()">
         <header class="ck-modal__header">
@@ -148,85 +141,74 @@
               </div>
             </div>
           <% end %>
-          <% if pred.rubric_bands != v.rubric_bands %>
+          <% pred_bands = CompletionKit::Metric.normalize_rubric_bands(pred.rubric_bands) %>
+          <% v_bands = CompletionKit::Metric.normalize_rubric_bands(v.rubric_bands) %>
+          <% if pred_bands != v_bands %>
             <p class="ck-kicker ck-kicker--inset">Rubric changes</p>
             <%= render "completion_kit/metrics/rubric_diff",
-                  current_bands: pred.rubric_bands || [],
-                  draft_bands: v.rubric_bands || [] %>
+                  current_bands: pred_bands,
+                  draft_bands: v_bands %>
           <% end %>
         </div>
+        <footer class="ck-modal__footer ck-modal__footer--split">
+          <% if v.current? %>
+            <span class="ck-modal__foot-note">This is the metric's published version.</span>
+          <% elsif v.draft? %>
+            <span class="ck-modal__foot-note">Happy with it? Publish to use <%= v.version_label %> for this metric from now on. Tweak it with Edit.</span>
+            <span class="ck-modal__foot-actions">
+              <%= button_to dismiss_suggestion_metric_path(@metric, draft_id: v.id),
+                    method: :delete, form_class: "inline-block", class: "ck-icon-btn",
+                    title: "Discard draft #{v.version_label}", "aria-label": "Discard draft #{v.version_label}",
+                    data: { turbo_confirm: "Discard draft #{v.version_label}? This can't be undone." } do %><%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
+              <%= link_to "Edit", edit_metric_path(@metric), class: ck_button_classes(:light, variant: :outline) %>
+              <%= button_to "Publish #{v.version_label} →", publish_draft_metric_path(@metric, draft_id: v.id),
+                    method: :post, form_class: "inline-block", class: ck_button_classes(:dark) %>
+            </span>
+          <% else %>
+            <span class="ck-modal__foot-note">Roll this metric back to this version.</span>
+            <%= button_to "Make #{v.version_label} current →", publish_draft_metric_path(@metric, draft_id: v.id),
+                  method: :post, form_class: "inline-block", class: ck_button_classes(:dark),
+                  data: { turbo_confirm: "Make #{v.version_label} the version to use? #{v.version_label} will be used in test runs using this metric now. Reviews you have already given stay with the version they were made against." } %>
+          <% end %>
+        </footer>
       </article>
     </dialog>
   <% end %>
 <% end %>
 
-<% if CompletionKit.config.judge_calibration_enabled && @disagreements.any? %>
+<% if CompletionKit.config.judge_calibration_enabled %>
   <section class="ck-card ck-card--spaced">
-    <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Cases to learn from</p>
-      <span class="ck-chip"><%= pluralize(@disagreements.size, "case") %></span>
-    </div>
-    <% mixed_versions = @disagreements.any? { |c| c.metric_version_id != @published_metric_version.id } %>
-    <p class="ck-meta-copy">Cases where a reviewer's score didn't match the judge's. Pin useful ones with <strong>Remember this</strong> so the judge sees them next time it grades<%= " (pins flow into the current judge regardless of which version produced the verdict)" if mixed_versions %>.</p>
-    <% existing_ids = Array(@metric.few_shot_examples).map { |fs| fs["calibration_id"] } %>
-    <ul class="ck-disagreement-list">
-      <% @disagreements.each do |cal| %>
-        <% review = cal.response.reviews.find { |r| r.metric_id == @metric.id } %>
-        <% already = existing_ids.include?(cal.id) %>
-        <% cal_metric_version = cal.metric_version %>
-        <% on_current = cal_metric_version&.id == @published_metric_version.id %>
-        <li class="ck-disagreement<%= " ck-disagreement--remembered" if already %><%= " ck-disagreement--stale" unless on_current %>">
-          <div class="ck-disagreement__head">
-            <div class="ck-disagreement__scores">
-              <% if cal_metric_version && mixed_versions %>
-                <span class="ck-source-chip <%= on_current ? "ck-source-chip--current" : "ck-source-chip--past" %>" title="<%= on_current ? "Verdict on the live judge version." : "Verdict on a superseded judge version." %>"><%= cal_metric_version.version_label %></span>
-              <% end %>
-              <span class="ck-disagreement__scores-label">Judge</span>
-              <% if review&.ai_score %>
-                <span class="<%= ck_badge_classes(ck_score_kind(review.ai_score.to_f)) %>"><%= review.ai_score %></span>
-              <% else %>
-                <span class="ck-meta-copy">—</span>
-              <% end %>
-              <span class="ck-disagreement__scores-arrow">→</span>
-              <span class="ck-disagreement__scores-label">Human</span>
-              <% if cal.corrected_score %>
-                <span class="<%= ck_badge_classes(ck_score_kind(cal.corrected_score.to_f)) %>"><%= cal.corrected_score %></span>
-              <% else %>
-                <span class="ck-meta-copy">—</span>
-              <% end %>
-            </div>
-            <div class="ck-disagreement__action">
-              <% if already %>
-                <%= button_to "Forget",
-                      remove_few_shot_metric_path(@metric, calibration_id: cal.id),
-                      method: :delete,
-                      form_class: "inline-block",
-                      class: ck_button_classes(:light, variant: :outline),
-                      title: "Stop showing this case to the judge.",
-                      data: { turbo_confirm: "Stop showing this case to the judge?" } %>
-                <span class="ck-chip ck-chip--done" title="The judge sees this row when it grades for this metric.">Remembered</span>
-              <% else %>
-                <%= button_to "Remember this",
-                      add_few_shot_metric_path(@metric, calibration_id: cal.id),
-                      method: :post,
-                      form_class: "inline-block",
-                      class: ck_button_classes(:light, variant: :outline),
-                      title: "Pin this case so the judge sees it next time it grades for this metric." %>
-              <% end %>
-            </div>
-          </div>
-          <% if cal.note.to_s.present? %>
-            <p class="ck-disagreement__note"><%= cal.note %></p>
-          <% end %>
-          <p class="ck-disagreement__source ck-meta-copy">
-            <%= link_to run_response_path(cal.response.run, cal.response, anchor: @metric.name.parameterize),
-                    class: "ck-disagreement__source-link" do %>
-              <% case_display = cal.response.row_index.nil? ? "##{cal.response.id}" : (cal.response.row_index + 1).to_s %>
-              View case <%= case_display %> in <%= cal.response.run.name.to_s.truncate(50) %> →
-            <% end %>
-          </p>
-        </li>
-      <% end %>
-    </ul>
+    <p class="ck-kicker">Calibration</p>
+    <p class="ck-meta-copy">This is a measure of how often the judge's scores match a human reviewer. Review its scores to build that signal; the scores you disagree with become the cases the model learns from when you improve the metric.</p>
+    <%= render "completion_kit/calibrations/trust_panel",
+          stats: CompletionKit::MetricCalibrationStats.for(@metric),
+          metric: @metric %>
+    <% draft = @suggestion_draft || @edit_draft %>
+    <% if draft %>
+      <div class="ck-cal-foot">
+        <span class="ck-cal-foot__note">A draft improvement (<%= draft.version_label %>) is waiting in the Versions table above. Open its change to compare, then Publish to use it.</span>
+      </div>
+    <% elsif @improve_disagreement_count.positive? %>
+      <div class="ck-cal-foot">
+        <span class="ck-cal-foot__note"><%= pluralize(@improve_disagreement_count, "case") %> where a reviewer's score didn't match the judge.</span>
+        <%= button_to suggest_variants_metric_path(@metric),
+              method: :post, form_class: "inline-block",
+              class: ck_button_classes(:light, variant: :outline) + " ck-button--sm",
+              data: { turbo_confirm: "Draft improvements to this metric from your human reviews? It stays a draft until you compare it and publish." } do %>
+          <%= heroicon_tag "sparkles", variant: :outline, class: "ck-magic-icon", "aria-hidden": "true" %>
+          Suggest improvements
+        <% end %>
+      </div>
+    <% end %>
   </section>
 <% end %>
+
+<% if params[:show_change].present? %>
+  <script>
+    (function () {
+      var dialog = document.getElementById("ck-mvdiff-<%= params[:show_change].to_i %>");
+      if (dialog && typeof dialog.showModal === "function") dialog.showModal();
+    })();
+  </script>
+<% end %>
+

data/app/views/completion_kit/responses/show.html.erb

@@ -105,7 +105,7 @@
             <span class="ck-review-card__metric"><% if review.metric %><%= link_to review.metric_name, metric_path(review.metric), class: "ck-link" %><% else %><%= review.metric_name %><% end %></span>
             <div class="ck-inline">
               <% if review_version %>
-                <span class="ck-source-chip <%= stale ? "ck-source-chip--past" : "ck-source-chip--current" %>" title="<%= stale ? "Score produced by #{review_version.version_label} of this metric. The live judge has changed since." : "Score produced by the live judge (#{review_version.version_label})." %>"><%= review_version.version_label %></span>
+                <span class="ck-source-chip <%= stale ? "ck-source-chip--past" : "ck-source-chip--current" %>" title="<%= stale ? "Scored against #{review_version.version_label} of this metric. The metric has been republished since." : "Scored against the metric's current version (#{review_version.version_label})." %>"><%= review_version.version_label %></span>
               <% end %>
               <% if review.ai_score %>
                 <% 5.times do |i| %>
@@ -117,7 +117,7 @@
             </div>
           </div>
           <% if stale %>
-            <p class="ck-review-card__stale-note">Scored against a superseded version of this metric. The live judge may score this differently.</p>
+            <p class="ck-review-card__stale-note">Scored against a superseded version of this metric. Its current version may score this differently.</p>
           <% end %>
           <% if review.ai_feedback.present? %>
             <p class="ck-review-card__feedback"><%= review.ai_feedback %></p>

data/app/views/completion_kit/runs/show.html.erb

@@ -23,25 +23,25 @@
   <% if stale_summary.any? %>
     <div class="ck-stale-versions-banner" role="status">
       <div class="ck-stale-versions-banner__body">
-        <p class="ck-kicker">Stale judge versions</p>
+        <p class="ck-kicker">Stale metric versions</p>
         <p class="ck-meta-copy">
           This run was scored against metric versions that are no longer live.
           <% stale_summary.values.each_with_index do |s, i| %>
             <%= ", " if i > 0 %><strong><%= s[:metric_name] %></strong> (scored by <%= s[:scored_labels].join(", ") %>; live is <%= s[:current_label] %>)<% end %>.
-          Re-run to refresh the scores with the current judge.
+          Re-run to refresh the scores with the current metrics.
         </p>
       </div>
       <% if @run.status == "completed" %>
         <%= button_to "Re-run from scratch",
               rerun_run_path(@run), method: :post,
               class: ck_button_classes(:light, variant: :outline), form_class: "inline-block",
-              title: "Create a new run that regenerates responses and grades them with the current judge.",
-              data: { turbo_confirm: "Create a new run with fresh responses and the current judge? The original run stays as a record." } %>
-        <%= button_to "Re-grade with current judge",
+              title: "Create a new run that regenerates responses and grades them with the current metrics.",
+              data: { turbo_confirm: "Create a new run with fresh responses and the current metrics? The original run stays as a record." } %>
+        <%= button_to "Re-grade with current metrics",
               regrade_run_path(@run), method: :post,
               class: ck_button_classes(:dark), form_class: "inline-block",
-              title: "Re-judge this run's existing responses against the current judge. Faster and cheaper than re-running.",
-              data: { turbo_confirm: "Re-judge this run's existing responses against the current judge?" } %>
+              title: "Re-grade this run's existing responses against the current metrics. Faster and cheaper than re-running.",
+              data: { turbo_confirm: "Re-grade this run's existing responses against the current metrics?" } %>
       <% end %>
     </div>
   <% end %>

data/config/routes.rb

@@ -19,8 +19,6 @@ CompletionKit::Engine.routes.draw do
       post "starters/:key/dismiss", to: "metrics#dismiss_starter", as: :dismiss_starter
     end
     member do
-      post   :add_few_shot
-      delete :remove_few_shot
       post :publish_draft
       post :suggest_variants
       delete :dismiss_suggestion
@@ -89,8 +87,6 @@ CompletionKit::Engine.routes.draw do
         end
         member do
           post :suggest_variants
-          post :add_few_shot
-          delete :remove_few_shot
         end
       end
       resources :metric_groups

data/db/migrate/20260529000001_remove_few_shot_examples_from_completion_kit_metrics.rb

@@ -0,0 +1,5 @@
+class RemoveFewShotExamplesFromCompletionKitMetrics < ActiveRecord::Migration[8.1]
+  def change
+    remove_column :completion_kit_metrics, :few_shot_examples, :text
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.7.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -429,6 +429,7 @@ files:
 - db/migrate/20260525000001_add_version_number_and_published_at_to_judge_versions.rb
 - db/migrate/20260528000001_rename_judge_version_to_metric_version.rb
 - db/migrate/20260528000002_add_metric_version_to_reviews.rb
+- db/migrate/20260529000001_remove_few_shot_examples_from_completion_kit_metrics.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb