completion-kit 0.5.37 → 0.5.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ccc7d1feb86aed6af17569a642d8b8e81fe522f0a7c68ca4ebb34abc113dbce
-  data.tar.gz: 88793eabe6b04c3497c761cde5b61511623c5a9844ce7e101560f8eb3b492e18
+  metadata.gz: fc7d527828189c2993060b315dca634fb958d2da11fd7fae63c4790179c46701
+  data.tar.gz: f0323b980bdfb35d36742b548ddd3629e66d39e587775521678dc80b4cd2f068
 SHA512:
-  metadata.gz: d133a9d0db55ee41eb07e290b9657e044c8a0836806bbd055d0b7b6d1cf8b981056b40e3a8795a951e36e7d7dbcc7626e249c2a2f4cba9492fad38aa931b6bfc
-  data.tar.gz: 71bbbe827f33648b12f121c949af74fe8d02702d44c70fd39beb4795c8f95d2b9941aa755d631a78029e0b412cdec7ca9e2bea107d1395ea001abe46dcfddf3f
+  metadata.gz: '020946bdac698194bb5246cfbe21fdf45c56006c80c15d1d7bcfda4d3494d95cde45645e090df14a411b172c83dcde42be777d74811b25a340e5710dba6ae7ce'
+  data.tar.gz: 1b4f0ea8cf4e613df783ac428404ef1ae19b285db04f8a6768119760c65fb81d9ee72d52905ab2ef30e077ae910eb00e622925c1dcc43aee0c9e3a0f748718b1

data/app/assets/stylesheets/completion_kit/application.css.erb

@@ -5361,3 +5361,14 @@ a.tag-mark {
   border-radius: 4px;
   border: 1px solid var(--ck-line);
 }
+
+.ck-metrics-table__trust {
+  font-family: var(--ck-mono);
+  font-size: 0.78rem;
+  letter-spacing: 0.03em;
+}
+.ck-metrics-table__trust-rate {
+  font-weight: 600;
+  color: var(--ck-success);
+  margin-right: 6px;
+}

data/app/controllers/completion_kit/metrics_controller.rb

@@ -54,12 +54,14 @@ module CompletionKit
         return
       end
       generator.persist!(variants)
-      label = variants.length == 1 ? "judge variant" : "judge variants"
-      redirect_to metric_path(@metric), notice: "Generated #{variants.length} #{label} as drafts. Pick one to publish."
+      label = variants.length == 1 ? "alternative" : "alternatives"
+      redirect_to metric_path(@metric), notice: "Wrote #{variants.length} #{label} for the judge instruction. Pick one to make it live."
     end
 
     def publish_draft
-      draft = JudgeVersion.drafts.where(metric_id: @metric.id).order(created_at: :desc).first
+      scope = JudgeVersion.drafts.where(metric_id: @metric.id)
+      draft = params[:draft_id].present? ? scope.find_by(id: params[:draft_id]) : scope.order(created_at: :desc).first
+
       if draft.nil?
         redirect_to metric_path(@metric), alert: "No draft to publish."
         return
@@ -68,9 +70,13 @@ module CompletionKit
       JudgeVersion.transaction do
         JudgeVersion.where(metric_id: @metric.id, state: "published").update_all(current: false)
         draft.update!(state: "published", current: true)
+        @metric.update_columns(
+          instruction: draft.instruction,
+          rubric_bands: Array(draft.rubric_bands).to_json
+        )
       end
 
-      redirect_to metric_path(@metric), notice: "Draft published as the current judge version."
+      redirect_to metric_path(@metric), notice: "This judge version is now live."
     end
 
     def add_few_shot
@@ -88,7 +94,7 @@ module CompletionKit
         "added_at" => Time.current.utc.iso8601
       }
       @metric.update!(few_shot_examples: examples)
-      redirect_to metric_path(@metric), notice: "Added as a judge few-shot."
+      redirect_to metric_path(@metric), notice: "Saved as a teaching example. The judge will see it next time it grades."
     end
 
     private

data/app/views/completion_kit/calibrations/_buttons.html.erb

@@ -4,17 +4,25 @@
   <p class="ck-calibration__prompt">
     Your verdict
     <% if verdict_count > 0 %>
-      <span class="ck-calibration__count"><%= pluralize(verdict_count, "verdict") %> on this score</span>
+      <span class="ck-calibration__count"><%= pluralize(verdict_count, "verdict") %> on this score · <%= link_to "judge trust →", metric_path(metric), class: "ck-link" %></span>
+    <% else %>
+      <span class="ck-calibration__count">Tell us what you think — was the score right? Verdicts roll up into <%= link_to "judge trust", metric_path(metric), class: "ck-link" %>.</span>
     <% end %>
   </p>
   <div class="ck-calibration__buttons">
     <% verdict_icons = { "agree" => "hand-thumb-up", "disagree" => "hand-thumb-down", "borderline" => "scale" } %>
+    <% verdict_hints = {
+         "agree" => "The score looks right.",
+         "disagree" => "The score is wrong — you'll pick the right one.",
+         "borderline" => "The rubric is unclear here; either score could be defensible."
+       } %>
     <% CompletionKit::Calibration::VERDICTS.each do |verdict| %>
       <%= button_to run_response_calibrations_path(run, response_row, metric_id: metric.id, verdict: verdict),
               method: :post,
               form: { data: { turbo: "true" } },
               class: "ck-calibration__pill ck-calibration__pill--#{verdict}#{' is-active' if verdict == current_verdict}",
-              "aria-pressed": (verdict == current_verdict).to_s do %>
+              "aria-pressed": (verdict == current_verdict).to_s,
+              title: verdict_hints[verdict] do %>
         <%= heroicon_tag verdict_icons[verdict], variant: :outline, size: 14, "aria-hidden": "true" %>
         <span><%= verdict %></span>
       <% end %>
@@ -28,7 +36,7 @@
       <%= hidden_field_tag :metric_id, metric.id %>
       <%= hidden_field_tag :verdict, "disagree" %>
       <label class="ck-label">
-        Your score
+        What should the score have been?
         <span class="ck-calibration__value" data-calibration-value><%= calibration.corrected_score || review&.ai_score || 3 %></span>
       </label>
       <input type="range" name="corrected_score" min="1" max="5" step="0.5"

data/app/views/completion_kit/calibrations/_trust_panel.html.erb

@@ -1,34 +1,31 @@
 <% stats = local_assigns[:stats] %>
 <div class="ck-trust-panel ck-trust-panel--<%= stats.gate %>">
-  <p class="ck-trust-panel__label">Judge trust</p>
+  <p class="ck-trust-panel__label" title="How often the judge agrees with the humans who reviewed its scores.">Judge trust</p>
   <% if stats.counter_only? %>
     <div class="ck-trust-panel__body">
       <span class="ck-trust-panel__counter"><%= stats.sample_size %><span class="ck-trust-panel__counter-of">/ <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %></span></span>
-      <span class="ck-trust-panel__hint">verdicts<% if stats.short_to_target > 0 %> · <%= pluralize(stats.short_to_target, "more") %> to score<% end %></span>
+      <span class="ck-trust-panel__hint">verdicts so far<% if stats.short_to_target > 0 %> · <%= stats.short_to_target %> more before we can score the judge<% end %></span>
     </div>
   <% else %>
     <div class="ck-trust-panel__body">
-      <span class="ck-trust-panel__score">~<%= (stats.agreement_point * 100).round %><span class="ck-trust-panel__score-pct">%</span></span>
-      <span class="ck-trust-panel__margin">±<%= (stats.margin * 100).round %> pt</span>
-      <span class="ck-trust-panel__gate"><%= stats.firm? ? "settled" : "provisional" %></span>
+      <span class="ck-trust-panel__score"
+            title="Roughly how often the judge and the humans agreed.">~<%= (stats.agreement_point * 100).round %><span class="ck-trust-panel__score-pct">%</span></span>
+      <span class="ck-trust-panel__margin"
+            title="The range we're confident the true rate sits in, given how few verdicts we have so far.">±<%= (stats.margin * 100).round %> pt</span>
+      <span class="ck-trust-panel__gate"
+            title="<%= stats.firm? ? 'Enough verdicts for a reliable read.' : 'Early read. Keep giving verdicts to tighten the margin.' %>"><%= stats.firm? ? "settled" : "early read" %></span>
     </div>
     <div class="ck-trust-panel__details">
-      <span><%= stats.sample_size %> verdicts</span>
+      <span><%= pluralize(stats.sample_size, "verdict") %></span>
       <% if stats.borderline_rate && stats.borderline_rate > 0 %>
         <% level = if stats.borderline_rate > 0.30 then "danger"
                    elsif stats.borderline_rate > 0.15 then "warning"
                    else "ok" end %>
         <span class="ck-trust-panel__borderline ck-trust-panel__borderline--<%= level %>"
-              title="<%= level == 'ok' ? '' : 'Rubric ambiguous. Consider splitting the metric or clarifying the rubric.' %>">
-          <%= (stats.borderline_rate * 100).round %>% borderline
+              title="<%= level == 'ok' ? 'Some reviewers said the rubric was unclear here.' : 'A lot of reviewers say the rubric is unclear here. Consider splitting the metric or rewriting the rubric.' %>">
+          <%= (stats.borderline_rate * 100).round %>% said "unclear"
         </span>
       <% end %>
-      <% if stats.mae %>
-        <span>MAE <%= stats.mae.round(2) %></span>
-      <% end %>
-      <% if stats.kappa %>
-        <span>κ <%= stats.kappa.round(2) %></span>
-      <% end %>
     </div>
   <% end %>
 </div>

data/app/views/completion_kit/metrics/index.html.erb

@@ -19,6 +19,7 @@
       <tr>
         <th scope="col">Name</th>
         <th scope="col">Instruction</th>
+        <th scope="col" title="How often the judge agrees with the humans who reviewed its scores.">Judge trust</th>
         <th scope="col">In groups</th>
         <th scope="col"></th>
       </tr>
@@ -35,6 +36,23 @@
             <% end %>
           </td>
           <td data-label="Instruction" class="ck-meta-copy"><div class="ck-clamp-2"><%= metric.instruction.presence || "—" %></div></td>
+          <td data-label="Judge trust" class="ck-metrics-table__trust">
+            <% if CompletionKit.config.judge_calibration_enabled %>
+              <% s = CompletionKit::MetricCalibrationStats.for(metric) %>
+              <% if s.counter_only? %>
+                <% if s.sample_size.zero? %>
+                  <span class="ck-meta-copy">No verdicts yet</span>
+                <% else %>
+                  <span class="ck-meta-copy"><%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts</span>
+                <% end %>
+              <% else %>
+                <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read — keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
+                <span class="ck-meta-copy">±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %></span>
+              <% end %>
+            <% else %>
+              <span class="ck-meta-copy">—</span>
+            <% end %>
+          </td>
           <td data-label="In groups">
             <% groups = metric.metric_groups %>
             <% if groups.any? %>

data/app/views/completion_kit/metrics/show.html.erb

@@ -12,8 +12,8 @@
       <% if @latest_draft %>
         <div class="ck-draft-banner">
           <span class="ck-chip ck-chip--soft">Draft pending</span>
-          <span class="ck-meta-copy">An edit forked a draft judge version. Publish it to make this the current judge.</span>
-          <%= button_to "Publish draft", publish_draft_metric_path(@metric),
+          <span class="ck-meta-copy">A draft version of this judge is saved. Publishing it replaces the live instruction and rubric.</span>
+          <%= button_to "Publish this version", publish_draft_metric_path(@metric, draft_id: @latest_draft.id),
                 method: :post, form_class: "inline-block",
                 class: ck_button_classes(:dark) %>
         </div>
@@ -22,10 +22,11 @@
   </div>
   <div class="ck-actions">
     <% if CompletionKit.config.judge_calibration_enabled %>
-      <%= button_to "Suggest improvements", suggest_variants_metric_path(@metric),
+      <%= button_to "Suggest rewrites", suggest_variants_metric_path(@metric),
             method: :post, form_class: "inline-block",
             class: ck_button_classes(:light, variant: :outline),
-            data: { turbo_confirm: "Ask the model to propose new judge instructions based on the disagreements collected so far?" } %>
+            title: "Ask the model to rewrite this judge instruction based on the disagreements collected so far.",
+            data: { turbo_confirm: "Ask the model to rewrite this judge instruction based on the disagreements collected so far?" } %>
     <% end %>
     <%= link_to "Edit", edit_metric_path(@metric), class: ck_button_classes(:light, variant: :outline) %>
   </div>
@@ -65,14 +66,15 @@
 <% if CompletionKit.config.judge_calibration_enabled %>
   <section class="ck-card ck-card--spaced">
     <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Disagreements</p>
+      <p class="ck-kicker">Where the judge got it wrong</p>
       <% if @disagreements.any? %>
         <span class="ck-chip"><%= pluralize(@disagreements.size, "row") %></span>
       <% end %>
     </div>
     <% if @disagreements.empty? %>
-      <p class="ck-meta-copy">No disagreements yet. As humans give the verdict "disagree" on individual rows, the judge's misses will show up here for review.</p>
+      <p class="ck-meta-copy">Nothing here yet. As people give a "disagree" verdict on response rows, those rows show up below so you can review the judge's misses and turn them into teaching examples.</p>
     <% else %>
+      <p class="ck-meta-copy">Rows where a reviewer said the judge got it wrong. Save the best ones as teaching examples — the judge will see them next time it grades.</p>
       <table class="ck-results-table ck-disagreements-table">
         <thead>
           <tr>
@@ -112,13 +114,14 @@
               <td class="ck-meta-copy"><%= cal.note.to_s.truncate(120) %></td>
               <td>
                 <% if already %>
-                  <span class="ck-chip ck-chip--done">Added</span>
+                  <span class="ck-chip ck-chip--done">Saved as example</span>
                 <% else %>
-                  <%= button_to "Add as judge few-shot",
+                  <%= button_to "Teach the judge",
                         add_few_shot_metric_path(@metric, calibration_id: cal.id),
                         method: :post,
                         form_class: "inline-block",
-                        class: ck_button_classes(:light, variant: :outline) %>
+                        class: ck_button_classes(:light, variant: :outline),
+                        title: "Save this row as a teaching example. The judge will see it next time it grades." %>
                 <% end %>
               </td>
             </tr>
@@ -131,20 +134,20 @@
   <% if @suggestion_drafts.any? %>
     <section class="ck-card ck-card--spaced">
       <div class="ck-prompt-preview__header">
-        <p class="ck-kicker">Suggested judge variants</p>
-        <span class="ck-chip"><%= @suggestion_drafts.size %> draft<%= @suggestion_drafts.size == 1 ? "" : "s" %></span>
+        <p class="ck-kicker">Suggested rewrites</p>
+        <span class="ck-chip"><%= @suggestion_drafts.size %> option<%= @suggestion_drafts.size == 1 ? "" : "s" %></span>
       </div>
-      <p class="ck-meta-copy">Pick one and publish it to make it the current judge. The previous published version stays in history.</p>
+      <p class="ck-meta-copy">The model wrote these alternate instructions based on the disagreements above. Pick one to make it the live judge — the previous version is kept in history.</p>
       <div class="ck-suggestion-list">
         <% @suggestion_drafts.each do |draft| %>
           <article class="ck-suggestion-card">
             <header class="ck-suggestion-card__header">
-              <span class="ck-chip ck-chip--soft">Draft #<%= draft.id %></span>
+              <span class="ck-chip ck-chip--soft">Option #<%= draft.id %></span>
               <time class="ck-meta-copy" data-relative-time datetime="<%= draft.created_at.utc.iso8601 %>"><%= time_ago_in_words(draft.created_at) %> ago</time>
             </header>
             <pre class="ck-code ck-suggestion-card__instruction"><%= draft.instruction %></pre>
             <div class="ck-actions">
-              <%= button_to "Publish this draft", publish_draft_metric_path(@metric),
+              <%= button_to "Use this version", publish_draft_metric_path(@metric, draft_id: draft.id),
                     method: :post, form_class: "inline-block",
                     class: ck_button_classes(:dark) %>
             </div>
@@ -157,10 +160,10 @@
   <% if Array(@metric.few_shot_examples).any? %>
     <section class="ck-card ck-card--spaced">
       <div class="ck-prompt-preview__header">
-        <p class="ck-kicker">Judge few-shot examples</p>
+        <p class="ck-kicker">Teaching examples</p>
         <span class="ck-chip"><%= pluralize(Array(@metric.few_shot_examples).size, "example") %></span>
       </div>
-      <p class="ck-meta-copy">Disagreements added here will be injected as worked examples when the judge runs on this metric. Used by Phase 4 / 5 to retrain the judge.</p>
+      <p class="ck-meta-copy">The judge sees these worked examples whenever it grades for this metric. Each shows what the judge gave and what a human said it should have been.</p>
       <ol class="ck-few-shot-list">
         <% Array(@metric.few_shot_examples).each do |fs| %>
           <li class="ck-few-shot-item">

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.37"
+  VERSION = "0.5.38"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.37
+  version: 0.5.38
 platform: ruby
 authors:
 - Damien Bastin