completion-kit 0.5.40 → 0.5.41

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8875fa22de9c8626e401706818b271f3cb26bffa7faae1f583b29503228cead
-  data.tar.gz: 5bf39fad883b2eed2f505b11403ab60fddf9efc5735073ff61c290d53f59a36d
+  metadata.gz: c92ab72bfe3b2fe9fa296a21e8430f7b8c3d9f949a933f7675f4ffd8059dd8ae
+  data.tar.gz: ec53ee3e3b29a4d283db95d90e7a1dd8afef3572647ae243c9458500600714c9
 SHA512:
-  metadata.gz: a8f2a7f14235c1214b567b891defaf523a645a21a2409ed81df964893a260cccb1fc9bf63903794f952c91bc9c91f3c1e3850db751a08ce0edc49b360ad9642d
-  data.tar.gz: e148b500e498a00dc370bf203fea3e2618b9f1c8fccd1dc5f220ae77a4988a8934198ad32bbf25a29bc84f948c687fab9a0ae16b9c03fe575fe9fca4cf98a0ea
+  metadata.gz: 8b783ec1b478a365f9e8a04da486fc7a87251d53dd6c63720c7d88795a524546bf71da8a07eca79337863c6787f52c776e51c75940b854a6e477f8074bf1d016
+  data.tar.gz: 7444db06e0adf5e29c7d68426496e643c743187d9cc8d974367cfdf025c993c0e061ad2b4b3f9d6a70f1aca3d70d124744b2a7ae416d5827efab9b6339b496f7

data/app/assets/stylesheets/completion_kit/application.css

@@ -5413,9 +5413,11 @@ a.tag-mark {
 }
 
 .ck-metrics-table__trust {
+  margin: 4px 0 0;
   font-family: var(--ck-mono);
-  font-size: 0.78rem;
+  font-size: 0.72rem;
   letter-spacing: 0.03em;
+  color: var(--ck-dim);
 }
 .ck-metrics-table__trust-rate {
   font-weight: 600;
@@ -5485,3 +5487,54 @@ a.tag-mark {
 @keyframes ck-saved-flash {
   0% { background: var(--ck-success); border-color: var(--ck-success); }
 }
+
+.ck-disagreement-list {
+  list-style: none;
+  padding: 0;
+  margin: 12px 0 0;
+  display: flex;
+  flex-direction: column;
+  gap: 12px;
+}
+.ck-disagreement {
+  padding: 14px;
+  background: var(--ck-surface-soft);
+  border: 1px solid var(--ck-line);
+  border-radius: 6px;
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+}
+.ck-disagreement__head {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 12px;
+  flex-wrap: wrap;
+}
+.ck-disagreement__scores {
+  display: inline-flex;
+  align-items: center;
+  gap: 8px;
+  flex-wrap: wrap;
+}
+.ck-disagreement__scores-label {
+  font-family: var(--ck-mono);
+  font-size: 0.7rem;
+  letter-spacing: 0.08em;
+  text-transform: uppercase;
+  color: var(--ck-dim);
+}
+.ck-disagreement__scores-arrow {
+  color: var(--ck-dim);
+}
+.ck-disagreement__note {
+  margin: 0;
+  color: var(--ck-text);
+  font-size: 0.92rem;
+  line-height: 1.45;
+}
+.ck-disagreement__source {
+  margin: 0;
+  font-size: 0.78rem;
+}

data/app/controllers/completion_kit/metrics_controller.rb

@@ -109,7 +109,7 @@ module CompletionKit
         "added_at" => Time.current.utc.iso8601
       }
       @metric.update!(few_shot_examples: examples)
-      redirect_to metric_path(@metric), notice: "Saved as a teaching example. The judge will see it next time it grades."
+      redirect_to metric_path(@metric), notice: "Got it. The judge will remember this next time it grades."
     end
 
     private

data/app/views/completion_kit/metrics/index.html.erb

@@ -19,7 +19,6 @@
       <tr>
         <th scope="col">Name</th>
         <th scope="col">Instruction</th>
-        <th scope="col" title="How often this metric's scores match the humans who reviewed them.">Trust level</th>
         <th scope="col">In groups</th>
         <th scope="col"></th>
       </tr>
@@ -29,6 +28,21 @@
         <tr onclick="window.location='<%= metric_path(metric) %>'" style="cursor: pointer;">
           <td>
             <%= link_to metric_path(metric), class: "ck-record-name" do %><strong><%= metric.name %></strong><% end %>
+            <% if CompletionKit.config.judge_calibration_enabled %>
+              <% s = CompletionKit::MetricCalibrationStats.for(metric) %>
+              <p class="ck-metrics-table__trust">
+                <% if s.counter_only? %>
+                  <% if s.sample_size.zero? %>
+                    No verdicts yet
+                  <% else %>
+                    <%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts
+                  <% end %>
+                <% else %>
+                  <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read — keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
+                  ±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %>
+                <% end %>
+              </p>
+            <% end %>
             <% if metric.tags.any? %>
               <div class="tag-marks-row">
                 <%= render "completion_kit/tags/marks", tags: metric.tags %>
@@ -36,23 +50,6 @@
             <% end %>
           </td>
           <td data-label="Instruction" class="ck-meta-copy"><div class="ck-clamp-2"><%= metric.instruction.presence || "—" %></div></td>
-          <td data-label="Trust level" class="ck-metrics-table__trust">
-            <% if CompletionKit.config.judge_calibration_enabled %>
-              <% s = CompletionKit::MetricCalibrationStats.for(metric) %>
-              <% if s.counter_only? %>
-                <% if s.sample_size.zero? %>
-                  <span class="ck-meta-copy">No verdicts yet</span>
-                <% else %>
-                  <span class="ck-meta-copy"><%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts</span>
-                <% end %>
-              <% else %>
-                <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read — keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
-                <span class="ck-meta-copy">±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %></span>
-              <% end %>
-            <% else %>
-              <span class="ck-meta-copy">—</span>
-            <% end %>
-          </td>
           <td data-label="In groups">
             <% groups = metric.metric_groups %>
             <% if groups.any? %>

data/app/views/completion_kit/metrics/show.html.erb

@@ -140,82 +140,69 @@
   </section>
 <% end %>
 
-<% if CompletionKit.config.judge_calibration_enabled %>
+<% if CompletionKit.config.judge_calibration_enabled && @disagreements.any? %>
   <section class="ck-card ck-card--spaced">
     <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Where the judge got it wrong</p>
-      <% if @disagreements.any? %>
-        <span class="ck-chip"><%= pluralize(@disagreements.size, "row") %></span>
-      <% end %>
+      <p class="ck-kicker">Cases to learn from</p>
+      <span class="ck-chip"><%= pluralize(@disagreements.size, "row") %></span>
     </div>
-    <% if @disagreements.empty? %>
-      <p class="ck-meta-copy">Nothing here yet. As people give a "disagree" verdict on response rows, those rows show up below so you can review the judge's misses and turn them into teaching examples.</p>
-    <% else %>
-      <p class="ck-meta-copy">Rows where a reviewer said the judge got it wrong. Save the best ones as teaching examples — the judge will see them next time it grades.</p>
-      <table class="ck-results-table ck-disagreements-table">
-        <thead>
-          <tr>
-            <th scope="col">Run · row</th>
-            <th scope="col">Judge</th>
-            <th scope="col">Human</th>
-            <th scope="col">Note</th>
-            <th scope="col"></th>
-          </tr>
-        </thead>
-        <tbody>
-          <% existing_ids = Array(@metric.few_shot_examples).map { |fs| fs["calibration_id"] } %>
-          <% @disagreements.each do |cal| %>
-            <% review = cal.response.reviews.find { |r| r.metric_id == @metric.id } %>
-            <% already = existing_ids.include?(cal.id) %>
-            <tr>
-              <td>
-                <%= link_to ck_run_path(cal.response.run), class: "ck-record-name" do %>
-                  <strong><%= cal.response.run.name.to_s.truncate(40) %></strong>
-                <% end %>
-                <span class="ck-meta-copy">· <%= link_to "row ##{cal.response.id}", run_response_path(cal.response.run, cal.response), class: "ck-link" %></span>
-              </td>
-              <td>
-                <% if review&.ai_score %>
-                  <span class="<%= ck_badge_classes(ck_score_kind(review.ai_score.to_f)) %>"><%= review.ai_score %></span>
-                <% else %>
-                  <span class="ck-meta-copy">—</span>
-                <% end %>
-              </td>
-              <td>
-                <% if cal.corrected_score %>
-                  <span class="<%= ck_badge_classes(ck_score_kind(cal.corrected_score.to_f)) %>"><%= cal.corrected_score %></span>
-                <% else %>
-                  <span class="ck-meta-copy">—</span>
-                <% end %>
-              </td>
-              <td class="ck-meta-copy"><%= cal.note.to_s.truncate(120) %></td>
-              <td>
-                <% if already %>
-                  <span class="ck-chip ck-chip--done">Saved as example</span>
-                <% else %>
-                  <%= button_to "Teach the judge",
-                        add_few_shot_metric_path(@metric, calibration_id: cal.id),
-                        method: :post,
-                        form_class: "inline-block",
-                        class: ck_button_classes(:light, variant: :outline),
-                        title: "Save this row as a teaching example. The judge will see it next time it grades." %>
-                <% end %>
-              </td>
-            </tr>
+    <p class="ck-meta-copy">Rows where a reviewer's score didn't match the judge's. Pin useful ones with <strong>Remember this</strong> — the judge sees them next time it grades.</p>
+    <% existing_ids = Array(@metric.few_shot_examples).map { |fs| fs["calibration_id"] } %>
+    <ul class="ck-disagreement-list">
+      <% @disagreements.each do |cal| %>
+        <% review = cal.response.reviews.find { |r| r.metric_id == @metric.id } %>
+        <% already = existing_ids.include?(cal.id) %>
+        <li class="ck-disagreement">
+          <div class="ck-disagreement__head">
+            <div class="ck-disagreement__scores">
+              <span class="ck-disagreement__scores-label">Judge</span>
+              <% if review&.ai_score %>
+                <span class="<%= ck_badge_classes(ck_score_kind(review.ai_score.to_f)) %>"><%= review.ai_score %></span>
+              <% else %>
+                <span class="ck-meta-copy">—</span>
+              <% end %>
+              <span class="ck-disagreement__scores-arrow">→</span>
+              <span class="ck-disagreement__scores-label">Human</span>
+              <% if cal.corrected_score %>
+                <span class="<%= ck_badge_classes(ck_score_kind(cal.corrected_score.to_f)) %>"><%= cal.corrected_score %></span>
+              <% else %>
+                <span class="ck-meta-copy">—</span>
+              <% end %>
+            </div>
+            <div class="ck-disagreement__action">
+              <% if already %>
+                <span class="ck-chip ck-chip--done">Remembered</span>
+              <% else %>
+                <%= button_to "Remember this",
+                      add_few_shot_metric_path(@metric, calibration_id: cal.id),
+                      method: :post,
+                      form_class: "inline-block",
+                      class: ck_button_classes(:light, variant: :outline),
+                      title: "Pin this row so the judge sees it next time it grades for this metric." %>
+              <% end %>
+            </div>
+          </div>
+          <% if cal.note.to_s.present? %>
+            <p class="ck-disagreement__note"><%= cal.note %></p>
           <% end %>
-        </tbody>
-      </table>
-    <% end %>
+          <p class="ck-disagreement__source ck-meta-copy">
+            <%= link_to cal.response.run.name.to_s.truncate(50), ck_run_path(cal.response.run), class: "ck-link" %>
+            ·
+            <%= link_to "row ##{cal.response.id}", run_response_path(cal.response.run, cal.response), class: "ck-link" %>
+          </p>
+        </li>
+      <% end %>
+    </ul>
   </section>
 
 
   <% if Array(@metric.few_shot_examples).any? %>
     <section class="ck-card ck-card--spaced">
       <div class="ck-prompt-preview__header">
-        <p class="ck-kicker">Teaching examples</p>
-        <span class="ck-chip"><%= pluralize(Array(@metric.few_shot_examples).size, "example") %></span>
+        <p class="ck-kicker">What the judge remembers</p>
+        <span class="ck-chip"><%= pluralize(Array(@metric.few_shot_examples).size, "case") %></span>
       </div>
-      <p class="ck-meta-copy">The judge sees these worked examples whenever it grades for this metric. Each shows what the judge gave and what a human said it should have been.</p>
+      <p class="ck-meta-copy">Rows you've pinned so the judge sees them next time it grades. Each one shows what the judge gave and what a human said it should have been.</p>
       <ol class="ck-few-shot-list">
         <% Array(@metric.few_shot_examples).each do |fs| %>
           <li class="ck-few-shot-item">

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.40"
+  VERSION = "0.5.41"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.40
+  version: 0.5.41
 platform: ruby
 authors:
 - Damien Bastin