completion-kit 0.5.42 → 0.5.44

data/app/views/completion_kit/calibrations/_buttons.html.erb

@@ -3,14 +3,19 @@
   <% pending_verdict = local_assigns[:pending_verdict] %>
   <% active_verdict = pending_verdict || current_verdict %>
   <% error = local_assigns[:error] %>
-  <% verdict_count = CompletionKit::Calibration.where(response_id: response_row.id, metric_id: metric.id).count %>
+  <% me = CompletionKit.config.username.presence || "operator" %>
+  <% other_calibrations = CompletionKit::Calibration
+        .where(response_id: response_row.id, metric_id: metric.id)
+        .where.not(created_by: me)
+        .order(created_at: :asc).to_a %>
+  <% verdict_icons = { "agree" => "hand-thumb-up", "disagree" => "hand-thumb-down", "borderline" => "scale" } %>
   <p class="ck-calibration__prompt">
     <span class="ck-calibration__label">Your verdict</span>
-    <% if verdict_count > 0 %>
-      <span class="ck-calibration__count"><%= pluralize(verdict_count, "verdict") %> on this score · <%= link_to "trust level →", metric_path(metric), class: "ck-link" %></span>
-    <% else %>
-      <span class="ck-calibration__hint">Tell us what you think — was the score right? Verdicts roll up into the metric's <%= link_to "trust level", metric_path(metric), class: "ck-link" %>.</span>
+    <% if other_calibrations.any? %>
+      <span class="ck-calibration__meta"><%= pluralize(other_calibrations.size, "other verdict") %> on this score</span>
+      <span class="ck-calibration__sep">·</span>
     <% end %>
+    <%= link_to metric_path(metric), class: "ck-calibration__meta-link" do %><%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true" %>Calibration →<% end %>
   </p>
   <div class="ck-calibration__buttons">
     <% verdict_icons = { "agree" => "hand-thumb-up", "disagree" => "hand-thumb-down", "borderline" => "scale" } %>
@@ -36,6 +41,38 @@
     <p class="ck-calibration__error" role="alert"><%= error %></p>
   <% end %>
 
+  <% if other_calibrations.any? %>
+    <details class="ck-calibration__others">
+      <summary class="ck-calibration__others-summary">
+        <%= heroicon_tag "chevron-right", variant: :outline, size: 14, "aria-hidden": "true" %>
+        <span>What others said (<%= other_calibrations.size %>)</span>
+      </summary>
+      <ul class="ck-calibration__others-list">
+        <% other_calibrations.each do |other| %>
+          <li class="ck-calibration__others-item ck-calibration__others-item--<%= other.verdict %>">
+            <div class="ck-calibration__others-row">
+              <span class="ck-calibration__others-verdict">
+                <%= heroicon_tag verdict_icons[other.verdict], variant: :outline, size: 14, "aria-hidden": "true" %>
+                <%= other.verdict %>
+              </span>
+              <span class="ck-calibration__others-by"><%= other.created_by %></span>
+              <% if other.corrected_score %>
+                <span class="ck-calibration__others-stars" aria-label="<%= pluralize(other.corrected_score.to_i, 'star') %>" title="<%= pluralize(other.corrected_score.to_i, 'star') %>">
+                  <% 5.times do |i| %>
+                    <svg viewBox="0 0 24 24" width="12" height="12" stroke-width="1.75" class="ck-star <%= i < other.corrected_score.to_i ? "ck-star--filled" : "ck-star--empty" %>" aria-hidden="true"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
+                  <% end %>
+                </span>
+              <% end %>
+            </div>
+            <% if other.note.to_s.present? %>
+              <p class="ck-calibration__others-note">"<%= other.note.to_s.truncate(140) %>"</p>
+            <% end %>
+          </li>
+        <% end %>
+      </ul>
+    </details>
+  <% end %>
+
   <% if active_verdict == "disagree" %>
     <% existing_score = (calibration&.corrected_score || review&.ai_score)&.round %>
     <%= form_with url: run_response_calibrations_path(run, response_row),
@@ -51,7 +88,7 @@
             <% radio_id = "ck-star-#{response_row.id}-#{metric.id}-#{n}" %>
             <input type="radio" name="corrected_score" id="<%= radio_id %>" value="<%= n %>" <%= "checked" if existing_score == n %> required>
             <label for="<%= radio_id %>" title="<%= pluralize(n, 'star') %>" aria-label="<%= pluralize(n, 'star') %>">
-              <svg viewBox="0 0 24 24" width="28" height="28" stroke-width="1.5" aria-hidden="true"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
+              <svg viewBox="0 0 24 24" stroke-width="1.5" aria-hidden="true"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
             </label>
           <% end %>
         </div>

data/app/views/completion_kit/calibrations/_trust_panel.html.erb

@@ -1,31 +1,35 @@
 <% stats = local_assigns[:stats] %>
-<div class="ck-trust-panel ck-trust-panel--<%= stats.gate %>">
-  <p class="ck-trust-panel__label" title="How often this metric's scores match the humans who reviewed them.">Trust level</p>
-  <% if stats.counter_only? %>
-    <div class="ck-trust-panel__body">
-      <span class="ck-trust-panel__counter"><%= stats.sample_size %><span class="ck-trust-panel__counter-of">/ <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %></span></span>
-      <span class="ck-trust-panel__hint">verdicts so far<% if stats.short_to_target > 0 %> · <%= stats.short_to_target %> more before we can score the judge<% end %></span>
-    </div>
+<% metric = local_assigns[:metric] %>
+<% anchor = metric&.name&.parameterize %>
+<% target_response = if (stats.sample_size.zero? || stats.counter_only?) && metric
+     created_by = CompletionKit.config.username.presence || "operator"
+     current_metric_version = CompletionKit::MetricVersion.current.find_by(metric_id: metric.id)
+     verdicted_ids = if current_metric_version
+       CompletionKit::Calibration.where(metric_id: metric.id, created_by: created_by, metric_version_id: current_metric_version.id).pluck(:response_id)
+     else
+       []
+     end
+     CompletionKit::Response.joins(:reviews)
+       .where(reviews: { metric_id: metric.id })
+       .where.not(reviews: { ai_score: nil })
+       .where.not(id: verdicted_ids)
+       .order(created_at: :desc).first
+   end %>
+
+<p class="ck-trust-line ck-trust-line--<%= stats.gate %>">
+  <span class="ck-trust-line__label"><%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true" %>Calibration</span>
+  <% if stats.sample_size.zero? %>
+    <span class="ck-trust-line__state">Not measured yet.</span>
+    <span class="ck-trust-line__hint">Needs <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts on the judge's scores.<% if target_response %>
+      <%= link_to "Give a verdict →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-trust-line__link" %>
+    <% else %> Run this metric on a dataset, then give a verdict.<% end %></span>
+  <% elsif stats.counter_only? %>
+    <span class="ck-trust-line__counter"><%= stats.sample_size %>/<%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %></span>
+    <span class="ck-trust-line__hint"><%= pluralize(stats.sample_size, "verdict") %><% if stats.short_to_target > 0 %> · <%= stats.short_to_target %> more before this can be measured<% end %><% if target_response %> · <%= link_to "Give another verdict →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-trust-line__link" %><% end %></span>
   <% else %>
-    <div class="ck-trust-panel__body">
-      <span class="ck-trust-panel__score"
-            title="Roughly how often the judge and the humans agreed.">~<%= (stats.agreement_point * 100).round %><span class="ck-trust-panel__score-pct">%</span></span>
-      <span class="ck-trust-panel__margin"
-            title="The range we're confident the true rate sits in, given how few verdicts we have so far.">±<%= (stats.margin * 100).round %> pt</span>
-      <span class="ck-trust-panel__gate"
-            title="<%= stats.firm? ? 'Enough verdicts for a reliable read.' : 'Early read. Keep giving verdicts to tighten the margin.' %>"><%= stats.firm? ? "settled" : "early read" %></span>
-    </div>
-    <div class="ck-trust-panel__details">
-      <span><%= pluralize(stats.sample_size, "verdict") %></span>
-      <% if stats.borderline_rate && stats.borderline_rate > 0 %>
-        <% level = if stats.borderline_rate > 0.30 then "danger"
-                   elsif stats.borderline_rate > 0.15 then "warning"
-                   else "ok" end %>
-        <span class="ck-trust-panel__borderline ck-trust-panel__borderline--<%= level %>"
-              title="<%= level == 'ok' ? 'Some reviewers said the rubric was unclear here.' : 'A lot of reviewers say the rubric is unclear here. Consider splitting the metric or rewriting the rubric.' %>">
-          <%= (stats.borderline_rate * 100).round %>% said "unclear"
-        </span>
-      <% end %>
-    </div>
+    <span class="ck-trust-line__score" title="Roughly how often the judge and the humans agreed.">~<%= (stats.agreement_point * 100).round %>%</span>
+    <span class="ck-trust-line__margin" title="The range we're confident the true rate sits in.">±<%= (stats.margin * 100).round %> pt</span>
+    <span class="ck-trust-line__gate" title="<%= stats.firm? ? 'Enough verdicts for a reliable read.' : 'Early read. Keep giving verdicts.' %>"><%= stats.firm? ? "settled" : "early" %></span>
+    <span class="ck-trust-line__hint"><%= pluralize(stats.sample_size, "verdict") %><% if stats.borderline_rate && stats.borderline_rate > 0 %><% level = stats.borderline_rate > 0.30 ? "danger" : stats.borderline_rate > 0.15 ? "warning" : "ok" %> · <span class="ck-trust-line__borderline ck-trust-line__borderline--<%= level %>" title="<%= level == 'ok' ? '' : 'Reviewers said the rubric was unclear here.' %>"><%= (stats.borderline_rate * 100).round %>% unclear</span><% end %></span>
   <% end %>
-</div>
+</p>

data/app/views/completion_kit/metrics/_form.html.erb

@@ -1,3 +1,9 @@
+<% suggestion = local_assigns[:suggestion_draft] %>
+<% edit_draft = local_assigns[:edit_draft] %>
+<% suggestion_bands = suggestion ? Array(suggestion.rubric_bands).each_with_object({}) { |b, h| h[b["stars"].to_i] = b["description"].to_s } : {} %>
+<% suggested_instruction = suggestion&.instruction.to_s %>
+<% instruction_changed = suggestion && suggested_instruction.present? && suggested_instruction != metric.instruction.to_s %>
+
 <%= form_with(model: metric, local: true) do |form| %>
   <% if metric.errors.any? %>
     <div class="ck-flash ck-flash--alert" role="alert">
@@ -10,6 +16,50 @@
     </div>
   <% end %>
 
+  <% if edit_draft %>
+    <% pub = local_assigns[:published_metric_version] %>
+    <% draft_instr_changed = pub && pub.instruction.to_s != edit_draft.instruction.to_s %>
+    <% draft_rubric_changed = pub && pub.rubric_bands != edit_draft.rubric_bands %>
+    <div class="ck-suggestion-banner" role="status">
+      <div class="ck-suggestion-banner__body">
+        <p class="ck-kicker">Draft pending</p>
+        <p class="ck-meta-copy">The form below shows your unpublished draft. Publish to replace the live<%= " instruction" if draft_instr_changed %><%= " and" if draft_instr_changed && draft_rubric_changed %><%= " rubric" if draft_rubric_changed %> for future runs, or keep editing.</p>
+      </div>
+      <div class="ck-suggestion-banner__actions">
+        <%= button_to "Discard draft", dismiss_suggestion_metric_path(metric, draft_id: edit_draft.id, back_to: "edit"),
+              method: :delete, form_class: "inline-block",
+              class: ck_button_classes(:light, variant: :outline),
+              data: { turbo_confirm: "Drop this draft?" } %>
+        <%= button_to "Publish this version", publish_draft_metric_path(metric, draft_id: edit_draft.id),
+              method: :post, form_class: "inline-block",
+              class: ck_button_classes(:dark) %>
+      </div>
+    </div>
+  <% end %>
+
+  <% if suggestion %>
+    <div class="ck-suggestion-banner" role="status">
+      <div class="ck-suggestion-banner__body">
+        <p class="ck-kicker">Proposed improvements</p>
+        <p class="ck-meta-copy">Based on your disagreements, the model proposed these changes to the instruction and rubric. Apply pieces inline below, take everything at once, try again, or discard.</p>
+      </div>
+      <div class="ck-suggestion-banner__actions">
+        <%= button_to "Try again", suggest_variants_metric_path(metric, back_to: "edit"),
+              method: :post, form_class: "inline-block",
+              class: ck_button_classes(:light, variant: :outline),
+              title: "Discard these improvements and ask the model for fresh ones.",
+              data: { turbo_confirm: "Replace these improvements with fresh ones from the model?" } %>
+        <%= button_to "Discard", dismiss_suggestion_metric_path(metric, draft_id: suggestion.id, back_to: "edit"),
+              method: :delete, form_class: "inline-block",
+              class: ck_button_classes(:light, variant: :outline),
+              data: { turbo_confirm: "Drop these improvements?" } %>
+        <%= button_to "Take everything", publish_draft_metric_path(metric, draft_id: suggestion.id),
+              method: :post, form_class: "inline-block",
+              class: ck_button_classes(:dark) %>
+      </div>
+    </div>
+  <% end %>
+
   <div class="ck-card ck-form-card">
     <div class="ck-field">
       <%= form.label :name, "Metric name", class: "ck-label" %>
@@ -22,14 +72,34 @@
       <p class="ck-hint">What should the judge assess? This instruction is sent to the LLM judge when scoring outputs.</p>
       <%= form.text_area :instruction, rows: 8, class: "ck-input ck-input--area", placeholder: "Evaluate whether the output...", **ck_field_aria(form, :instruction) %>
       <%= ck_field_error(form, :instruction) %>
+
+      <% if instruction_changed %>
+        <div class="ck-inline-suggestion">
+          <div class="ck-inline-suggestion__head">
+            <p class="ck-kicker">Suggested wording</p>
+            <button type="button"
+                    class="<%= ck_button_classes(:light, variant: :outline) %> ck-inline-suggestion__apply"
+                    data-ck-apply
+                    data-target="metric[instruction]"
+                    data-value="<%= h(suggested_instruction) %>">Use this wording</button>
+          </div>
+          <div class="ck-inline-suggestion__diff">
+            <pre class="ck-inline-suggestion__pane ck-inline-suggestion__pane--before"><%= ck_word_diff_old(metric.instruction.to_s, suggested_instruction) %></pre>
+            <pre class="ck-inline-suggestion__pane ck-inline-suggestion__pane--after"><%= ck_word_diff_new(metric.instruction.to_s, suggested_instruction) %></pre>
+          </div>
+        </div>
+      <% end %>
     </div>
 
     <div class="ck-field ck-field--spacious">
-      <p class="ck-section-title">Rubric</p>
+      <p class="ck-section-title">Rubric<%= render "completion_kit/metrics/rubric_hint" %></p>
       <p class="ck-hint">What each star rating means for this metric.</p>
 
       <div class="ck-rubric-builder">
         <% metric.rubric_bands_for_form.each_with_index do |band, index| %>
+          <% suggested_band = suggestion_bands[band["stars"].to_i].to_s %>
+          <% band_changed = suggestion && suggested_band.present? && suggested_band != band["description"].to_s %>
+          <% target_name = "metric[rubric_bands][#{index}][description]" %>
           <div class="ck-rubric-row">
             <div class="ck-rubric-row__stars">
               <% 5.times do |i| %>
@@ -38,7 +108,23 @@
               <input type="hidden" name="metric[rubric_bands][<%= index %>][stars]" value="<%= band["stars"] %>">
             </div>
             <div class="ck-rubric-row__fields">
-              <textarea name="metric[rubric_bands][<%= index %>][description]" rows="2" class="ck-input ck-input--area"><%= band["description"] %></textarea>
+              <textarea name="<%= target_name %>" rows="2" class="ck-input ck-input--area"><%= band["description"] %></textarea>
+              <% if band_changed %>
+                <div class="ck-inline-suggestion ck-inline-suggestion--band">
+                  <div class="ck-inline-suggestion__head">
+                    <p class="ck-kicker">Suggested band</p>
+                    <button type="button"
+                            class="<%= ck_button_classes(:light, variant: :outline) %> ck-inline-suggestion__apply"
+                            data-ck-apply
+                            data-target="<%= target_name %>"
+                            data-value="<%= h(suggested_band) %>">Use this band</button>
+                  </div>
+                  <div class="ck-inline-suggestion__diff">
+                    <pre class="ck-inline-suggestion__pane ck-inline-suggestion__pane--before"><%= ck_word_diff_old(band["description"].to_s, suggested_band) %></pre>
+                    <pre class="ck-inline-suggestion__pane ck-inline-suggestion__pane--after"><%= ck_word_diff_new(band["description"].to_s, suggested_band) %></pre>
+                  </div>
+                </div>
+              <% end %>
             </div>
           </div>
         <% end %>
@@ -57,11 +143,11 @@
         <% confirm = parts.empty? ? "Delete \"#{metric.name}\"? It's not in use." : "Delete \"#{metric.name}\"? It's #{parts.to_sentence}." %>
         <%= button_to metric_path(metric), method: :delete,
               form_class: "inline-block",
-              class: "ck-icon-btn",
+              class: "ck-icon-btn ck-icon-btn--form",
               title: "Delete metric",
               "aria-label": "Delete metric",
               data: { turbo_confirm: confirm } do %>
-          <%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %>
+          <%= heroicon_tag "trash", variant: :outline, size: 24, "aria-hidden": "true" %>
         <% end %>
       <% end %>
       <%= link_to "Cancel", metrics_path, class: ck_button_classes(:light, variant: :outline), tabindex: "0" %>

data/app/views/completion_kit/metrics/_rubric_diff.html.erb

@@ -0,0 +1,25 @@
+<% current_bands = local_assigns[:current_bands] || [] %>
+<% draft_bands = local_assigns[:draft_bands] || [] %>
+<% lookup = ->(bands, stars) { bands.find { |b| b["stars"].to_i == stars }&.dig("description").to_s } %>
+<div class="ck-rubric-diff">
+  <% 5.downto(1) do |stars| %>
+    <% old_band = lookup.call(current_bands, stars) %>
+    <% new_band = lookup.call(draft_bands, stars) %>
+    <% changed = old_band != new_band %>
+    <div class="ck-rubric-diff__row ck-rubric-diff__row--<%= changed ? "changed" : "unchanged" %>">
+      <div class="ck-rubric-diff__stars">
+        <% 5.times do |i| %>
+          <svg viewBox="0 0 24 24" width="14" height="14" stroke-width="1.75" class="ck-star <%= i < stars ? "ck-star--filled" : "ck-star--empty" %>"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
+        <% end %>
+      </div>
+      <% if changed %>
+        <div class="ck-rubric-diff__panes">
+          <pre class="ck-rubric-diff__pane ck-rubric-diff__pane--before"><%= ck_word_diff_old(old_band, new_band) %></pre>
+          <pre class="ck-rubric-diff__pane ck-rubric-diff__pane--after"><%= ck_word_diff_new(old_band, new_band) %></pre>
+        </div>
+      <% else %>
+        <p class="ck-rubric-diff__unchanged"><%= old_band.presence || "—" %></p>
+      <% end %>
+    </div>
+  <% end %>
+</div>

data/app/views/completion_kit/metrics/_rubric_hint.html.erb

@@ -0,0 +1,4 @@
+<span class="ck-info-hint" tabindex="0" role="button" aria-label="What is a rubric?">
+  <%= heroicon_tag "information-circle", variant: :outline, "aria-hidden": "true" %>
+  <span class="ck-info-popup">How the judge picks 1 to 5. Each row says what an output has to look like to earn that many stars. The judge reads these descriptions when it scores, so clearer rows give you more consistent scoring.</span>
+</span>

data/app/views/completion_kit/metrics/_starter_card.html.erb

@@ -1,11 +1,15 @@
-<article class="ck-starter-card">
-  <h3 class="ck-starter-card__name"><%= link_to starter.name, starter_preview_metrics_path(key: starter.key), class: "ck-link" %></h3>
-  <p class="ck-starter-card__desc"><%= starter.description %></p>
-  <div class="ck-starter-card__actions">
-    <%= link_to "Preview", starter_preview_metrics_path(key: starter.key), class: ck_button_classes(:dark) + " ck-button--sm" %>
-    <%= button_to "Don't show this one", dismiss_starter_metrics_path(key: starter.key),
-          method: :post, form_class: "inline-block",
-          class: "ck-link ck-starter-card__dismiss",
+<%= link_to starter_preview_metrics_path(key: starter.key), class: "ck-starter-card" do %>
+  <div class="ck-starter-card__body">
+    <p class="ck-starter-card__name"><strong><%= starter.name %></strong></p>
+    <p class="ck-starter-card__desc"><%= starter.description %></p>
+  </div>
+  <div class="ck-starter-card__foot">
+    <span class="ck-starter-card__cta">Preview →</span>
+    <%= button_to "dismiss", dismiss_starter_metrics_path(key: starter.key),
+          method: :post,
+          form: { onclick: "event.stopPropagation();" },
+          form_class: "inline-block ck-starter-card__dismiss-form",
+          class: "ck-starter-card__dismiss",
           data: { turbo_confirm: "Hide \"#{starter.name}\" from this list?" } %>
   </div>
-</article>
+<% end %>

data/app/views/completion_kit/metrics/edit.html.erb

@@ -10,4 +10,8 @@
   </div>
 </section>
 
-<%= render "form", metric: @metric %>
+<%= render "form",
+      metric: @metric,
+      suggestion_draft: @suggestion_draft,
+      edit_draft: @edit_draft,
+      published_metric_version: @published_metric_version %>

data/app/views/completion_kit/metrics/index.html.erb

@@ -30,15 +30,17 @@
             <%= link_to metric_path(metric), class: "ck-record-name" do %><strong><%= metric.name %></strong><% end %>
             <% if CompletionKit.config.judge_calibration_enabled %>
               <% s = CompletionKit::MetricCalibrationStats.for(metric) %>
-              <p class="ck-metrics-table__trust">
+              <p class="ck-metrics-table__trust" title="Calibration: how often this metric's scores match the humans who reviewed them.">
+                <%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true", class: "ck-trust-icon" %>
+                <span class="ck-metrics-table__trust-label">Calibration</span>
                 <% if s.counter_only? %>
                   <% if s.sample_size.zero? %>
-                    No verdicts yet
+                    <span class="ck-metrics-table__trust-state">Not measured yet</span>
                   <% else %>
                     <%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts
                   <% end %>
                 <% else %>
-                  <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read — keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
+                  <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read. Keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
                   ±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %>
                 <% end %>
               </p>