completion-kit 0.8.0 → 0.10.0

data/app/views/completion_kit/metrics/_guiding_examples.html.erb

@@ -0,0 +1,23 @@
+<div id="ck-guiding-<%= metric.id %>" class="ck-guiding">
+  <% if examples.any? %>
+    <div class="ck-guiding__head">
+      <p class="ck-kicker ck-kicker--inset">Guiding the judge</p>
+      <span class="ck-guiding__legend">Judge &rarr; Human</span>
+    </div>
+    <ul class="ck-guiding__list">
+      <% examples.each do |example| %>
+        <li class="ck-guiding__item">
+          <%= link_to run_response_path(example[:run_id], example[:response_id], anchor: metric.name.parameterize),
+                class: "ck-guiding__link", title: "Open this review" do %>
+            <span class="ck-guiding__output"><%= truncate(example[:output].to_s, length: 90) %></span>
+            <span class="ck-guiding__scores"><span class="ck-guiding__judge"><%= example[:judge_score].to_i %></span> &rarr; <span class="ck-guiding__human"><%= example[:human_score].to_i %></span></span>
+          <% end %>
+          <%= button_to exclude_example_metric_path(metric, calibration_id: example[:id]),
+                method: :post, form_class: "inline-block", class: "ck-icon-btn",
+                title: "Stop using this case", "aria-label": "Stop using this case",
+                data: { turbo_confirm: "Stop using this corrected case to guide the judge?" } do %><%= heroicon_tag "x-mark", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
+        </li>
+      <% end %>
+    </ul>
+  <% end %>
+</div>

data/app/views/completion_kit/metrics/edit.html.erb

@@ -10,6 +10,24 @@
   </div>
 </section>
 
+<% if CompletionKit.config.judge_calibration_enabled && @suggestion_draft.nil? && @edit_draft.nil? && @improve_disagreement_count.to_i.positive? %>
+  <div class="ck-suggestion-banner" role="status">
+    <div class="ck-suggestion-banner__body">
+      <p class="ck-kicker">Improve from reviews</p>
+      <p class="ck-meta-copy">Based on human reviews, the model can propose changes to this metric.</p>
+    </div>
+    <div class="ck-suggestion-banner__actions">
+      <%= button_to suggest_variants_metric_path(@metric, back_to: "edit"),
+            method: :post, form_class: "inline-block",
+            class: ck_button_classes(:light, variant: :outline) + " ck-button--sm",
+            data: { turbo_confirm: "Draft improvements to this metric from your human reviews? You can edit or apply them here before publishing." } do %>
+        <%= heroicon_tag "sparkles", variant: :outline, class: "ck-magic-icon", "aria-hidden": "true" %>
+        Suggest improvements
+      <% end %>
+    </div>
+  </div>
+<% end %>
+
 <%= render "form",
       metric: @metric,
       suggestion_draft: @suggestion_draft,

data/app/views/completion_kit/metrics/index.html.erb

@@ -18,6 +18,7 @@
     <thead>
       <tr>
         <th scope="col">Name</th>
+        <th scope="col">Version</th>
         <th scope="col">Instruction</th>
         <th scope="col">In groups</th>
         <th scope="col"></th>
@@ -28,29 +29,16 @@
         <tr onclick="window.location='<%= metric_path(metric) %>'" style="cursor: pointer;">
           <td>
             <%= link_to metric_path(metric), class: "ck-record-name" do %><strong><%= metric.name %></strong><% end %>
-            <% if CompletionKit.config.judge_calibration_enabled %>
-              <% s = CompletionKit::MetricCalibrationStats.for(metric) %>
-              <p class="ck-metrics-table__trust" title="Calibration: how often this metric's scores match the humans who reviewed them.">
-                <%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true", class: "ck-trust-icon" %>
-                <span class="ck-metrics-table__trust-label">Calibration</span>
-                <% if s.counter_only? %>
-                  <% if s.sample_size.zero? %>
-                    <span class="ck-metrics-table__trust-state">Not measured yet</span>
-                  <% else %>
-                    <%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts
-                  <% end %>
-                <% else %>
-                  <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read. Keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
-                  ±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %>
-                <% end %>
-              </p>
-            <% end %>
             <% if metric.tags.any? %>
               <div class="tag-marks-row">
                 <%= render "completion_kit/tags/marks", tags: metric.tags %>
               </div>
             <% end %>
           </td>
+          <td data-label="Version">
+            <% v = @current_versions[metric.id] %>
+            <span class="ck-chip ck-chip--soft"><%= v ? v.version_label : "v1" %></span>
+          </td>
           <td data-label="Instruction" class="ck-meta-copy"><div class="ck-clamp-2"><%= metric.instruction.presence || "—" %></div></td>
           <td data-label="In groups">
             <% groups = metric.metric_groups %>

data/app/views/completion_kit/metrics/show.html.erb

@@ -6,11 +6,6 @@
 <section class="ck-page-header">
   <div>
     <h1 class="ck-title"><%= @metric.name %></h1>
-    <% if CompletionKit.config.judge_calibration_enabled %>
-      <%= render "completion_kit/calibrations/trust_panel",
-            stats: CompletionKit::MetricCalibrationStats.for(@metric),
-            metric: @metric %>
-    <% end %>
     <% if @metric.tags.any? %>
       <div class="tag-marks-row tag-marks-row--header">
         <%= render "completion_kit/tags/marks", tags: @metric.tags %>
@@ -18,23 +13,6 @@
     <% end %>
   </div>
   <div class="ck-actions">
-    <% if CompletionKit.config.judge_calibration_enabled %>
-      <% if @suggestion_draft || @edit_draft %>
-        <% review_title = @suggestion_draft ? "The model proposed improvements based on your disagreements. Review and apply what you want." : "An unpublished draft of this metric is saved. Review and publish, or keep editing." %>
-        <%= link_to "Review changes →", edit_metric_path(@metric),
-              class: ck_button_classes(:dark),
-              title: review_title %>
-      <% elsif @improve_disagreement_count.positive? %>
-        <%= button_to "Improve the metric", suggest_variants_metric_path(@metric),
-              method: :post, form_class: "inline-block",
-              class: ck_button_classes(:light, variant: :outline),
-              title: "Ask the model to suggest improvements to this metric's instruction and rubric based on the disagreements collected so far.",
-              data: { turbo_confirm: "Ask the model for suggested improvements based on the disagreements collected so far?" } %>
-      <% else %>
-        <button type="button" class="<%= ck_button_classes(:light, variant: :outline) %>" disabled
-                title="Mark at least one case as Disagree before the model can suggest a change.">Improve the metric</button>
-      <% end %>
-    <% end %>
     <%= link_to "Edit", edit_metric_path(@metric), class: ck_button_classes(:light, variant: :outline) %>
   </div>
 </section>
@@ -64,9 +42,8 @@
   </div>
 </section>
 
-<% if CompletionKit.config.judge_calibration_enabled && @versions.size > 1 %>
+<% if CompletionKit.config.judge_calibration_enabled && @versions.any? %>
   <% predecessor_of = @versions.index_with { |v| @versions.detect { |o| o.version_number < v.version_number } } %>
-  <% version_changed = ->(v, pred) { pred && (pred.instruction.to_s != v.instruction.to_s || pred.rubric_bands != v.rubric_bands) } %>
   <section class="ck-card ck-card--spaced">
     <p class="ck-kicker">Versions</p>
     <table class="ck-results-table ck-metric-versions-table">
@@ -82,6 +59,7 @@
           <% pred = predecessor_of[v] %>
           <tr>
             <td>
+              <% summary = v.change_summary_against(pred) %>
               <div class="ck-version-cell">
                 <div class="ck-version-cell__label">
                   <strong><%= v.version_label %></strong>
@@ -95,19 +73,17 @@
                     <%= button_to "Make current", publish_draft_metric_path(@metric, draft_id: v.id),
                           method: :post, form_class: "inline-block",
                           class: "ck-chip ck-chip--publish",
-                          data: { turbo_confirm: "Roll the live judge back to #{v.version_label}? Calibration verdicts collected against the current version stay tied to it." } %>
+                          data: { turbo_confirm: "Make #{v.version_label} the version to use? #{v.version_label} will be used in test runs using this metric now. Reviews you have already given stay with the version they were made against." } %>
                   <% end %>
                 </div>
-                <% if version_changed.call(v, pred) %>
-                  <button type="button" class="ck-cell-link ck-cell-link--delta"
-                          title="What changed from #{pred.version_label}"
-                          onclick="document.getElementById('ck-mvdiff-<%= v.id %>').showModal()">&Delta;</button>
+                <% if summary %>
+                  <button type="button" class="ck-cell-link ck-cell-link--delta" title="What changed from <%= pred.version_label %>" onclick="document.getElementById('ck-mvdiff-<%= v.id %>').showModal()">&Delta;</button>
                 <% end %>
               </div>
             </td>
             <td>
               <% source_label, source_class = case v.source
-                                              when "suggestion" then ["AI suggestion", "ck-source-chip ck-source-chip--ai"]
+                                              when "suggestion" then ["AI calibration", "ck-source-chip ck-source-chip--ai"]
                                               when "edit" then ["Manual edit", "ck-source-chip ck-source-chip--manual"]
                                               when "revert" then ["Reverted", "ck-source-chip ck-source-chip--revert"]
                                               else ["Original", "ck-source-chip ck-source-chip--initial"]
@@ -115,7 +91,15 @@
               <span class="<%= source_class %>"><%= source_label %></span>
             </td>
             <td class="ck-meta-copy">
-              <time datetime="<%= v.created_at.utc.iso8601 %>" data-relative-time><%= time_ago_in_words(v.created_at) %> ago</time>
+              <div class="ck-version-created">
+                <time datetime="<%= v.created_at.utc.iso8601 %>" data-relative-time><%= time_ago_in_words(v.created_at) %> ago</time>
+                <% if v.draft? %>
+                  <%= button_to dismiss_suggestion_metric_path(@metric, draft_id: v.id),
+                        method: :delete, form_class: "inline-block", class: "ck-icon-btn",
+                        title: "Discard draft #{v.version_label}", "aria-label": "Discard draft #{v.version_label}",
+                        data: { turbo_confirm: "Discard draft #{v.version_label}? This can't be undone." } do %><%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
+                <% end %>
+              </div>
             </td>
           </tr>
         <% end %>
@@ -125,7 +109,7 @@
 
   <% @versions.each do |v| %>
     <% pred = predecessor_of[v] %>
-    <% next unless version_changed.call(v, pred) %>
+    <% next unless v.change_summary_against(pred) %>
     <dialog id="ck-mvdiff-<%= v.id %>" class="ck-modal" onclick="if(event.target===this)this.close()">
       <article class="ck-modal__panel" tabindex="-1" onclick="event.stopPropagation()">
         <header class="ck-modal__header">
@@ -148,85 +132,77 @@
               </div>
             </div>
           <% end %>
-          <% if pred.rubric_bands != v.rubric_bands %>
+          <% pred_bands = CompletionKit::Metric.normalize_rubric_bands(pred.rubric_bands) %>
+          <% v_bands = CompletionKit::Metric.normalize_rubric_bands(v.rubric_bands) %>
+          <% if pred_bands != v_bands %>
             <p class="ck-kicker ck-kicker--inset">Rubric changes</p>
             <%= render "completion_kit/metrics/rubric_diff",
-                  current_bands: pred.rubric_bands || [],
-                  draft_bands: v.rubric_bands || [] %>
+                  current_bands: pred_bands,
+                  draft_bands: v_bands %>
           <% end %>
         </div>
+        <footer class="ck-modal__footer ck-modal__footer--split">
+          <% if v.current? %>
+            <span class="ck-modal__foot-note">This is the metric's published version.</span>
+          <% elsif v.draft? %>
+            <span class="ck-modal__foot-note">Happy with it? Publish to use <%= v.version_label %> for this metric from now on. Tweak it with Edit.</span>
+            <span class="ck-modal__foot-actions">
+              <%= button_to dismiss_suggestion_metric_path(@metric, draft_id: v.id),
+                    method: :delete, form_class: "inline-block", class: "ck-icon-btn",
+                    title: "Discard draft #{v.version_label}", "aria-label": "Discard draft #{v.version_label}",
+                    data: { turbo_confirm: "Discard draft #{v.version_label}? This can't be undone." } do %><%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
+              <%= link_to "Edit", edit_metric_path(@metric), class: ck_button_classes(:light, variant: :outline) %>
+              <%= button_to "Publish #{v.version_label} →", publish_draft_metric_path(@metric, draft_id: v.id),
+                    method: :post, form_class: "inline-block", class: ck_button_classes(:dark) %>
+            </span>
+          <% else %>
+            <span class="ck-modal__foot-note">Roll this metric back to this version.</span>
+            <%= button_to "Make #{v.version_label} current →", publish_draft_metric_path(@metric, draft_id: v.id),
+                  method: :post, form_class: "inline-block", class: ck_button_classes(:dark),
+                  data: { turbo_confirm: "Make #{v.version_label} the version to use? #{v.version_label} will be used in test runs using this metric now. Reviews you have already given stay with the version they were made against." } %>
+          <% end %>
+        </footer>
       </article>
     </dialog>
   <% end %>
 <% end %>
 
-<% if CompletionKit.config.judge_calibration_enabled && @disagreements.any? %>
+<% if CompletionKit.config.judge_calibration_enabled %>
+  <% draft = @suggestion_draft || @edit_draft %>
   <section class="ck-card ck-card--spaced">
     <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Cases to learn from</p>
-      <span class="ck-chip"><%= pluralize(@disagreements.size, "case") %></span>
-    </div>
-    <% mixed_versions = @disagreements.any? { |c| c.metric_version_id != @published_metric_version.id } %>
-    <p class="ck-meta-copy">Cases where a reviewer's score didn't match the judge's. Pin useful ones with <strong>Remember this</strong> so the judge sees them next time it grades<%= " (pins flow into the current judge regardless of which version produced the verdict)" if mixed_versions %>.</p>
-    <% existing_ids = Array(@metric.few_shot_examples).map { |fs| fs["calibration_id"] } %>
-    <ul class="ck-disagreement-list">
-      <% @disagreements.each do |cal| %>
-        <% review = cal.response.reviews.find { |r| r.metric_id == @metric.id } %>
-        <% already = existing_ids.include?(cal.id) %>
-        <% cal_metric_version = cal.metric_version %>
-        <% on_current = cal_metric_version&.id == @published_metric_version.id %>
-        <li class="ck-disagreement<%= " ck-disagreement--remembered" if already %><%= " ck-disagreement--stale" unless on_current %>">
-          <div class="ck-disagreement__head">
-            <div class="ck-disagreement__scores">
-              <% if cal_metric_version && mixed_versions %>
-                <span class="ck-source-chip <%= on_current ? "ck-source-chip--current" : "ck-source-chip--past" %>" title="<%= on_current ? "Verdict on the live judge version." : "Verdict on a superseded judge version." %>"><%= cal_metric_version.version_label %></span>
-              <% end %>
-              <span class="ck-disagreement__scores-label">Judge</span>
-              <% if review&.ai_score %>
-                <span class="<%= ck_badge_classes(ck_score_kind(review.ai_score.to_f)) %>"><%= review.ai_score %></span>
-              <% else %>
-                <span class="ck-meta-copy">—</span>
-              <% end %>
-              <span class="ck-disagreement__scores-arrow">→</span>
-              <span class="ck-disagreement__scores-label">Human</span>
-              <% if cal.corrected_score %>
-                <span class="<%= ck_badge_classes(ck_score_kind(cal.corrected_score.to_f)) %>"><%= cal.corrected_score %></span>
-              <% else %>
-                <span class="ck-meta-copy">—</span>
-              <% end %>
-            </div>
-            <div class="ck-disagreement__action">
-              <% if already %>
-                <%= button_to "Forget",
-                      remove_few_shot_metric_path(@metric, calibration_id: cal.id),
-                      method: :delete,
-                      form_class: "inline-block",
-                      class: ck_button_classes(:light, variant: :outline),
-                      title: "Stop showing this case to the judge.",
-                      data: { turbo_confirm: "Stop showing this case to the judge?" } %>
-                <span class="ck-chip ck-chip--done" title="The judge sees this row when it grades for this metric.">Remembered</span>
-              <% else %>
-                <%= button_to "Remember this",
-                      add_few_shot_metric_path(@metric, calibration_id: cal.id),
-                      method: :post,
-                      form_class: "inline-block",
-                      class: ck_button_classes(:light, variant: :outline),
-                      title: "Pin this case so the judge sees it next time it grades for this metric." %>
-              <% end %>
-            </div>
-          </div>
-          <% if cal.note.to_s.present? %>
-            <p class="ck-disagreement__note"><%= cal.note %></p>
-          <% end %>
-          <p class="ck-disagreement__source ck-meta-copy">
-            <%= link_to run_response_path(cal.response.run, cal.response, anchor: @metric.name.parameterize),
-                    class: "ck-disagreement__source-link" do %>
-              <% case_display = cal.response.row_index.nil? ? "##{cal.response.id}" : (cal.response.row_index + 1).to_s %>
-              View case <%= case_display %> in <%= cal.response.run.name.to_s.truncate(50) %> →
-            <% end %>
-          </p>
-        </li>
+      <p class="ck-kicker">Calibration</p>
+      <% if draft.nil? && @improve_disagreement_count.positive? %>
+        <%= button_to suggest_variants_metric_path(@metric),
+              method: :post, form_class: "inline-block",
+              class: ck_button_classes(:light, variant: :outline) + " ck-button--sm",
+              data: { turbo_confirm: "Draft improvements to this metric from your human reviews? It stays a draft until you compare it and publish." } do %>
+          <%= heroicon_tag "sparkles", variant: :outline, class: "ck-magic-icon", "aria-hidden": "true" %>
+          Suggest improvements
+        <% end %>
       <% end %>
-    </ul>
+    </div>
+    <p class="ck-meta-copy">This is a measure of how often the judge's scores match a human reviewer. Review its scores to build that signal; the scores you disagree with become the cases the model learns from when you improve the metric.</p>
+    <%= render "completion_kit/calibrations/trust_panel",
+          stats: CompletionKit::MetricCalibrationStats.for(@metric),
+          metric: @metric %>
+    <% if CompletionKit.config.judge_examples_from_reviews %>
+      <%= render "completion_kit/metrics/guiding_examples", metric: @metric, examples: @guiding_examples %>
+    <% end %>
+    <% if draft %>
+      <div class="ck-cal-foot">
+        <span class="ck-cal-foot__note">A draft improvement (<%= draft.version_label %>) is waiting in the Versions table above. Open its change to compare, then Publish to use it.</span>
+      </div>
+    <% end %>
   </section>
 <% end %>
+
+<% if params[:show_change].present? %>
+  <script>
+    (function () {
+      var dialog = document.getElementById("ck-mvdiff-<%= params[:show_change].to_i %>");
+      if (dialog && typeof dialog.showModal === "function") dialog.showModal();
+    })();
+  </script>
+<% end %>
+

data/app/views/completion_kit/responses/show.html.erb

@@ -100,12 +100,17 @@
       <% @reviews.each do |review| %>
         <% review_version = review.metric_version %>
         <% stale = review.stale_against_current_judge? %>
-        <div class="ck-review-card<%= " ck-review-card--stale" if stale %>" id="<%= review.metric&.name&.parameterize || "review-#{review.id}" %>">
+        <div class="ck-review-card" id="<%= review.metric&.name&.parameterize || "review-#{review.id}" %>">
           <div class="ck-review-card__header">
             <span class="ck-review-card__metric"><% if review.metric %><%= link_to review.metric_name, metric_path(review.metric), class: "ck-link" %><% else %><%= review.metric_name %><% end %></span>
             <div class="ck-inline">
               <% if review_version %>
-                <span class="ck-source-chip <%= stale ? "ck-source-chip--past" : "ck-source-chip--current" %>" title="<%= stale ? "Score produced by #{review_version.version_label} of this metric. The live judge has changed since." : "Score produced by the live judge (#{review_version.version_label})." %>"><%= review_version.version_label %></span>
+                <% if stale %>
+                  <% current_version = CompletionKit::MetricVersion.current.find_by(metric_id: review.metric_id) %>
+                  <span class="ck-source-chip ck-source-chip--past" title="Scored on <%= review_version.version_label %>; the metric is now on <%= current_version.version_label %>, which may score this differently."><%= review_version.version_label %> &rarr; <%= current_version.version_label %></span>
+                <% else %>
+                  <span class="ck-source-chip ck-source-chip--current" title="Scored on the metric's current version (<%= review_version.version_label %>)."><%= review_version.version_label %></span>
+                <% end %>
               <% end %>
               <% if review.ai_score %>
                 <% 5.times do |i| %>
@@ -116,9 +121,6 @@
               <% end %>
             </div>
           </div>
-          <% if stale %>
-            <p class="ck-review-card__stale-note">Scored against a superseded version of this metric. The live judge may score this differently.</p>
-          <% end %>
           <% if review.ai_feedback.present? %>
             <p class="ck-review-card__feedback"><%= review.ai_feedback %></p>
           <% end %>

data/app/views/completion_kit/runs/show.html.erb

@@ -23,25 +23,25 @@
   <% if stale_summary.any? %>
     <div class="ck-stale-versions-banner" role="status">
       <div class="ck-stale-versions-banner__body">
-        <p class="ck-kicker">Stale judge versions</p>
+        <p class="ck-kicker">Stale metric versions</p>
         <p class="ck-meta-copy">
           This run was scored against metric versions that are no longer live.
           <% stale_summary.values.each_with_index do |s, i| %>
             <%= ", " if i > 0 %><strong><%= s[:metric_name] %></strong> (scored by <%= s[:scored_labels].join(", ") %>; live is <%= s[:current_label] %>)<% end %>.
-          Re-run to refresh the scores with the current judge.
+          Re-run to refresh the scores with the current metrics.
         </p>
       </div>
       <% if @run.status == "completed" %>
         <%= button_to "Re-run from scratch",
               rerun_run_path(@run), method: :post,
               class: ck_button_classes(:light, variant: :outline), form_class: "inline-block",
-              title: "Create a new run that regenerates responses and grades them with the current judge.",
-              data: { turbo_confirm: "Create a new run with fresh responses and the current judge? The original run stays as a record." } %>
-        <%= button_to "Re-grade with current judge",
+              title: "Create a new run that regenerates responses and grades them with the current metrics.",
+              data: { turbo_confirm: "Create a new run with fresh responses and the current metrics? The original run stays as a record." } %>
+        <%= button_to "Re-grade with current metrics",
               regrade_run_path(@run), method: :post,
               class: ck_button_classes(:dark), form_class: "inline-block",
-              title: "Re-judge this run's existing responses against the current judge. Faster and cheaper than re-running.",
-              data: { turbo_confirm: "Re-judge this run's existing responses against the current judge?" } %>
+              title: "Re-grade this run's existing responses against the current metrics. Faster and cheaper than re-running.",
+              data: { turbo_confirm: "Re-grade this run's existing responses against the current metrics?" } %>
       <% end %>
     </div>
   <% end %>

data/config/routes.rb

@@ -19,11 +19,10 @@ CompletionKit::Engine.routes.draw do
       post "starters/:key/dismiss", to: "metrics#dismiss_starter", as: :dismiss_starter
     end
     member do
-      post   :add_few_shot
-      delete :remove_few_shot
       post :publish_draft
       post :suggest_variants
       delete :dismiss_suggestion
+      post :exclude_example
     end
   end
   resources :metric_groups
@@ -89,8 +88,6 @@ CompletionKit::Engine.routes.draw do
         end
         member do
           post :suggest_variants
-          post :add_few_shot
-          delete :remove_few_shot
         end
       end
       resources :metric_groups

data/db/migrate/20260529000001_remove_few_shot_examples_from_completion_kit_metrics.rb

@@ -0,0 +1,5 @@
+class RemoveFewShotExamplesFromCompletionKitMetrics < ActiveRecord::Migration[8.1]
+  def change
+    remove_column :completion_kit_metrics, :few_shot_examples, :text
+  end
+end

data/db/migrate/20260530000001_add_excluded_from_examples_to_completion_kit_calibrations.rb

@@ -0,0 +1,5 @@
+class AddExcludedFromExamplesToCompletionKitCalibrations < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_calibrations, :excluded_from_examples, :boolean, null: false, default: false
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.8.0"
+  VERSION = "0.10.0"
 end

data/lib/completion_kit.rb

@@ -13,6 +13,7 @@ module CompletionKit
     attr_accessor :api_rate_limit, :web_rate_limit
     attr_accessor :allow_loopback_endpoints
     attr_accessor :judge_calibration_enabled
+    attr_accessor :judge_examples_from_reviews
 
     def initialize
       @openai_api_key = ENV['OPENAI_API_KEY']
@@ -29,6 +30,7 @@ module CompletionKit
 
       @allow_loopback_endpoints = true
       @judge_calibration_enabled = true
+      @judge_examples_from_reviews = false
 
       @api_reference_authentication_partial = "completion_kit/api_reference/authentication"
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -311,6 +311,7 @@ files:
 - app/services/completion_kit/mcp_tools/responses.rb
 - app/services/completion_kit/mcp_tools/runs.rb
 - app/services/completion_kit/mcp_tools/tags.rb
+- app/services/completion_kit/metric_calibration_examples.rb
 - app/services/completion_kit/metric_calibration_stats.rb
 - app/services/completion_kit/metric_variant_generator.rb
 - app/services/completion_kit/model_discovery_service.rb
@@ -350,6 +351,7 @@ files:
 - app/views/completion_kit/metric_groups/new.html.erb
 - app/views/completion_kit/metric_groups/show.html.erb
 - app/views/completion_kit/metrics/_form.html.erb
+- app/views/completion_kit/metrics/_guiding_examples.html.erb
 - app/views/completion_kit/metrics/_rubric_diff.html.erb
 - app/views/completion_kit/metrics/_rubric_hint.html.erb
 - app/views/completion_kit/metrics/_starter_card.html.erb
@@ -429,6 +431,8 @@ files:
 - db/migrate/20260525000001_add_version_number_and_published_at_to_judge_versions.rb
 - db/migrate/20260528000001_rename_judge_version_to_metric_version.rb
 - db/migrate/20260528000002_add_metric_version_to_reviews.rb
+- db/migrate/20260529000001_remove_few_shot_examples_from_completion_kit_metrics.rb
+- db/migrate/20260530000001_add_excluded_from_examples_to_completion_kit_calibrations.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb