completion-kit 0.10.0 → 0.12.0

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -17,7 +17,7 @@
   <input type="radio" name="ck-api-tab" id="ck-tab-datasets" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-metrics" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-metric-groups" class="ck-api-tabs__radio">
-  <input type="radio" name="ck-api-tab" id="ck-tab-calibrations" class="ck-api-tabs__radio">
+  <input type="radio" name="ck-api-tab" id="ck-tab-agreements" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-tags" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-providers" class="ck-api-tabs__radio">
 
@@ -29,7 +29,7 @@
     <label for="ck-tab-datasets" class="ck-api-tabs__label">Datasets <span class="ck-api-tabs__count">5</span></label>
     <label for="ck-tab-metrics" class="ck-api-tabs__label">Metrics <span class="ck-api-tabs__count">10</span></label>
     <label for="ck-tab-metric-groups" class="ck-api-tabs__label">Metric Groups <span class="ck-api-tabs__count">5</span></label>
-    <label for="ck-tab-calibrations" class="ck-api-tabs__label">Calibrations <span class="ck-api-tabs__count">3</span></label>
+    <label for="ck-tab-agreements" class="ck-api-tabs__label">Agreements <span class="ck-api-tabs__count">3</span></label>
     <label for="ck-tab-tags" class="ck-api-tabs__label">Tags <span class="ck-api-tabs__count">5</span></label>
     <label for="ck-tab-providers" class="ck-api-tabs__label">Providers <span class="ck-api-tabs__count">5</span></label>
   </nav>
@@ -238,8 +238,8 @@
             } %>
 
       <div class="ck-api-endpoint" style="padding-top: 1.5rem;">
-        <p class="ck-kicker" style="margin-bottom: 0.5rem;">Calibration loop</p>
-        <p class="ck-meta-copy">Drive metric improvement from disagree-flagged calibrations: ask the model to rewrite the instruction and rubric into a new draft version.</p>
+        <p class="ck-kicker" style="margin-bottom: 0.5rem;">Agreement loop</p>
+        <p class="ck-meta-copy">Drive metric improvement from disagree-flagged agreements: ask the model to rewrite the instruction and rubric into a new draft version.</p>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/metrics/:id/suggest_variants</p>
@@ -250,7 +250,7 @@
 
       <div class="ck-api-endpoint" style="padding-top: 1.5rem;">
         <p class="ck-kicker" style="margin-bottom: 0.5rem;">Metric versions</p>
-        <p class="ck-meta-copy">Every metric carries a history of versions (the current published one, prior published ones, and any draft suggestions). Reviews and calibrations record the version they ran against, so the API can surface stale state and let you revert.</p>
+        <p class="ck-meta-copy">Every metric carries a history of versions (the current published one, prior published ones, and any draft suggestions). Reviews and agreements record the version they ran against, so the API can surface stale state and let you revert.</p>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/metrics/:metric_id/metric_versions</p>
@@ -294,23 +294,23 @@
     </div>
 
     <div class="ck-api-tabs__panel">
-      <h2 class="ck-section-title">Calibrations</h2>
-      <p class="ck-copy">Per-verdict feedback events on a response/metric pair: agree, disagree (with a corrected score and note), or borderline. Calibrations capture the metric version that was current when the verdict was cast, which is what drives the trust signal and the "stale" indicators across the rest of the API.</p>
+      <h2 class="ck-section-title">Agreements</h2>
+      <p class="ck-copy">Per-verdict feedback events on a response/metric pair: agree, disagree (with a corrected score and note), or borderline. Agreements capture the metric version that was current when the verdict was cast, which is what drives the trust signal and the "stale" indicators across the rest of the API.</p>
       <div class="ck-api-endpoint">
-        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/calibrations</p>
-        <p class="ck-meta-copy">List calibrations across all runs. Supports filtering by any combination of the query params below.</p>
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/agreements</p>
+        <p class="ck-meta-copy">List agreements across all runs. Supports filtering by any combination of the query params below.</p>
         <p class="ck-api-params"><strong>Optional filters:</strong>&ensp;<code>run_id</code>, <code>response_id</code>, <code>metric_id</code>, <code>metric_version_id</code>, <code>created_by</code>, <code>verdict</code> (<code>agree</code>, <code>disagree</code>, or <code>borderline</code>)</p>
-        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl \"#{base_url}/api/v1/calibrations?metric_id=1&verdict=disagree\" \\\n  -H \"Authorization: Bearer #{token}\"" %>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl \"#{base_url}/api/v1/agreements?metric_id=1&verdict=disagree\" \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <div class="ck-api-endpoint">
-        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:run_id/responses/:response_id/metrics/:metric_id/calibrations</p>
-        <p class="ck-meta-copy">Cast a calibration on a specific response/metric pair. The metric version on the record is set automatically from the run's review.</p>
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:run_id/responses/:response_id/metrics/:metric_id/agreements</p>
+        <p class="ck-meta-copy">Cast an agreement on a specific response/metric pair. The metric version on the record is set automatically from the run's review.</p>
         <p class="ck-api-params"><strong>Required:</strong>&ensp;<code>verdict</code>, <code>created_by</code>&emsp;<strong>Optional:</strong>&ensp;<code>corrected_score</code>, <code>note</code></p>
-        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs/1/responses/42/metrics/3/calibrations \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"verdict\": \"disagree\", \"corrected_score\": 3, \"note\": \"too generous\", \"created_by\": \"alice\"}'" %>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs/1/responses/42/metrics/3/agreements \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"verdict\": \"disagree\", \"corrected_score\": 3, \"note\": \"too generous\", \"created_by\": \"alice\"}'" %>
       </div>
       <div class="ck-api-endpoint">
-        <p class="ck-api-method"><span class="ck-chip" style="color: var(--ck-danger);">DELETE</span> /api/v1/calibrations/:id</p>
-        <p class="ck-meta-copy">Delete a calibration. Returns 204 No Content.</p>
+        <p class="ck-api-method"><span class="ck-chip" style="color: var(--ck-danger);">DELETE</span> /api/v1/agreements/:id</p>
+        <p class="ck-meta-copy">Delete an agreement. Returns 204 No Content.</p>
       </div>
     </div>
 

data/app/views/completion_kit/metrics/_guiding_examples.html.erb

@@ -12,7 +12,7 @@
             <span class="ck-guiding__output"><%= truncate(example[:output].to_s, length: 90) %></span>
             <span class="ck-guiding__scores"><span class="ck-guiding__judge"><%= example[:judge_score].to_i %></span> &rarr; <span class="ck-guiding__human"><%= example[:human_score].to_i %></span></span>
           <% end %>
-          <%= button_to exclude_example_metric_path(metric, calibration_id: example[:id]),
+          <%= button_to exclude_example_metric_path(metric, agreement_id: example[:id]),
                 method: :post, form_class: "inline-block", class: "ck-icon-btn",
                 title: "Stop using this case", "aria-label": "Stop using this case",
                 data: { turbo_confirm: "Stop using this corrected case to guide the judge?" } do %><%= heroicon_tag "x-mark", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>

data/app/views/completion_kit/metrics/_suggestion_failed.html.erb

@@ -0,0 +1,3 @@
+<div id="ck-suggestion-status-<%= metric.id %>" class="ck-suggestion-status">
+  <span class="ck-cal-foot__note">The model returned no usable change. Try again, or review a few more scores first.</span>
+</div>

data/app/views/completion_kit/metrics/_suggestion_pending.html.erb

@@ -0,0 +1,3 @@
+<div id="ck-suggestion-status-<%= metric.id %>" class="ck-suggestion-status ck-suggestion-status--pending">
+  <span class="ck-cal-foot__note">Drafting a change and testing it against your <%= pluralize(count, "review") %>…</span>
+</div>

data/app/views/completion_kit/metrics/_suggestion_ready.html.erb

@@ -0,0 +1,4 @@
+<div id="ck-suggestion-status-<%= metric.id %>" class="ck-suggestion-status ck-suggestion-status--ready">
+  <span class="ck-cal-foot__note">Drafted <%= draft.version_label %> and tested it against your reviews.</span>
+  <%= link_to "Compare and publish →", CompletionKit::Engine.routes.url_helpers.metric_path(metric, show_change: draft.id), class: "ck-cal-link" %>
+</div>

data/app/views/completion_kit/metrics/_validation_scoreboard.html.erb

@@ -0,0 +1,12 @@
+<% s = summary %>
+<div class="ck-scoreboard">
+  <p class="ck-scoreboard__headline">Matches you on <strong><%= s["after"] %> of <%= s["tested"] %></strong> of your reviews <span class="ck-scoreboard__was">was <%= s["before"] %> of <%= s["tested"] %></span></p>
+  <ul class="ck-scoreboard__tally">
+    <li class="ck-scoreboard__stat ck-scoreboard__stat--fix">Fixes <strong><%= s["fixes"] %></strong></li>
+    <li class="ck-scoreboard__stat ck-scoreboard__stat--keep">Keeps <strong><%= s["keeps"] %></strong></li>
+    <li class="ck-scoreboard__stat ck-scoreboard__stat--break">Breaks <strong><%= s["breaks"] %></strong></li>
+  </ul>
+  <% if s["capped"] %>
+    <p class="ck-scoreboard__note">Tested against your 30 most recent reviews.</p>
+  <% end %>
+</div>

data/app/views/completion_kit/metrics/edit.html.erb

@@ -10,7 +10,7 @@
   </div>
 </section>
 
-<% if CompletionKit.config.judge_calibration_enabled && @suggestion_draft.nil? && @edit_draft.nil? && @improve_disagreement_count.to_i.positive? %>
+<% if CompletionKit.config.judge_agreement_enabled && @suggestion_draft.nil? && @edit_draft.nil? && @improve_disagreement_count.to_i.positive? %>
   <div class="ck-suggestion-banner" role="status">
     <div class="ck-suggestion-banner__body">
       <p class="ck-kicker">Improve from reviews</p>

data/app/views/completion_kit/metrics/show.html.erb

@@ -42,7 +42,7 @@
   </div>
 </section>
 
-<% if CompletionKit.config.judge_calibration_enabled && @versions.any? %>
+<% if CompletionKit.config.judge_agreement_enabled && @versions.any? %>
   <% predecessor_of = @versions.index_with { |v| @versions.detect { |o| o.version_number < v.version_number } } %>
   <section class="ck-card ck-card--spaced">
     <p class="ck-kicker">Versions</p>
@@ -73,17 +73,23 @@
                     <%= button_to "Make current", publish_draft_metric_path(@metric, draft_id: v.id),
                           method: :post, form_class: "inline-block",
                           class: "ck-chip ck-chip--publish",
-                          data: { turbo_confirm: "Make #{v.version_label} the version to use? #{v.version_label} will be used in test runs using this metric now. Reviews you have already given stay with the version they were made against." } %>
+                          data: { turbo_confirm: "Make #{v.version_label} the version to use? It becomes the version used in test runs, and the reviews you gave on it count again. Reviews on the version you're leaving stay with it." } %>
                   <% end %>
                 </div>
+                <% vs = v.validation_summary %>
                 <% if summary %>
-                  <button type="button" class="ck-cell-link ck-cell-link--delta" title="What changed from <%= pred.version_label %>" onclick="document.getElementById('ck-mvdiff-<%= v.id %>').showModal()">&Delta;</button>
+                  <div class="ck-version-change">
+                    <% if v.draft? && vs.present? %>
+                      <span class="ck-version-score"><span class="ck-version-score__label">Match</span> <%= vs["after"] %>/<%= vs["tested"] %></span>
+                    <% end %>
+                    <button type="button" class="ck-cell-link ck-cell-link--delta" title="What changed from <%= pred.version_label %>" onclick="document.getElementById('ck-mvdiff-<%= v.id %>').showModal()">&Delta;</button>
+                  </div>
                 <% end %>
               </div>
             </td>
             <td>
               <% source_label, source_class = case v.source
-                                              when "suggestion" then ["AI calibration", "ck-source-chip ck-source-chip--ai"]
+                                              when "suggestion" then ["AI suggestion", "ck-source-chip ck-source-chip--ai"]
                                               when "edit" then ["Manual edit", "ck-source-chip ck-source-chip--manual"]
                                               when "revert" then ["Reverted", "ck-source-chip ck-source-chip--revert"]
                                               else ["Original", "ck-source-chip ck-source-chip--initial"]
@@ -110,6 +116,7 @@
   <% @versions.each do |v| %>
     <% pred = predecessor_of[v] %>
     <% next unless v.change_summary_against(pred) %>
+    <% vs = v.validation_summary %>
     <dialog id="ck-mvdiff-<%= v.id %>" class="ck-modal" onclick="if(event.target===this)this.close()">
       <article class="ck-modal__panel" tabindex="-1" onclick="event.stopPropagation()">
         <header class="ck-modal__header">
@@ -120,6 +127,9 @@
           <button type="button" class="ck-modal__close" aria-label="Close" onclick="this.closest('dialog').close()">&times;</button>
         </header>
         <div class="ck-modal__body">
+          <% if v.draft? && vs.present? %>
+            <%= render "completion_kit/metrics/validation_scoreboard", summary: vs %>
+          <% end %>
           <% if pred.instruction.to_s != v.instruction.to_s %>
             <div class="ck-suggest-diff">
               <div class="ck-suggest-diff__pane">
@@ -152,14 +162,16 @@
                     title: "Discard draft #{v.version_label}", "aria-label": "Discard draft #{v.version_label}",
                     data: { turbo_confirm: "Discard draft #{v.version_label}? This can't be undone." } do %><%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
               <%= link_to "Edit", edit_metric_path(@metric), class: ck_button_classes(:light, variant: :outline) %>
+              <% net_negative = vs.present? && (vs["after"].to_i < vs["before"].to_i || vs["breaks"].to_i > vs["fixes"].to_i) %>
               <%= button_to "Publish #{v.version_label} →", publish_draft_metric_path(@metric, draft_id: v.id),
-                    method: :post, form_class: "inline-block", class: ck_button_classes(:dark) %>
+                    method: :post, form_class: "inline-block", class: ck_button_classes(:dark),
+                    data: net_negative ? { turbo_confirm: "This agrees with you less than the current version. Publish anyway?" } : {} %>
             </span>
           <% else %>
             <span class="ck-modal__foot-note">Roll this metric back to this version.</span>
             <%= button_to "Make #{v.version_label} current →", publish_draft_metric_path(@metric, draft_id: v.id),
                   method: :post, form_class: "inline-block", class: ck_button_classes(:dark),
-                  data: { turbo_confirm: "Make #{v.version_label} the version to use? #{v.version_label} will be used in test runs using this metric now. Reviews you have already given stay with the version they were made against." } %>
+                  data: { turbo_confirm: "Make #{v.version_label} the version to use? It becomes the version used in test runs, and the reviews you gave on it count again. Reviews on the version you're leaving stay with it." } %>
           <% end %>
         </footer>
       </article>
@@ -167,11 +179,11 @@
   <% end %>
 <% end %>
 
-<% if CompletionKit.config.judge_calibration_enabled %>
+<% if CompletionKit.config.judge_agreement_enabled %>
   <% draft = @suggestion_draft || @edit_draft %>
   <section class="ck-card ck-card--spaced">
     <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Calibration</p>
+      <p class="ck-kicker">Agreement</p>
       <% if draft.nil? && @improve_disagreement_count.positive? %>
         <%= button_to suggest_variants_metric_path(@metric),
               method: :post, form_class: "inline-block",
@@ -182,9 +194,11 @@
         <% end %>
       <% end %>
     </div>
-    <p class="ck-meta-copy">This is a measure of how often the judge's scores match a human reviewer. Review its scores to build that signal; the scores you disagree with become the cases the model learns from when you improve the metric.</p>
-    <%= render "completion_kit/calibrations/trust_panel",
-          stats: CompletionKit::MetricCalibrationStats.for(@metric),
+    <%= turbo_stream_from "metric_#{@metric.id}_suggestion" %>
+    <div id="ck-suggestion-status-<%= @metric.id %>" class="ck-suggestion-status"></div>
+    <p class="ck-meta-copy">How often the judge lands on the same score you would. Review its scores to build that signal, and improve the metric to raise it.</p>
+    <%= render "completion_kit/agreements/trust_panel",
+          stats: CompletionKit::MetricAgreementStats.for(@metric),
           metric: @metric %>
     <% if CompletionKit.config.judge_examples_from_reviews %>
       <%= render "completion_kit/metrics/guiding_examples", metric: @metric, examples: @guiding_examples %>

data/app/views/completion_kit/responses/show.html.erb

@@ -124,13 +124,13 @@
           <% if review.ai_feedback.present? %>
             <p class="ck-review-card__feedback"><%= review.ai_feedback %></p>
           <% end %>
-          <% if CompletionKit.config.judge_calibration_enabled && review.metric && review.ai_score %>
-            <% existing = CompletionKit::Calibration.find_by(
+          <% if CompletionKit.config.judge_agreement_enabled && review.metric && review.ai_score %>
+            <% existing = CompletionKit::Agreement.find_by(
                  response_id: @response.id, metric_id: review.metric_id,
                  created_by: CompletionKit.config.username.presence || "operator"
                ) %>
-            <%= render "completion_kit/calibrations/buttons",
-                  review: review, calibration: existing, run: @run,
+            <%= render "completion_kit/agreements/buttons",
+                  review: review, agreement: existing, run: @run,
                   response_row: @response, metric: review.metric %>
           <% end %>
         </div>

data/app/views/completion_kit/runs/show.html.erb

@@ -18,7 +18,7 @@
   <% dataset_preview_lines = dataset_lines.first(50) %>
 <% end %>
 
-<% if CompletionKit.config.judge_calibration_enabled %>
+<% if CompletionKit.config.judge_agreement_enabled %>
   <% stale_summary = @run.stale_review_summary %>
   <% if stale_summary.any? %>
     <div class="ck-stale-versions-banner" role="status">

data/config/routes.rb

@@ -41,7 +41,7 @@ CompletionKit::Engine.routes.draw do
       get :compare
     end
     resources :responses, only: [:show] do
-      resources :calibrations, only: [:create]
+      resources :agreements, only: [:create]
     end
   end
 
@@ -75,7 +75,7 @@ CompletionKit::Engine.routes.draw do
         end
         resources :responses, only: [:index, :show] do
           resources :metrics, only: [] do
-            resources :calibrations, only: [:index, :create]
+            resources :agreements, only: [:index, :create]
           end
         end
       end
@@ -93,7 +93,7 @@ CompletionKit::Engine.routes.draw do
       resources :metric_groups
       resources :tags
       resources :provider_credentials
-      resources :calibrations, only: [:index, :destroy]
+      resources :agreements, only: [:index, :destroy]
     end
   end
 

data/db/migrate/20260531000001_add_validation_summary_to_completion_kit_metric_versions.rb

@@ -0,0 +1,5 @@
+class AddValidationSummaryToCompletionKitMetricVersions < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_metric_versions, :validation_summary, :text
+  end
+end

data/db/migrate/20260531000002_backfill_review_metric_versions.rb

@@ -0,0 +1,33 @@
+class BackfillReviewMetricVersions < ActiveRecord::Migration[8.1]
+  def up
+    quoted_true = ActiveRecord::Base.connection.quote(true)
+    now = ActiveRecord::Base.connection.quote(Time.current)
+
+    execute <<~SQL
+      INSERT INTO completion_kit_metric_versions
+        (metric_id, instruction, rubric_bands, current, state, version_number, published_at, created_at, updated_at)
+      SELECT m.id, m.instruction, m.rubric_bands, #{quoted_true}, 'published', 1, #{now}, #{now}, #{now}
+      FROM completion_kit_metrics m
+      WHERE NOT EXISTS (
+        SELECT 1 FROM completion_kit_metric_versions mv WHERE mv.metric_id = m.id
+      )
+    SQL
+
+    execute <<~SQL
+      UPDATE completion_kit_reviews
+      SET metric_version_id = (
+        SELECT mv.id FROM completion_kit_metric_versions mv
+        WHERE mv.metric_id = completion_kit_reviews.metric_id AND mv.current = #{quoted_true}
+        LIMIT 1
+      )
+      WHERE metric_id IS NOT NULL
+        AND (
+          metric_version_id IS NULL
+          OR metric_version_id NOT IN (SELECT id FROM completion_kit_metric_versions)
+        )
+    SQL
+  end
+
+  def down
+  end
+end

data/db/migrate/20260531000003_add_metric_version_fk_to_reviews.rb

@@ -0,0 +1,6 @@
+class AddMetricVersionFkToReviews < ActiveRecord::Migration[8.1]
+  def change
+    add_foreign_key :completion_kit_reviews, :completion_kit_metric_versions,
+                    column: :metric_version_id, on_delete: :nullify
+  end
+end

data/db/migrate/20260531000004_rename_calibrations_to_agreements.rb

@@ -0,0 +1,19 @@
+class RenameCalibrationsToAgreements < ActiveRecord::Migration[8.1]
+  CALIBRATION_INDEXES = {
+    "index_ck_calibrations_on_metric_id" => "index_ck_agreements_on_metric_id",
+    "index_ck_calibrations_on_metric_version_id" => "index_ck_agreements_on_metric_version_id",
+    "index_ck_calibrations_on_response_id" => "index_ck_agreements_on_response_id",
+    "index_ck_calibrations_on_run_id" => "index_ck_agreements_on_run_id",
+    "index_ck_calibrations_on_response_metric_user" => "index_ck_agreements_on_response_metric_user"
+  }.freeze
+
+  def up
+    rename_table :completion_kit_calibrations, :completion_kit_agreements
+    CALIBRATION_INDEXES.each { |old_name, new_name| rename_index :completion_kit_agreements, old_name, new_name }
+  end
+
+  def down
+    CALIBRATION_INDEXES.each { |old_name, new_name| rename_index :completion_kit_agreements, new_name, old_name }
+    rename_table :completion_kit_agreements, :completion_kit_calibrations
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.10.0"
+  VERSION = "0.12.0"
 end

data/lib/completion_kit.rb

@@ -12,7 +12,7 @@ module CompletionKit
     attr_accessor :api_reference_authentication_partial
     attr_accessor :api_rate_limit, :web_rate_limit
     attr_accessor :allow_loopback_endpoints
-    attr_accessor :judge_calibration_enabled
+    attr_accessor :judge_agreement_enabled
     attr_accessor :judge_examples_from_reviews
 
     def initialize
@@ -29,7 +29,7 @@ module CompletionKit
       @web_rate_limit = 300
 
       @allow_loopback_endpoints = true
-      @judge_calibration_enabled = true
+      @judge_agreement_enabled = true
       @judge_examples_from_reviews = false
 
       @api_reference_authentication_partial = "completion_kit/api_reference/authentication"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -234,8 +234,9 @@ files:
 - app/assets/images/completion_kit/logo.png
 - app/assets/javascripts/completion_kit/application.js
 - app/assets/stylesheets/completion_kit/application.css
+- app/controllers/completion_kit/agreements_controller.rb
+- app/controllers/completion_kit/api/v1/agreements_controller.rb
 - app/controllers/completion_kit/api/v1/base_controller.rb
-- app/controllers/completion_kit/api/v1/calibrations_controller.rb
 - app/controllers/completion_kit/api/v1/datasets_controller.rb
 - app/controllers/completion_kit/api/v1/metric_groups_controller.rb
 - app/controllers/completion_kit/api/v1/metric_versions_controller.rb
@@ -247,7 +248,6 @@ files:
 - app/controllers/completion_kit/api/v1/tags_controller.rb
 - app/controllers/completion_kit/api_reference_controller.rb
 - app/controllers/completion_kit/application_controller.rb
-- app/controllers/completion_kit/calibrations_controller.rb
 - app/controllers/completion_kit/dashboard_controller.rb
 - app/controllers/completion_kit/dashboard_dismissals_controller.rb
 - app/controllers/completion_kit/datasets_controller.rb
@@ -266,11 +266,12 @@ files:
 - app/jobs/completion_kit/application_job.rb
 - app/jobs/completion_kit/generate_row_job.rb
 - app/jobs/completion_kit/judge_review_job.rb
+- app/jobs/completion_kit/metric_suggestion_job.rb
 - app/jobs/completion_kit/model_discovery_job.rb
 - app/jobs/completion_kit/run_completion_check_job.rb
 - app/mailers/completion_kit/application_mailer.rb
+- app/models/completion_kit/agreement.rb
 - app/models/completion_kit/application_record.rb
-- app/models/completion_kit/calibration.rb
 - app/models/completion_kit/dashboard_dismissal.rb
 - app/models/completion_kit/dataset.rb
 - app/models/completion_kit/mcp_session.rb
@@ -291,16 +292,16 @@ files:
 - app/models/completion_kit/tagging.rb
 - app/models/concerns/completion_kit/has_job_status.rb
 - app/models/concerns/completion_kit/taggable.rb
+- app/services/completion_kit/agreement_math.rb
 - app/services/completion_kit/anthropic_client.rb
 - app/services/completion_kit/api_config.rb
-- app/services/completion_kit/calibration_math.rb
 - app/services/completion_kit/csv_processor.rb
 - app/services/completion_kit/dashboard_stats.rb
 - app/services/completion_kit/judge_service.rb
 - app/services/completion_kit/llm_client.rb
 - app/services/completion_kit/mcp_dispatcher.rb
+- app/services/completion_kit/mcp_tools/agreements.rb
 - app/services/completion_kit/mcp_tools/base.rb
-- app/services/completion_kit/mcp_tools/calibrations.rb
 - app/services/completion_kit/mcp_tools/datasets.rb
 - app/services/completion_kit/mcp_tools/judges.rb
 - app/services/completion_kit/mcp_tools/metric_groups.rb
@@ -311,8 +312,9 @@ files:
 - app/services/completion_kit/mcp_tools/responses.rb
 - app/services/completion_kit/mcp_tools/runs.rb
 - app/services/completion_kit/mcp_tools/tags.rb
-- app/services/completion_kit/metric_calibration_examples.rb
-- app/services/completion_kit/metric_calibration_stats.rb
+- app/services/completion_kit/metric_agreement_examples.rb
+- app/services/completion_kit/metric_agreement_stats.rb
+- app/services/completion_kit/metric_improvement_validator.rb
 - app/services/completion_kit/metric_variant_generator.rb
 - app/services/completion_kit/model_discovery_service.rb
 - app/services/completion_kit/ollama_client.rb
@@ -326,14 +328,14 @@ files:
 - app/services/completion_kit/starter_metrics.rb
 - app/services/completion_kit/worker_health.rb
 - app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb
+- app/views/completion_kit/agreements/_buttons.html.erb
+- app/views/completion_kit/agreements/_trust_panel.html.erb
 - app/views/completion_kit/api_reference/_authentication.html.erb
 - app/views/completion_kit/api_reference/_body.html.erb
 - app/views/completion_kit/api_reference/_example.html.erb
 - app/views/completion_kit/api_reference/_resource_card.html.erb
 - app/views/completion_kit/api_reference/_resource_list.html.erb
 - app/views/completion_kit/api_reference/index.html.erb
-- app/views/completion_kit/calibrations/_buttons.html.erb
-- app/views/completion_kit/calibrations/_trust_panel.html.erb
 - app/views/completion_kit/dashboard/_eye_icon.html.erb
 - app/views/completion_kit/dashboard/_eye_off_icon.html.erb
 - app/views/completion_kit/dashboard/_failures_card.html.erb
@@ -355,6 +357,10 @@ files:
 - app/views/completion_kit/metrics/_rubric_diff.html.erb
 - app/views/completion_kit/metrics/_rubric_hint.html.erb
 - app/views/completion_kit/metrics/_starter_card.html.erb
+- app/views/completion_kit/metrics/_suggestion_failed.html.erb
+- app/views/completion_kit/metrics/_suggestion_pending.html.erb
+- app/views/completion_kit/metrics/_suggestion_ready.html.erb
+- app/views/completion_kit/metrics/_validation_scoreboard.html.erb
 - app/views/completion_kit/metrics/edit.html.erb
 - app/views/completion_kit/metrics/index.html.erb
 - app/views/completion_kit/metrics/new.html.erb
@@ -433,6 +439,10 @@ files:
 - db/migrate/20260528000002_add_metric_version_to_reviews.rb
 - db/migrate/20260529000001_remove_few_shot_examples_from_completion_kit_metrics.rb
 - db/migrate/20260530000001_add_excluded_from_examples_to_completion_kit_calibrations.rb
+- db/migrate/20260531000001_add_validation_summary_to_completion_kit_metric_versions.rb
+- db/migrate/20260531000002_backfill_review_metric_versions.rb
+- db/migrate/20260531000003_add_metric_version_fk_to_reviews.rb
+- db/migrate/20260531000004_rename_calibrations_to_agreements.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb