completion-kit 0.11.0 → 0.12.1

data/app/views/completion_kit/metrics/_guiding_examples.html.erb

@@ -12,7 +12,7 @@
             <span class="ck-guiding__output"><%= truncate(example[:output].to_s, length: 90) %></span>
             <span class="ck-guiding__scores"><span class="ck-guiding__judge"><%= example[:judge_score].to_i %></span> &rarr; <span class="ck-guiding__human"><%= example[:human_score].to_i %></span></span>
           <% end %>
-          <%= button_to exclude_example_metric_path(metric, calibration_id: example[:id]),
+          <%= button_to exclude_example_metric_path(metric, agreement_id: example[:id]),
                 method: :post, form_class: "inline-block", class: "ck-icon-btn",
                 title: "Stop using this case", "aria-label": "Stop using this case",
                 data: { turbo_confirm: "Stop using this corrected case to guide the judge?" } do %><%= heroicon_tag "x-mark", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>

data/app/views/completion_kit/metrics/edit.html.erb

@@ -10,7 +10,7 @@
   </div>
 </section>
 
-<% if CompletionKit.config.judge_calibration_enabled && @suggestion_draft.nil? && @edit_draft.nil? && @improve_disagreement_count.to_i.positive? %>
+<% if CompletionKit.config.judge_agreement_enabled && @suggestion_draft.nil? && @edit_draft.nil? && @improve_disagreement_count.to_i.positive? %>
   <div class="ck-suggestion-banner" role="status">
     <div class="ck-suggestion-banner__body">
       <p class="ck-kicker">Improve from reviews</p>

data/app/views/completion_kit/metrics/show.html.erb

@@ -42,7 +42,7 @@
   </div>
 </section>
 
-<% if CompletionKit.config.judge_calibration_enabled && @versions.any? %>
+<% if CompletionKit.config.judge_agreement_enabled && @versions.any? %>
   <% predecessor_of = @versions.index_with { |v| @versions.detect { |o| o.version_number < v.version_number } } %>
   <section class="ck-card ck-card--spaced">
     <p class="ck-kicker">Versions</p>
@@ -73,7 +73,7 @@
                     <%= button_to "Make current", publish_draft_metric_path(@metric, draft_id: v.id),
                           method: :post, form_class: "inline-block",
                           class: "ck-chip ck-chip--publish",
-                          data: { turbo_confirm: "Make #{v.version_label} the version to use? #{v.version_label} will be used in test runs using this metric now. Reviews you have already given stay with the version they were made against." } %>
+                          data: { turbo_confirm: "Make #{v.version_label} the version to use? It becomes the version used in test runs, and the reviews you gave on it count again. Reviews on the version you're leaving stay with it." } %>
                   <% end %>
                 </div>
                 <% vs = v.validation_summary %>
@@ -171,7 +171,7 @@
             <span class="ck-modal__foot-note">Roll this metric back to this version.</span>
             <%= button_to "Make #{v.version_label} current →", publish_draft_metric_path(@metric, draft_id: v.id),
                   method: :post, form_class: "inline-block", class: ck_button_classes(:dark),
-                  data: { turbo_confirm: "Make #{v.version_label} the version to use? #{v.version_label} will be used in test runs using this metric now. Reviews you have already given stay with the version they were made against." } %>
+                  data: { turbo_confirm: "Make #{v.version_label} the version to use? It becomes the version used in test runs, and the reviews you gave on it count again. Reviews on the version you're leaving stay with it." } %>
           <% end %>
         </footer>
       </article>
@@ -179,7 +179,7 @@
   <% end %>
 <% end %>
 
-<% if CompletionKit.config.judge_calibration_enabled %>
+<% if CompletionKit.config.judge_agreement_enabled %>
   <% draft = @suggestion_draft || @edit_draft %>
   <section class="ck-card ck-card--spaced">
     <div class="ck-prompt-preview__header">
@@ -197,8 +197,8 @@
     <%= turbo_stream_from "metric_#{@metric.id}_suggestion" %>
     <div id="ck-suggestion-status-<%= @metric.id %>" class="ck-suggestion-status"></div>
     <p class="ck-meta-copy">How often the judge lands on the same score you would. Review its scores to build that signal, and improve the metric to raise it.</p>
-    <%= render "completion_kit/calibrations/trust_panel",
-          stats: CompletionKit::MetricCalibrationStats.for(@metric),
+    <%= render "completion_kit/agreements/trust_panel",
+          stats: CompletionKit::MetricAgreementStats.for(@metric),
           metric: @metric %>
     <% if CompletionKit.config.judge_examples_from_reviews %>
       <%= render "completion_kit/metrics/guiding_examples", metric: @metric, examples: @guiding_examples %>

data/app/views/completion_kit/provider_credentials/_models_card.html.erb

@@ -17,7 +17,7 @@
               <% if provider_credential.discovery_status == "completed" %>
                 <span class="ck-model-list__summary-stamp">updated <time data-relative-time datetime="<%= provider_credential.updated_at.utc.iso8601 %>"><%= time_ago_in_words(provider_credential.updated_at) %> ago</time></span>
               <% end %>
-              <button type="button" class="ck-icon-btn ck-model-list__refresh<%= ' ck-icon-btn--spinning' if discovering %>" title="Refresh models" aria-label="Refresh available models" <%= 'disabled' if discovering %> onclick="event.preventDefault();event.stopPropagation();fetch('<%= refresh_provider_credential_path(provider_credential) %>', {method:'POST',headers:{'X-CSRF-Token':document.querySelector('meta[name=csrf-token]').content}})">
+              <button type="button" class="ck-icon-btn ck-model-list__refresh<%= ' ck-icon-btn--spinning' if discovering %>" title="Refresh models" aria-label="Refresh available models" <%= 'disabled' if discovering %> onclick="event.preventDefault();event.stopPropagation();fetch('<%= refresh_provider_credential_path(provider_credential) %>', {method:'POST',headers:{'X-CSRF-Token':document.querySelector('meta[name=csrf-token]').content}});if(window.ckStartDiscoveryPolling)ckStartDiscoveryPolling(8000)">
                 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="13" height="13" aria-hidden="true"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg>
               </button>
             </span>

data/app/views/completion_kit/provider_credentials/statuses.turbo_stream.erb

@@ -0,0 +1,4 @@
+<% @provider_credentials.each do |pc| %>
+  <%= turbo_stream.replace "discovery_status_#{pc.id}" do %><%= render "discovery_status", provider_credential: pc %><% end %>
+  <%= turbo_stream.replace "provider_models_#{pc.id}" do %><%= render "models_card", provider_credential: pc %><% end %>
+<% end %>

data/app/views/completion_kit/responses/show.html.erb

@@ -124,13 +124,13 @@
           <% if review.ai_feedback.present? %>
             <p class="ck-review-card__feedback"><%= review.ai_feedback %></p>
           <% end %>
-          <% if CompletionKit.config.judge_calibration_enabled && review.metric && review.ai_score %>
-            <% existing = CompletionKit::Calibration.find_by(
+          <% if CompletionKit.config.judge_agreement_enabled && review.metric && review.ai_score %>
+            <% existing = CompletionKit::Agreement.find_by(
                  response_id: @response.id, metric_id: review.metric_id,
                  created_by: CompletionKit.config.username.presence || "operator"
                ) %>
-            <%= render "completion_kit/calibrations/buttons",
-                  review: review, calibration: existing, run: @run,
+            <%= render "completion_kit/agreements/buttons",
+                  review: review, agreement: existing, run: @run,
                   response_row: @response, metric: review.metric %>
           <% end %>
         </div>

data/app/views/completion_kit/runs/show.html.erb

@@ -18,7 +18,7 @@
   <% dataset_preview_lines = dataset_lines.first(50) %>
 <% end %>
 
-<% if CompletionKit.config.judge_calibration_enabled %>
+<% if CompletionKit.config.judge_agreement_enabled %>
   <% stale_summary = @run.stale_review_summary %>
   <% if stale_summary.any? %>
     <div class="ck-stale-versions-banner" role="status">

data/config/routes.rb

@@ -41,7 +41,7 @@ CompletionKit::Engine.routes.draw do
       get :compare
     end
     resources :responses, only: [:show] do
-      resources :calibrations, only: [:create]
+      resources :agreements, only: [:create]
     end
   end
 
@@ -53,6 +53,7 @@ CompletionKit::Engine.routes.draw do
 
   resources :provider_credentials, only: [:index, :new, :create, :edit, :update] do
     post :refresh, on: :member
+    get :statuses, on: :collection
   end
   post "refresh_models", to: "provider_credentials#refresh_all", as: :refresh_models
 
@@ -75,7 +76,7 @@ CompletionKit::Engine.routes.draw do
         end
         resources :responses, only: [:index, :show] do
           resources :metrics, only: [] do
-            resources :calibrations, only: [:index, :create]
+            resources :agreements, only: [:index, :create]
           end
         end
       end
@@ -93,7 +94,7 @@ CompletionKit::Engine.routes.draw do
       resources :metric_groups
       resources :tags
       resources :provider_credentials
-      resources :calibrations, only: [:index, :destroy]
+      resources :agreements, only: [:index, :destroy]
     end
   end
 

data/db/migrate/20260531000002_backfill_review_metric_versions.rb

@@ -0,0 +1,33 @@
+class BackfillReviewMetricVersions < ActiveRecord::Migration[8.1]
+  def up
+    quoted_true = ActiveRecord::Base.connection.quote(true)
+    now = ActiveRecord::Base.connection.quote(Time.current)
+
+    execute <<~SQL
+      INSERT INTO completion_kit_metric_versions
+        (metric_id, instruction, rubric_bands, current, state, version_number, published_at, created_at, updated_at)
+      SELECT m.id, m.instruction, m.rubric_bands, #{quoted_true}, 'published', 1, #{now}, #{now}, #{now}
+      FROM completion_kit_metrics m
+      WHERE NOT EXISTS (
+        SELECT 1 FROM completion_kit_metric_versions mv WHERE mv.metric_id = m.id
+      )
+    SQL
+
+    execute <<~SQL
+      UPDATE completion_kit_reviews
+      SET metric_version_id = (
+        SELECT mv.id FROM completion_kit_metric_versions mv
+        WHERE mv.metric_id = completion_kit_reviews.metric_id AND mv.current = #{quoted_true}
+        LIMIT 1
+      )
+      WHERE metric_id IS NOT NULL
+        AND (
+          metric_version_id IS NULL
+          OR metric_version_id NOT IN (SELECT id FROM completion_kit_metric_versions)
+        )
+    SQL
+  end
+
+  def down
+  end
+end

data/db/migrate/20260531000003_add_metric_version_fk_to_reviews.rb

@@ -0,0 +1,6 @@
+class AddMetricVersionFkToReviews < ActiveRecord::Migration[8.1]
+  def change
+    add_foreign_key :completion_kit_reviews, :completion_kit_metric_versions,
+                    column: :metric_version_id, on_delete: :nullify
+  end
+end

data/db/migrate/20260531000004_rename_calibrations_to_agreements.rb

@@ -0,0 +1,19 @@
+class RenameCalibrationsToAgreements < ActiveRecord::Migration[8.1]
+  CALIBRATION_INDEXES = {
+    "index_ck_calibrations_on_metric_id" => "index_ck_agreements_on_metric_id",
+    "index_ck_calibrations_on_metric_version_id" => "index_ck_agreements_on_metric_version_id",
+    "index_ck_calibrations_on_response_id" => "index_ck_agreements_on_response_id",
+    "index_ck_calibrations_on_run_id" => "index_ck_agreements_on_run_id",
+    "index_ck_calibrations_on_response_metric_user" => "index_ck_agreements_on_response_metric_user"
+  }.freeze
+
+  def up
+    rename_table :completion_kit_calibrations, :completion_kit_agreements
+    CALIBRATION_INDEXES.each { |old_name, new_name| rename_index :completion_kit_agreements, old_name, new_name }
+  end
+
+  def down
+    CALIBRATION_INDEXES.each { |old_name, new_name| rename_index :completion_kit_agreements, new_name, old_name }
+    rename_table :completion_kit_agreements, :completion_kit_calibrations
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.11.0"
+  VERSION = "0.12.1"
 end

data/lib/completion_kit.rb

@@ -12,7 +12,7 @@ module CompletionKit
     attr_accessor :api_reference_authentication_partial
     attr_accessor :api_rate_limit, :web_rate_limit
     attr_accessor :allow_loopback_endpoints
-    attr_accessor :judge_calibration_enabled
+    attr_accessor :judge_agreement_enabled
     attr_accessor :judge_examples_from_reviews
 
     def initialize
@@ -29,7 +29,7 @@ module CompletionKit
       @web_rate_limit = 300
 
       @allow_loopback_endpoints = true
-      @judge_calibration_enabled = true
+      @judge_agreement_enabled = true
       @judge_examples_from_reviews = false
 
       @api_reference_authentication_partial = "completion_kit/api_reference/authentication"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.1
 platform: ruby
 authors:
 - Damien Bastin
@@ -234,8 +234,9 @@ files:
 - app/assets/images/completion_kit/logo.png
 - app/assets/javascripts/completion_kit/application.js
 - app/assets/stylesheets/completion_kit/application.css
+- app/controllers/completion_kit/agreements_controller.rb
+- app/controllers/completion_kit/api/v1/agreements_controller.rb
 - app/controllers/completion_kit/api/v1/base_controller.rb
-- app/controllers/completion_kit/api/v1/calibrations_controller.rb
 - app/controllers/completion_kit/api/v1/datasets_controller.rb
 - app/controllers/completion_kit/api/v1/metric_groups_controller.rb
 - app/controllers/completion_kit/api/v1/metric_versions_controller.rb
@@ -247,7 +248,6 @@ files:
 - app/controllers/completion_kit/api/v1/tags_controller.rb
 - app/controllers/completion_kit/api_reference_controller.rb
 - app/controllers/completion_kit/application_controller.rb
-- app/controllers/completion_kit/calibrations_controller.rb
 - app/controllers/completion_kit/dashboard_controller.rb
 - app/controllers/completion_kit/dashboard_dismissals_controller.rb
 - app/controllers/completion_kit/datasets_controller.rb
@@ -270,8 +270,8 @@ files:
 - app/jobs/completion_kit/model_discovery_job.rb
 - app/jobs/completion_kit/run_completion_check_job.rb
 - app/mailers/completion_kit/application_mailer.rb
+- app/models/completion_kit/agreement.rb
 - app/models/completion_kit/application_record.rb
-- app/models/completion_kit/calibration.rb
 - app/models/completion_kit/dashboard_dismissal.rb
 - app/models/completion_kit/dataset.rb
 - app/models/completion_kit/mcp_session.rb
@@ -292,16 +292,16 @@ files:
 - app/models/completion_kit/tagging.rb
 - app/models/concerns/completion_kit/has_job_status.rb
 - app/models/concerns/completion_kit/taggable.rb
+- app/services/completion_kit/agreement_math.rb
 - app/services/completion_kit/anthropic_client.rb
 - app/services/completion_kit/api_config.rb
-- app/services/completion_kit/calibration_math.rb
 - app/services/completion_kit/csv_processor.rb
 - app/services/completion_kit/dashboard_stats.rb
 - app/services/completion_kit/judge_service.rb
 - app/services/completion_kit/llm_client.rb
 - app/services/completion_kit/mcp_dispatcher.rb
+- app/services/completion_kit/mcp_tools/agreements.rb
 - app/services/completion_kit/mcp_tools/base.rb
-- app/services/completion_kit/mcp_tools/calibrations.rb
 - app/services/completion_kit/mcp_tools/datasets.rb
 - app/services/completion_kit/mcp_tools/judges.rb
 - app/services/completion_kit/mcp_tools/metric_groups.rb
@@ -312,8 +312,8 @@ files:
 - app/services/completion_kit/mcp_tools/responses.rb
 - app/services/completion_kit/mcp_tools/runs.rb
 - app/services/completion_kit/mcp_tools/tags.rb
-- app/services/completion_kit/metric_calibration_examples.rb
-- app/services/completion_kit/metric_calibration_stats.rb
+- app/services/completion_kit/metric_agreement_examples.rb
+- app/services/completion_kit/metric_agreement_stats.rb
 - app/services/completion_kit/metric_improvement_validator.rb
 - app/services/completion_kit/metric_variant_generator.rb
 - app/services/completion_kit/model_discovery_service.rb
@@ -328,14 +328,14 @@ files:
 - app/services/completion_kit/starter_metrics.rb
 - app/services/completion_kit/worker_health.rb
 - app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb
+- app/views/completion_kit/agreements/_buttons.html.erb
+- app/views/completion_kit/agreements/_trust_panel.html.erb
 - app/views/completion_kit/api_reference/_authentication.html.erb
 - app/views/completion_kit/api_reference/_body.html.erb
 - app/views/completion_kit/api_reference/_example.html.erb
 - app/views/completion_kit/api_reference/_resource_card.html.erb
 - app/views/completion_kit/api_reference/_resource_list.html.erb
 - app/views/completion_kit/api_reference/index.html.erb
-- app/views/completion_kit/calibrations/_buttons.html.erb
-- app/views/completion_kit/calibrations/_trust_panel.html.erb
 - app/views/completion_kit/dashboard/_eye_icon.html.erb
 - app/views/completion_kit/dashboard/_eye_off_icon.html.erb
 - app/views/completion_kit/dashboard/_failures_card.html.erb
@@ -379,6 +379,7 @@ files:
 - app/views/completion_kit/provider_credentials/edit.html.erb
 - app/views/completion_kit/provider_credentials/index.html.erb
 - app/views/completion_kit/provider_credentials/new.html.erb
+- app/views/completion_kit/provider_credentials/statuses.turbo_stream.erb
 - app/views/completion_kit/responses/show.html.erb
 - app/views/completion_kit/runs/_actions.html.erb
 - app/views/completion_kit/runs/_form.html.erb
@@ -440,6 +441,9 @@ files:
 - db/migrate/20260529000001_remove_few_shot_examples_from_completion_kit_metrics.rb
 - db/migrate/20260530000001_add_excluded_from_examples_to_completion_kit_calibrations.rb
 - db/migrate/20260531000001_add_validation_summary_to_completion_kit_metric_versions.rb
+- db/migrate/20260531000002_backfill_review_metric_versions.rb
+- db/migrate/20260531000003_add_metric_version_fk_to_reviews.rb
+- db/migrate/20260531000004_rename_calibrations_to_agreements.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb