completion-kit 0.10.0 → 0.12.0

data/app/jobs/completion_kit/metric_suggestion_job.rb

@@ -0,0 +1,46 @@
+require "faraday"
+
+module CompletionKit
+  class MetricSuggestionJob < ApplicationJob
+    queue_as :llm
+
+    retry_on Faraday::TimeoutError, Faraday::ConnectionFailed, wait: :polynomially_longer, attempts: 5
+    retry_on CompletionKit::RateLimitError, wait: :polynomially_longer, attempts: 5
+
+    rescue_from(StandardError) do |error|
+      Rails.error.report(error, handled: true, context: { job: self.class.name })
+      broadcast_status(@metric, partial: "completion_kit/metrics/suggestion_failed", locals: { metric: @metric })
+    end
+
+    def perform(metric_id)
+      @metric = Metric.find_by(id: metric_id)
+      return unless @metric
+
+      MetricVersion.drafts.where(metric_id: @metric.id, source: "suggestion").destroy_all
+
+      generator = MetricVariantGenerator.new(@metric, count: 1)
+      variants = generator.call
+      if variants.empty?
+        broadcast_status(@metric, partial: "completion_kit/metrics/suggestion_failed", locals: { metric: @metric })
+        return
+      end
+
+      draft = generator.persist!(variants).max_by(&:version_number)
+      summary = MetricImprovementValidator.new(@metric, draft).call
+      draft.update!(validation_summary: summary)
+
+      broadcast_status(@metric, partial: "completion_kit/metrics/suggestion_ready", locals: { metric: @metric, draft: draft })
+    end
+
+    private
+
+    def broadcast_status(metric, partial:, locals:)
+      html = CompletionKit::ApplicationController.render(partial: partial, locals: locals)
+      Turbo::StreamsChannel.broadcast_replace_to(
+        "metric_#{metric.id}_suggestion",
+        target: "ck-suggestion-status-#{metric.id}",
+        html: html
+      )
+    end
+  end
+end

data/app/models/completion_kit/{calibration.rb → agreement.rb}

@@ -1,5 +1,5 @@
 module CompletionKit
-  class Calibration < ApplicationRecord
+  class Agreement < ApplicationRecord
     VERDICTS = %w[agree disagree borderline].freeze
 
     belongs_to :run

data/app/models/completion_kit/metric_version.rb

@@ -3,9 +3,10 @@ module CompletionKit
     STATES = %w[draft published].freeze
 
     belongs_to :metric
-    has_many :calibrations, dependent: :destroy
+    has_many :agreements, dependent: :destroy
 
     serialize :rubric_bands, coder: JSON
+    serialize :validation_summary, coder: JSON
 
     before_validation :assign_version_number, on: :create
 
@@ -82,22 +83,6 @@ module CompletionKit
       self
     end
 
-    def revert!
-      raise ArgumentError, "only a published version can be reverted to" unless published?
-      audit = nil
-      MetricVersion.transaction do
-        audit = self.class.create!(
-          metric: metric,
-          instruction: instruction,
-          rubric_bands: rubric_bands,
-          state: "draft",
-          source: "revert"
-        )
-        audit.publish!
-      end
-      audit
-    end
-
     def as_json(options = {})
       {
         id: id,

data/app/models/completion_kit/review.rb

@@ -8,6 +8,7 @@ module CompletionKit
     has_many :dashboard_dismissals, as: :dismissable, dependent: :destroy
 
     validates :metric_name, presence: true
+    validates :metric_version, presence: true
     validates :ai_score, numericality: { greater_than_or_equal_to: 1, less_than_or_equal_to: 5 }, allow_nil: true
 
     before_validation :set_default_status

data/app/services/completion_kit/{calibration_math.rb → agreement_math.rb}

@@ -1,5 +1,5 @@
 module CompletionKit
-  module CalibrationMath
+  module AgreementMath
     Z_95 = 1.959963984540054
 
     module_function

data/app/services/completion_kit/mcp_dispatcher.rb

@@ -35,7 +35,7 @@ module CompletionKit
         McpTools::MetricVersions.definitions +
         McpTools::ProviderCredentials.definitions +
         McpTools::Tags.definitions +
-        McpTools::Calibrations.definitions +
+        McpTools::Agreements.definitions +
         McpTools::Judges.definitions
     end
 
@@ -50,7 +50,7 @@ module CompletionKit
       when /\Ametrics_/              then McpTools::Metrics.call(name, arguments)
       when /\Aprovider_credentials_/ then McpTools::ProviderCredentials.call(name, arguments)
       when /\Atags_/                 then McpTools::Tags.call(name, arguments)
-      when /\Acalibrations_/         then McpTools::Calibrations.call(name, arguments)
+      when /\Aagreements_/           then McpTools::Agreements.call(name, arguments)
       when /\Ajudges_/               then McpTools::Judges.call(name, arguments)
       else raise MethodNotFound, "Unknown tool: #{name}"
       end

data/app/services/completion_kit/mcp_tools/{calibrations.rb → agreements.rb}

@@ -1,11 +1,11 @@
 module CompletionKit
   module McpTools
-    module Calibrations
+    module Agreements
       extend Base
 
       TOOLS = {
-        "calibrations_list" => {
-          description: "List calibrations. Filter by run_id, response_id, metric_id, or created_by.",
+        "agreements_list" => {
+          description: "List agreements. Filter by run_id, response_id, metric_id, or created_by.",
           inputSchema: {
             type: "object",
             properties: {
@@ -18,8 +18,8 @@ module CompletionKit
           },
           handler: :list
         },
-        "calibrations_create" => {
-          description: "Upsert a calibration for (run, response, metric, created_by). Verdict is one of agree, disagree, borderline. corrected_score (1..5) is required when verdict is 'disagree'.",
+        "agreements_create" => {
+          description: "Upsert an agreement for (run, response, metric, created_by). Verdict is one of agree, disagree, borderline. corrected_score (1..5) is required when verdict is 'disagree'.",
           inputSchema: {
             type: "object",
             properties: {
@@ -38,7 +38,7 @@ module CompletionKit
       }.freeze
 
       def self.list(args)
-        scope = CompletionKit::Calibration.all
+        scope = CompletionKit::Agreement.all
         scope = scope.where(run_id: args["run_id"]) if args["run_id"]
         scope = scope.where(response_id: args["response_id"]) if args["response_id"]
         scope = scope.where(metric_id: args["metric_id"]) if args["metric_id"]
@@ -52,20 +52,20 @@ module CompletionKit
         metric = CompletionKit::Metric.find(args["metric_id"])
         created_by = args["created_by"].presence || "mcp"
 
-        calibration = CompletionKit::Calibration.find_or_initialize_by(
+        agreement = CompletionKit::Agreement.find_or_initialize_by(
           run_id: run.id, response_id: response.id, metric_id: metric.id, created_by: created_by
         )
-        calibration.assign_attributes(
+        agreement.assign_attributes(
           metric_version: CompletionKit::MetricVersion.ensure_current_for(metric),
           verdict: args["verdict"],
           corrected_score: args["corrected_score"],
           note: args["note"]
         )
 
-        if calibration.save
-          text_result(calibration.as_json)
+        if agreement.save
+          text_result(agreement.as_json)
         else
-          error_result(calibration.errors.full_messages.join(", "))
+          error_result(agreement.errors.full_messages.join(", "))
         end
       end
     end

data/app/services/completion_kit/mcp_tools/judges.rb

@@ -33,7 +33,7 @@ module CompletionKit
           handler: :replay
         },
         "judges_compare" => {
-          description: "Compare two metric versions' calibration stats side by side. Pass either two metric_version_ids or one metric_id with metric_version_a_id / metric_version_b_id.",
+          description: "Compare two metric versions' agreement stats side by side. Pass either two metric_version_ids or one metric_id with metric_version_a_id / metric_version_b_id.",
           inputSchema: {
             type: "object",
             properties: {
@@ -77,8 +77,8 @@ module CompletionKit
         metric = CompletionKit::Metric.find(args["metric_id"])
         a = CompletionKit::MetricVersion.where(metric_id: metric.id).find(args["metric_version_a_id"])
         b = CompletionKit::MetricVersion.where(metric_id: metric.id).find(args["metric_version_b_id"])
-        stats_a = CompletionKit::MetricCalibrationStats.for(metric, metric_version: a)
-        stats_b = CompletionKit::MetricCalibrationStats.for(metric, metric_version: b)
+        stats_a = CompletionKit::MetricAgreementStats.for(metric, metric_version: a)
+        stats_b = CompletionKit::MetricAgreementStats.for(metric, metric_version: b)
         text_result({
           metric_id: metric.id,
           a: metric_version_payload(a, stats_a),

data/app/services/completion_kit/mcp_tools/metric_versions.rb

@@ -47,13 +47,8 @@ module CompletionKit
 
       def self.publish(args)
         version = CompletionKit::MetricVersion.find(args["metric_version_id"])
-        if version.published? && !version.current?
-          audit = version.revert!
-          text_result(audit.as_json)
-        else
-          version.publish!
-          text_result(version.reload.as_json)
-        end
+        version.publish!
+        text_result(version.reload.as_json)
       end
 
       def self.dismiss(args)

data/app/services/completion_kit/{metric_calibration_examples.rb → metric_agreement_examples.rb}

@@ -1,5 +1,5 @@
 module CompletionKit
-  module MetricCalibrationExamples
+  module MetricAgreementExamples
     DEFAULT_JUDGE_EXAMPLE_LIMIT = 5
 
     module_function
@@ -9,18 +9,18 @@ module CompletionKit
     end
 
     def disagreements_for(metric, limit: 8)
-      calibrations_for(metric, verdict: "disagree", limit: limit)
+      agreements_for(metric, verdict: "disagree", limit: limit)
     end
 
     def borderlines_for(metric, limit: 6)
-      calibrations_for(metric, verdict: "borderline", limit: limit)
+      agreements_for(metric, verdict: "borderline", limit: limit)
     end
 
     def judge_examples_for(metric, exclude_response_id: nil, limit: DEFAULT_JUDGE_EXAMPLE_LIMIT)
       current_version = MetricVersion.current.find_by(metric_id: metric.id)
       return [] unless current_version
 
-      relation = Calibration
+      relation = Agreement
                  .where(metric_id: metric.id, metric_version_id: current_version.id, excluded_from_examples: false)
                  .where.not(corrected_score: nil)
       relation = relation.where.not(response_id: exclude_response_id) if exclude_response_id
@@ -28,8 +28,8 @@ module CompletionKit
         .reject { |example| example[:judge_score].nil? }
     end
 
-    def calibrations_for(metric, verdict:, limit:)
-      base = Calibration.where(metric_id: metric.id, verdict: verdict)
+    def agreements_for(metric, verdict:, limit:)
+      base = Agreement.where(metric_id: metric.id, verdict: verdict)
       current_version = MetricVersion.current.find_by(metric_id: metric.id)
       scoped = current_version ? base.where(metric_version_id: current_version.id) : base
       effective = scoped.exists? ? scoped : base

data/app/services/completion_kit/{metric_calibration_stats.rb → metric_agreement_stats.rb}

@@ -1,5 +1,5 @@
 module CompletionKit
-  class MetricCalibrationStats
+  class MetricAgreementStats
     PROVISIONAL_MIN = 10
     FIRM_MIN = 30
 
@@ -49,7 +49,7 @@ module CompletionKit
     end
 
     def call
-      scope = Calibration.where(metric_id: @metric.id)
+      scope = Agreement.where(metric_id: @metric.id)
       if @metric_version
         scope = scope.where(metric_version_id: @metric_version.id)
       elsif !@all_versions
@@ -62,12 +62,12 @@ module CompletionKit
       disagrees = verdicts.count { |v, _, _| v == "disagree" }
       borderlines = verdicts.count { |v, _, _| v == "borderline" }
 
-      ci = CalibrationMath.wilson_interval(successes: agrees, n: n)
+      ci = AgreementMath.wilson_interval(successes: agrees, n: n)
 
       pairs = score_pairs(verdicts)
-      mae_value = CalibrationMath.mae(pairs)
-      pearson_value = CalibrationMath.pearson(pairs)
-      kappa_value = CalibrationMath.quadratic_weighted_kappa(pairs, categories: 1..5)
+      mae_value = AgreementMath.mae(pairs)
+      pearson_value = AgreementMath.pearson(pairs)
+      kappa_value = AgreementMath.quadratic_weighted_kappa(pairs, categories: 1..5)
 
       Result.new(
         sample_size: n,

data/app/services/completion_kit/metric_improvement_validator.rb

@@ -0,0 +1,101 @@
+module CompletionKit
+  class MetricImprovementValidator
+    ANSWER_KEY_LIMIT = 30
+
+    def initialize(metric, candidate, scorer: nil)
+      @metric = metric
+      @candidate = candidate
+      @scorer = scorer || method(:rescore)
+    end
+
+    def call
+      key = answer_key
+      rows = []
+      key.each do |entry|
+        begin
+          score = @scorer.call(entry[:response], @candidate)
+        rescue StandardError
+          next
+        end
+        rows << classify(entry, score.to_i)
+      end
+      summarize(rows, key.size, key_capped?)
+    end
+
+    private
+
+    def answer_key
+      current = MetricVersion.current.find_by(metric_id: @metric.id)
+      return [] unless current
+
+      base = Agreement.where(metric_id: @metric.id, metric_version_id: current.id, verdict: %w[agree disagree])
+      @key_size_before_cap = base.count
+      base.includes(response: :reviews)
+          .order(created_at: :desc)
+          .limit(ANSWER_KEY_LIMIT)
+          .filter_map do |cal|
+        response = cal.response
+        next unless response.response_text.present?
+        review = response.reviews.find { |r| r.metric_id == @metric.id }
+        position = cal.verdict == "disagree" ? cal.corrected_score : review&.ai_score
+        next if position.nil?
+        { response: response, verdict: cal.verdict, position: position }
+      end
+    end
+
+    def key_capped?
+      @key_size_before_cap.to_i > ANSWER_KEY_LIMIT
+    end
+
+    def classify(entry, candidate_score)
+      matched = candidate_score == entry[:position].to_i
+      outcome = if entry[:verdict] == "disagree"
+        matched ? "fix" : "still_off"
+      else
+        matched ? "keep" : "break"
+      end
+      {
+        "response_id" => entry[:response].id,
+        "verdict" => entry[:verdict],
+        "position" => entry[:position].to_i,
+        "candidate_score" => candidate_score,
+        "outcome" => outcome
+      }
+    end
+
+    def summarize(rows, total, capped)
+      fixes = rows.count { |r| r["outcome"] == "fix" }
+      keeps = rows.count { |r| r["outcome"] == "keep" }
+      breaks = rows.count { |r| r["outcome"] == "break" }
+      still_off = rows.count { |r| r["outcome"] == "still_off" }
+      agreements = rows.count { |r| r["verdict"] == "agree" }
+      {
+        "total" => total,
+        "tested" => rows.size,
+        "capped" => capped,
+        "fixes" => fixes,
+        "keeps" => keeps,
+        "breaks" => breaks,
+        "still_off" => still_off,
+        "before" => agreements,
+        "after" => fixes + keeps,
+        "rows" => rows
+      }
+    end
+
+    def rescore(response, candidate)
+      run = response.run
+      config = ApiConfig.for_model(run.judge_model).merge(judge_model: run.judge_model)
+      rubric_text = Metric.rubric_text_for(Metric.normalize_rubric_bands(candidate.rubric_bands))
+      result = JudgeService.new(config).evaluate(
+        response.response_text,
+        response.expected_output,
+        run.prompt&.template,
+        criteria: candidate.instruction.to_s,
+        rubric_text: rubric_text,
+        input_data: response.input_data
+      )
+      result[:score]
+    end
+  end
+end

data/app/services/completion_kit/metric_variant_generator.rb

@@ -41,8 +41,8 @@ module CompletionKit
     private
 
     def build_meta_prompt
-      disagreements = MetricCalibrationExamples.disagreements_for(@metric)
-      borderlines = MetricCalibrationExamples.borderlines_for(@metric)
+      disagreements = MetricAgreementExamples.disagreements_for(@metric)
+      borderlines = MetricAgreementExamples.borderlines_for(@metric)
       sections = []
       sections << "You are an expert evaluator. The judge below is misaligned with humans. Propose #{@count == 1 ? "a single" : "#{@count}"} concrete rewrite that closes the gap."
       sections << ""

data/app/views/completion_kit/{calibrations → agreements}/_buttons.html.erb

@@ -1,34 +1,34 @@
-<div id="calibration_<%= response_row.id %>_<%= metric.id %>" class="ck-calibration">
-  <% current_verdict = calibration&.verdict %>
+<div id="agreement_<%= response_row.id %>_<%= metric.id %>" class="ck-agreement">
+  <% current_verdict = agreement&.verdict %>
   <% pending_verdict = local_assigns[:pending_verdict] %>
   <% active_verdict = pending_verdict || current_verdict %>
   <% error = local_assigns[:error] %>
   <% me = CompletionKit.config.username.presence || "operator" %>
-  <% other_calibrations = CompletionKit::Calibration
+  <% other_agreements = CompletionKit::Agreement
         .where(response_id: response_row.id, metric_id: metric.id)
         .where.not(created_by: me)
         .order(created_at: :asc).to_a %>
   <% verdict_icons = { "agree" => "hand-thumb-up", "disagree" => "hand-thumb-down", "borderline" => "scale" } %>
-  <p class="ck-calibration__prompt">
-    <span class="ck-calibration__label">Your verdict</span>
-    <% if other_calibrations.any? %>
-      <span class="ck-calibration__meta"><%= pluralize(other_calibrations.size, "other verdict") %> on this score</span>
-      <span class="ck-calibration__sep">·</span>
+  <p class="ck-agreement__prompt">
+    <span class="ck-agreement__label">Your verdict</span>
+    <% if other_agreements.any? %>
+      <span class="ck-agreement__meta"><%= pluralize(other_agreements.size, "other verdict") %> on this score</span>
+      <span class="ck-agreement__sep">·</span>
     <% end %>
-    <%= link_to metric_path(metric), class: "ck-calibration__meta-link" do %><%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true" %>Calibration →<% end %>
+    <%= link_to metric_path(metric, anchor: "agreement"), class: "ck-agreement__meta-link" do %><%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true" %>Agreement →<% end %>
   </p>
-  <div class="ck-calibration__buttons">
+  <div class="ck-agreement__buttons">
     <% verdict_icons = { "agree" => "hand-thumb-up", "disagree" => "hand-thumb-down", "borderline" => "scale" } %>
     <% verdict_hints = {
          "agree" => "The score looks right.",
          "disagree" => "The score is wrong — you'll pick the right one.",
          "borderline" => "The rubric is unclear here; either score could be defensible."
        } %>
-    <% CompletionKit::Calibration::VERDICTS.each do |verdict| %>
-      <%= button_to run_response_calibrations_path(run, response_row, metric_id: metric.id, verdict: verdict),
+    <% CompletionKit::Agreement::VERDICTS.each do |verdict| %>
+      <%= button_to run_response_agreements_path(run, response_row, metric_id: metric.id, verdict: verdict),
               method: :post,
               form: { data: { turbo: "true" } },
-              class: "ck-calibration__pill ck-calibration__pill--#{verdict}#{' is-active' if verdict == active_verdict}",
+              class: "ck-agreement__pill ck-agreement__pill--#{verdict}#{' is-active' if verdict == active_verdict}",
               "aria-pressed": (verdict == active_verdict).to_s,
               title: verdict_hints[verdict] do %>
         <%= heroicon_tag verdict_icons[verdict], variant: :outline, size: 14, "aria-hidden": "true" %>
@@ -38,26 +38,26 @@
   </div>
 
   <% if error.present? %>
-    <p class="ck-calibration__error" role="alert"><%= error %></p>
+    <p class="ck-agreement__error" role="alert"><%= error %></p>
   <% end %>
 
-  <% if other_calibrations.any? %>
-    <details class="ck-calibration__others">
-      <summary class="ck-calibration__others-summary">
+  <% if other_agreements.any? %>
+    <details class="ck-agreement__others">
+      <summary class="ck-agreement__others-summary">
         <%= heroicon_tag "chevron-right", variant: :outline, size: 14, "aria-hidden": "true" %>
-        <span>What others said (<%= other_calibrations.size %>)</span>
+        <span>What others said (<%= other_agreements.size %>)</span>
       </summary>
-      <ul class="ck-calibration__others-list">
-        <% other_calibrations.each do |other| %>
-          <li class="ck-calibration__others-item ck-calibration__others-item--<%= other.verdict %>">
-            <div class="ck-calibration__others-row">
-              <span class="ck-calibration__others-verdict">
+      <ul class="ck-agreement__others-list">
+        <% other_agreements.each do |other| %>
+          <li class="ck-agreement__others-item ck-agreement__others-item--<%= other.verdict %>">
+            <div class="ck-agreement__others-row">
+              <span class="ck-agreement__others-verdict">
                 <%= heroicon_tag verdict_icons[other.verdict], variant: :outline, size: 14, "aria-hidden": "true" %>
                 <%= other.verdict %>
               </span>
-              <span class="ck-calibration__others-by"><%= other.created_by %></span>
+              <span class="ck-agreement__others-by"><%= other.created_by %></span>
               <% if other.corrected_score %>
-                <span class="ck-calibration__others-stars" aria-label="<%= pluralize(other.corrected_score.to_i, 'star') %>" title="<%= pluralize(other.corrected_score.to_i, 'star') %>">
+                <span class="ck-agreement__others-stars" aria-label="<%= pluralize(other.corrected_score.to_i, 'star') %>" title="<%= pluralize(other.corrected_score.to_i, 'star') %>">
                   <% 5.times do |i| %>
                     <svg viewBox="0 0 24 24" width="12" height="12" stroke-width="1.75" class="ck-star <%= i < other.corrected_score.to_i ? "ck-star--filled" : "ck-star--empty" %>" aria-hidden="true"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
                   <% end %>
@@ -65,7 +65,7 @@
               <% end %>
             </div>
             <% if other.note.to_s.present? %>
-              <p class="ck-calibration__others-note">"<%= other.note.to_s.truncate(140) %>"</p>
+              <p class="ck-agreement__others-note">"<%= other.note.to_s.truncate(140) %>"</p>
             <% end %>
           </li>
         <% end %>
@@ -74,10 +74,10 @@
   <% end %>
 
   <% if active_verdict == "disagree" %>
-    <% existing_score = (calibration&.corrected_score || review&.ai_score)&.round %>
-    <%= form_with url: run_response_calibrations_path(run, response_row),
+    <% existing_score = (agreement&.corrected_score || review&.ai_score)&.round %>
+    <%= form_with url: run_response_agreements_path(run, response_row),
                   method: :post, local: false,
-                  class: "ck-calibration__detail" do |f| %>
+                  class: "ck-agreement__detail" do |f| %>
       <%= hidden_field_tag :metric_id, metric.id %>
       <%= hidden_field_tag :verdict, "disagree" %>
       <p class="ck-label">What should the score have been?</p>
@@ -93,16 +93,16 @@
           <% end %>
         </div>
       </fieldset>
-      <textarea name="note" rows="2" class="ck-input ck-input--area" placeholder="Why? (optional)"><%= calibration&.note %></textarea>
+      <textarea name="note" rows="2" class="ck-input ck-input--area" placeholder="Why? (optional)"><%= agreement&.note %></textarea>
       <%= f.submit "Save", class: "#{ck_button_classes(:dark)}#{' ck-button--just-saved' if local_assigns[:just_saved]}" %>
     <% end %>
   <% elsif active_verdict == "borderline" %>
-    <%= form_with url: run_response_calibrations_path(run, response_row),
+    <%= form_with url: run_response_agreements_path(run, response_row),
                   method: :post, local: false,
-                  class: "ck-calibration__detail" do |f| %>
+                  class: "ck-agreement__detail" do |f| %>
       <%= hidden_field_tag :metric_id, metric.id %>
       <%= hidden_field_tag :verdict, "borderline" %>
-      <textarea name="note" rows="2" class="ck-input ck-input--area" placeholder="What made this borderline? (optional)"><%= calibration&.note %></textarea>
+      <textarea name="note" rows="2" class="ck-input ck-input--area" placeholder="What made this borderline? (optional)"><%= agreement&.note %></textarea>
       <%= f.submit "Save", class: "#{ck_button_classes(:dark)}#{' ck-button--just-saved' if local_assigns[:just_saved]}" %>
     <% end %>
   <% end %>

data/app/views/completion_kit/{calibrations → agreements}/_trust_panel.html.erb

@@ -4,7 +4,7 @@
 <% current_metric_version = metric && CompletionKit::MetricVersion.current.find_by(metric_id: metric.id) %>
 <% target_response = if (stats.sample_size.zero? || stats.counter_only?) && metric && current_metric_version
      created_by = CompletionKit.config.username.presence || "operator"
-     verdicted_ids = CompletionKit::Calibration.where(metric_id: metric.id, created_by: created_by, metric_version_id: current_metric_version.id).pluck(:response_id)
+     verdicted_ids = CompletionKit::Agreement.where(metric_id: metric.id, created_by: created_by, metric_version_id: current_metric_version.id).pluck(:response_id)
      CompletionKit::Response.joins(:reviews)
        .where(reviews: { metric_id: metric.id, metric_version_id: current_metric_version.id })
        .where.not(reviews: { ai_score: nil })
@@ -12,29 +12,26 @@
        .order(created_at: :desc).first
    end %>
 <% prior_version_verdicts = if stats.sample_size.zero? && metric && current_metric_version
-     CompletionKit::Calibration.where(metric_id: metric.id).where.not(metric_version_id: current_metric_version.id).count
+     CompletionKit::Agreement.where(metric_id: metric.id).where.not(metric_version_id: current_metric_version.id).count
    else
      0
    end %>
 
-<p class="ck-trust-line ck-trust-line--<%= stats.gate %>">
+<p id="agreement" class="ck-trust-line ck-trust-line--<%= stats.gate %>">
   <% if stats.sample_size.zero? %>
     <span class="ck-trust-line__lead">Not measured yet.</span>
-    <span class="ck-trust-line__hint"><%= current_metric_version ? "#{current_metric_version.version_label} needs" : "Needs" %> <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> human reviews of the judge's scores.</span>
+    <span class="ck-trust-line__hint"><%= current_metric_version ? "#{current_metric_version.version_label} needs" : "Needs" %> <%= CompletionKit::MetricAgreementStats::PROVISIONAL_MIN %> human reviews of the judge's scores.</span>
     <% if target_response %>
       <%= link_to "Review a judge's score →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-cal-link" %>
     <% end %>
   <% elsif stats.counter_only? %>
-    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Human reviews</span> <strong><%= stats.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %></strong></span>
+    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Human reviews</span> <strong><%= stats.sample_size %> / <%= CompletionKit::MetricAgreementStats::PROVISIONAL_MIN %></strong></span>
     <% if stats.short_to_target > 0 %><span class="ck-trust-line__hint"><%= stats.short_to_target %> more to report a rate</span><% end %>
     <% if target_response %>
       <%= link_to "Review another score →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-cal-link" %>
     <% end %>
   <% else %>
-    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Agreement</span> <strong class="ck-trust-line__figure">~<%= (stats.agreement_point * 100).round %>%</strong></span>
-    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Margin</span> ±<%= (stats.margin * 100).round %> pt</span>
-    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Read</span> <%= stats.firm? ? "settled" : "early" %></span>
-    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Sample</span> <%= stats.sample_size %></span>
+    <span class="ck-cal-stat"><span class="ck-cal-stat__label">Agrees with you</span> <strong class="ck-trust-line__figure">~<%= (stats.agreement_point * 100).round %>%</strong> of <%= stats.sample_size %> reviews</span>
     <% if stats.borderline_rate && stats.borderline_rate > 0 %>
       <% level = stats.borderline_rate > 0.30 ? "danger" : stats.borderline_rate > 0.15 ? "warning" : "ok" %>
       <span class="ck-cal-stat"><span class="ck-cal-stat__label">Unclear</span> <span class="ck-trust-line__borderline ck-trust-line__borderline--<%= level %>"><%= (stats.borderline_rate * 100).round %>%</span></span>