completion-kit 0.5.41 → 0.5.43

data/app/controllers/completion_kit/metrics_controller.rb

@@ -1,21 +1,50 @@
 module CompletionKit
   class MetricsController < ApplicationController
     include CompletionKit::TagFiltering
-    before_action :set_metric, only: [:show, :edit, :update, :destroy, :add_few_shot, :publish_draft, :suggest_variants, :dismiss_suggestion]
+    before_action :set_metric, only: [:show, :edit, :update, :destroy, :add_few_shot, :remove_few_shot, :publish_draft, :suggest_variants, :dismiss_suggestion]
 
     def index
       @metrics = apply_tag_filter(Metric.includes(:metric_groups, :tags).order(:name))
+      @available_starters = StarterMetrics.available
+    end
+
+    def starter_preview
+      @starter = StarterMetrics.find(params[:key])
+      return redirect_to(metrics_path, alert: "Unknown starter metric.") unless @starter
+    end
+
+    def adopt_starter
+      starter = StarterMetrics.find(params[:key])
+      return redirect_to(metrics_path, alert: "Unknown starter metric.") unless starter
+      if Metric.exists?(name: starter.name)
+        return redirect_to(metrics_path, alert: "A metric named \"#{starter.name}\" already exists.")
+      end
+      metric = Metric.create!(
+        name: starter.name,
+        instruction: starter.instruction,
+        rubric_bands: starter.rubric_bands
+      )
+      redirect_to metric_path(metric), notice: "Added the \"#{starter.name}\" starter. Tweak any band before you run a judge against it."
+    end
+
+    def dismiss_starter
+      starter = StarterMetrics.find(params[:key])
+      return redirect_to(metrics_path, alert: "Unknown starter metric.") unless starter
+      StarterMetricDismissal.find_or_create_by(starter_key: starter.key)
+      redirect_to metrics_path, notice: "Dismissed \"#{starter.name}\". It won't appear here again."
     end
 
     def show
+      @published_judge_version = JudgeVersion.ensure_current_for(@metric)
       @disagreements = Calibration.where(metric_id: @metric.id, verdict: "disagree")
-                                  .includes(response: [:reviews, :run])
+                                  .includes(:judge_version, response: [:reviews, :run])
                                   .order(created_at: :desc)
                                   .limit(50)
       @edit_draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "edit").order(created_at: :desc).first
-      @published_judge_version = JudgeVersion.published.where(metric_id: @metric.id, current: true).first
       @suggestion_draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "suggestion").order(created_at: :desc).first
-      @improve_disagreement_count = @disagreements.size
+      @improve_disagreement_count = Calibration.where(metric_id: @metric.id, verdict: "disagree",
+                                                      judge_version_id: @published_judge_version.id).count
+      @versions = JudgeVersion.where(metric_id: @metric.id).order(version_number: :desc).to_a
     end
 
     def new
@@ -23,6 +52,9 @@ module CompletionKit
     end
 
     def edit
+      @suggestion_draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "suggestion").order(created_at: :desc).first
+      @edit_draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "edit").order(created_at: :desc).first
+      @published_judge_version = JudgeVersion.published.where(metric_id: @metric.id, current: true).first
     end
 
     def create
@@ -49,9 +81,10 @@ module CompletionKit
     end
 
     def suggest_variants
+      target = params[:back_to] == "edit" ? edit_metric_path(@metric) : metric_path(@metric)
       disagreement_count = Calibration.where(metric_id: @metric.id, verdict: "disagree").count
       if disagreement_count.zero?
-        redirect_to metric_path(@metric), alert: "Mark at least one row as Disagree before asking the model to suggest a change."
+        redirect_to target, alert: "Mark at least one row as Disagree before asking the model to suggest a change."
         return
       end
 
@@ -60,38 +93,36 @@ module CompletionKit
       generator = JudgeVariantGenerator.new(@metric, count: 1)
       variants = generator.call
       if variants.empty?
-        redirect_to metric_path(@metric), alert: "The model returned no usable variants. Try again with a different model."
+        redirect_to target, alert: "The model returned no usable variants. Try again with a different model."
         return
       end
       generator.persist!(variants)
-      redirect_to metric_path(@metric), notice: "Drafted a new version. Review it below."
+      redirect_to target, notice: "Drafted a new version. Review it below."
     end
 
     def dismiss_suggestion
-      draft = JudgeVersion.drafts.where(metric_id: @metric.id, source: "suggestion").find_by(id: params[:draft_id])
+      draft = JudgeVersion.drafts.where(metric_id: @metric.id).find_by(id: params[:draft_id])
       draft&.destroy
-      redirect_to metric_path(@metric), notice: "Dismissed."
+      target = params[:back_to] == "edit" ? edit_metric_path(@metric) : metric_path(@metric)
+      redirect_to target, notice: "Dismissed."
     end
 
     def publish_draft
-      scope = JudgeVersion.drafts.where(metric_id: @metric.id)
-      draft = params[:draft_id].present? ? scope.find_by(id: params[:draft_id]) : scope.order(created_at: :desc).first
-
-      if draft.nil?
-        redirect_to metric_path(@metric), alert: "No draft to publish."
+      scope = JudgeVersion.where(metric_id: @metric.id)
+      version = if params[:draft_id].present?
+                  scope.find_by(id: params[:draft_id])
+                else
+                  JudgeVersion.drafts.where(metric_id: @metric.id).order(created_at: :desc).first
+                end
+
+      if version.nil?
+        redirect_to metric_path(@metric), alert: "No version to publish."
         return
       end
 
-      JudgeVersion.transaction do
-        JudgeVersion.where(metric_id: @metric.id, state: "published").update_all(current: false)
-        draft.update!(state: "published", current: true)
-        @metric.update_columns(
-          instruction: draft.instruction,
-          rubric_bands: Array(draft.rubric_bands).to_json
-        )
-      end
-
-      redirect_to metric_path(@metric), notice: "This judge version is now live."
+      version.publish!
+      redirect_to metric_path(@metric),
+                  notice: "#{@metric.name} #{version.version_label} is now the published version."
     end
 
     def add_few_shot
@@ -112,6 +143,13 @@ module CompletionKit
       redirect_to metric_path(@metric), notice: "Got it. The judge will remember this next time it grades."
     end
 
+    def remove_few_shot
+      cal_id = params[:calibration_id].to_i
+      remaining = Array(@metric.few_shot_examples).reject { |fs| fs["calibration_id"].to_i == cal_id }
+      @metric.update!(few_shot_examples: remaining)
+      redirect_to metric_path(@metric), notice: "Forgotten. The judge won't see this case next time."
+    end
+
     private
 
     def set_metric

data/app/jobs/completion_kit/judge_review_job.rb

@@ -57,6 +57,7 @@ module CompletionKit
         run.prompt&.template,
         criteria: metric.instruction.to_s,
         rubric_text: metric.display_rubric_text,
+        human_examples: few_shot_payload(metric),
         input_data: response.input_data
       )
 
@@ -119,5 +120,15 @@ module CompletionKit
       response = Response.find_by(id: response_id)
       RunCompletionCheckJob.perform_later(response.run_id) if response
     end
+
+    def few_shot_payload(metric)
+      Array(metric.few_shot_examples).map do |fs|
+        {
+          human_score: fs["human_score"],
+          response_text: fs["response"].to_s,
+          human_note: fs["human_note"].to_s
+        }
+      end
+    end
   end
 end

data/app/models/completion_kit/judge_version.rb

@@ -7,8 +7,11 @@ module CompletionKit
 
     serialize :rubric_bands, coder: JSON
 
+    before_validation :assign_version_number, on: :create
+
     validates :metric_id, presence: true
     validates :state, inclusion: { in: STATES }
+    validates :version_number, presence: true, uniqueness: { scope: :metric_id }
 
     scope :current, -> { where(current: true) }
     scope :published, -> { where(state: "published") }
@@ -20,7 +23,8 @@ module CompletionKit
         instruction: metric.instruction,
         rubric_bands: metric.rubric_bands,
         current: true,
-        state: "published"
+        state: "published",
+        published_at: Time.current
       )
     end
 
@@ -32,17 +36,44 @@ module CompletionKit
       state == "published"
     end
 
+    def version_label
+      "v#{version_number}"
+    end
+
+    def publish!
+      JudgeVersion.transaction do
+        self.class.where(metric_id: metric_id).where.not(id: id).update_all(current: false)
+        reload
+        update!(state: "published", current: true, published_at: published_at || Time.current)
+        metric.update_columns(
+          instruction: instruction,
+          rubric_bands: Array(rubric_bands).to_json
+        )
+      end
+      self
+    end
+
     def as_json(options = {})
       {
         id: id,
         metric_id: metric_id,
+        version_number: version_number,
         instruction: instruction,
         rubric_bands: rubric_bands,
         current: current,
         state: state,
         source: source,
+        published_at: published_at,
         created_at: created_at
       }
     end
+
+    private
+
+    def assign_version_number
+      return if version_number.present?
+      max = self.class.where(metric_id: metric_id).maximum(:version_number).to_i
+      self.version_number = max + 1
+    end
   end
 end

data/app/models/completion_kit/starter_metric_dismissal.rb

@@ -0,0 +1,5 @@
+module CompletionKit
+  class StarterMetricDismissal < ApplicationRecord
+    validates :starter_key, presence: true, uniqueness: true
+  end
+end

data/app/services/completion_kit/judge_variant_generator.rb

@@ -86,7 +86,7 @@ module CompletionKit
       sections << "REASONING: <one short sentence: what changes and why>"
       sections << "INSTRUCTION:"
       sections << "<the rewritten instruction>"
-      sections << "RUBRIC:                  # optional — omit this block if the rubric is unchanged"
+      sections << "RUBRIC:                  # optional. Omit this block if the rubric is unchanged."
       sections << "5: <description for 5 stars>"
       sections << "4: <description for 4 stars>"
       sections << "3: <description for 3 stars>"
@@ -133,11 +133,13 @@ module CompletionKit
     end
 
     def calibrations_for(metric, verdict:, limit:)
-      Calibration.where(metric_id: metric.id, verdict: verdict)
-                 .includes(response: :reviews)
-                 .order(created_at: :desc)
-                 .limit(limit)
-                 .map do |cal|
+      scope = Calibration.where(metric_id: metric.id, verdict: verdict)
+      current_version = JudgeVersion.current.find_by(metric_id: metric.id)
+      scope = scope.where(judge_version_id: current_version.id) if current_version
+      scope.includes(response: :reviews)
+           .order(created_at: :desc)
+           .limit(limit)
+           .map do |cal|
         review = cal.response.reviews.find { |r| r.metric_id == metric.id }
         {
           input: cal.response.input_data,

data/app/services/completion_kit/metric_calibration_stats.rb

@@ -31,18 +31,30 @@ module CompletionKit
       end
     end
 
-    def self.for(metric, judge_version: nil)
-      new(metric: metric, judge_version: judge_version).call
+    CURRENT = :current
+
+    def self.for(metric, judge_version: CURRENT)
+      resolved = case judge_version
+                 when CURRENT then JudgeVersion.current.find_by(metric_id: metric.id)
+                 when nil then nil
+                 else judge_version
+                 end
+      new(metric: metric, judge_version: resolved, all_versions: judge_version.nil?).call
     end
 
-    def initialize(metric:, judge_version: nil)
+    def initialize(metric:, judge_version: nil, all_versions: false)
       @metric = metric
       @judge_version = judge_version
+      @all_versions = all_versions
     end
 
     def call
       scope = Calibration.where(metric_id: @metric.id)
-      scope = scope.where(judge_version_id: @judge_version.id) if @judge_version
+      if @judge_version
+        scope = scope.where(judge_version_id: @judge_version.id)
+      elsif !@all_versions
+        scope = scope.none
+      end
 
       verdicts = scope.pluck(:verdict, :corrected_score, :response_id)
       n = verdicts.length

data/app/services/completion_kit/starter_metrics.rb

@@ -0,0 +1,94 @@
+module CompletionKit
+  module StarterMetrics
+    Starter = Struct.new(:key, :name, :description, :catches, :instruction, :rubric_bands, keyword_init: true)
+
+    ALL = [
+      Starter.new(
+        key: "correctness",
+        name: "Correctness",
+        description: "Is the output factually right and free of made-up information?",
+        catches: "Hallucinations, wrong facts, subtle distortions. The most universally-asked question about an LLM's output.",
+        instruction: "Is the output factually right and free of made-up information? Penalise hallucinations and subtle factual distortions; reward outputs whose every claim checks out.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Every fact in the output checks out." },
+          { "stars" => 4, "description" => "Right in substance; minor imprecision or omission." },
+          { "stars" => 3, "description" => "Mostly right, one or two facts are off." },
+          { "stars" => 2, "description" => "Mostly wrong with a few right details." },
+          { "stars" => 1, "description" => "Wrong, misleading, or contains fabricated facts." }
+        ]
+      ),
+      Starter.new(
+        key: "instruction_following",
+        name: "Instruction following",
+        description: "Did the model do everything that was asked?",
+        catches: "The response is factually right but ignores \"answer in two sentences\", \"use bullet points\", \"do not include X\". Distinct from Correctness — a response can be right and still fail this.",
+        instruction: "Did the model do every concrete thing the prompt asked for? Score against the explicit requirements in the prompt (format constraints, count limits, exclusions, audience cues). Factual accuracy is a different dimension — score that elsewhere.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Followed every requirement in the prompt exactly." },
+          { "stars" => 4, "description" => "Followed every requirement with a small slip." },
+          { "stars" => 3, "description" => "Did the main thing, missed at least one explicit requirement." },
+          { "stars" => 2, "description" => "Did some of what was asked, missed the main requirement." },
+          { "stars" => 1, "description" => "Ignored the instructions or did something different." }
+        ]
+      ),
+      Starter.new(
+        key: "format_compliance",
+        name: "Format compliance",
+        description: "Does the output follow the required structure?",
+        catches: "Invalid JSON, missing schema fields, extra prose around a structured response, wrong casing on keys. Critical for any LLM wired into an API.",
+        instruction: "Does the output match the format the prompt asked for — JSON shape, schema, keys, casing, no stray prose? Score on whether a downstream parser would accept it without massaging.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Exact spec, ready to consume programmatically." },
+          { "stars" => 4, "description" => "Spec-compliant with one cosmetic issue." },
+          { "stars" => 3, "description" => "Right shape, minor deviations (extra commentary, casing, ordering)." },
+          { "stars" => 2, "description" => "Right format with substantive deviations (missing required fields, wrong types)." },
+          { "stars" => 1, "description" => "Wrong format or unparseable." }
+        ]
+      ),
+      Starter.new(
+        key: "tone",
+        name: "Tone",
+        description: "Does the voice fit the audience the prompt asked for?",
+        catches: "Rude, robotic, off-brand, too casual, too formal. The dimension hardest to eyeball at scale and the one most user-facing surfaces care about.",
+        instruction: "Does the voice match the audience and brand the prompt called for? Reward outputs that sound like the persona the prompt asked for. Penalise rude, robotic, off-brand, or wrong-register replies.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Sounds like the brand or persona the prompt asked for." },
+          { "stars" => 4, "description" => "Right tone with a slip or two." },
+          { "stars" => 3, "description" => "Acceptable, generic, no personality." },
+          { "stars" => 2, "description" => "Mismatched tone; sounds like a different audience." },
+          { "stars" => 1, "description" => "Off-tone in a way a user would notice (rude, condescending, jarring)." }
+        ]
+      ),
+      Starter.new(
+        key: "conciseness",
+        name: "Conciseness",
+        description: "Is it the right length — no padding, no missing detail?",
+        catches: "Rambling responses, repetitive caveats, over-hedging. LLMs default to verbose. Conciseness is the dimension where users most often see scores move after tuning.",
+        instruction: "Is the output the right length for the task — no padding, no missing detail, no hedging filler? Penalise rambling, repetition, over-caveating, and unnecessary preamble. Penalise too-short outputs that drop information.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Exactly as long as the task needs, no more, no less." },
+          { "stars" => 4, "description" => "Right length with a small redundancy." },
+          { "stars" => 3, "description" => "Acceptable; trims could happen or detail could be added." },
+          { "stars" => 2, "description" => "Noticeable filler or visible gaps." },
+          { "stars" => 1, "description" => "Padded, repetitive, or so short it loses information." }
+        ]
+      )
+    ].freeze
+
+    module_function
+
+    def find(key)
+      ALL.find { |s| s.key == key }
+    end
+
+    def available
+      adopted_names = Metric.where(name: ALL.map(&:name)).pluck(:name).to_set
+      dismissed_keys = StarterMetricDismissal.pluck(:starter_key).to_set
+      ALL.reject { |s| adopted_names.include?(s.name) || dismissed_keys.include?(s.key) }
+    end
+
+    def adopted?(starter)
+      Metric.exists?(name: starter.name)
+    end
+  end
+end

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -187,7 +187,7 @@
       </div>
       <%= render "completion_kit/api_reference/resource_list", title: "Your datasets",
             items: datasets.map { |d|
-              { name: d.name, meta: pluralize([d.csv_data.to_s.lines.count - 1, 0].max, "row"),
+              { name: d.name, meta: pluralize([d.csv_data.to_s.lines.count - 1, 0].max, "entry"),
                 url: "#{base_url}/api/v1/datasets/#{d.id}", dom_id: "dataset_ep_#{d.id}" }
             } %>
     </div>

data/app/views/completion_kit/calibrations/_buttons.html.erb

@@ -3,14 +3,19 @@
   <% pending_verdict = local_assigns[:pending_verdict] %>
   <% active_verdict = pending_verdict || current_verdict %>
   <% error = local_assigns[:error] %>
-  <% verdict_count = CompletionKit::Calibration.where(response_id: response_row.id, metric_id: metric.id).count %>
+  <% me = CompletionKit.config.username.presence || "operator" %>
+  <% other_calibrations = CompletionKit::Calibration
+        .where(response_id: response_row.id, metric_id: metric.id)
+        .where.not(created_by: me)
+        .order(created_at: :asc).to_a %>
+  <% verdict_icons = { "agree" => "hand-thumb-up", "disagree" => "hand-thumb-down", "borderline" => "scale" } %>
   <p class="ck-calibration__prompt">
     <span class="ck-calibration__label">Your verdict</span>
-    <% if verdict_count > 0 %>
-      <span class="ck-calibration__count"><%= pluralize(verdict_count, "verdict") %> on this score · <%= link_to "trust level →", metric_path(metric), class: "ck-link" %></span>
-    <% else %>
-      <span class="ck-calibration__hint">Tell us what you think — was the score right? Verdicts roll up into the metric's <%= link_to "trust level", metric_path(metric), class: "ck-link" %>.</span>
+    <% if other_calibrations.any? %>
+      <span class="ck-calibration__meta"><%= pluralize(other_calibrations.size, "other verdict") %> on this score</span>
+      <span class="ck-calibration__sep">·</span>
     <% end %>
+    <%= link_to metric_path(metric), class: "ck-calibration__meta-link" do %><%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true" %>Calibration →<% end %>
   </p>
   <div class="ck-calibration__buttons">
     <% verdict_icons = { "agree" => "hand-thumb-up", "disagree" => "hand-thumb-down", "borderline" => "scale" } %>
@@ -36,6 +41,38 @@
     <p class="ck-calibration__error" role="alert"><%= error %></p>
   <% end %>
 
+  <% if other_calibrations.any? %>
+    <details class="ck-calibration__others">
+      <summary class="ck-calibration__others-summary">
+        <%= heroicon_tag "chevron-right", variant: :outline, size: 14, "aria-hidden": "true" %>
+        <span>What others said (<%= other_calibrations.size %>)</span>
+      </summary>
+      <ul class="ck-calibration__others-list">
+        <% other_calibrations.each do |other| %>
+          <li class="ck-calibration__others-item ck-calibration__others-item--<%= other.verdict %>">
+            <div class="ck-calibration__others-row">
+              <span class="ck-calibration__others-verdict">
+                <%= heroicon_tag verdict_icons[other.verdict], variant: :outline, size: 14, "aria-hidden": "true" %>
+                <%= other.verdict %>
+              </span>
+              <span class="ck-calibration__others-by"><%= other.created_by %></span>
+              <% if other.corrected_score %>
+                <span class="ck-calibration__others-stars" aria-label="<%= pluralize(other.corrected_score.to_i, 'star') %>" title="<%= pluralize(other.corrected_score.to_i, 'star') %>">
+                  <% 5.times do |i| %>
+                    <svg viewBox="0 0 24 24" width="12" height="12" stroke-width="1.75" class="ck-star <%= i < other.corrected_score.to_i ? "ck-star--filled" : "ck-star--empty" %>" aria-hidden="true"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
+                  <% end %>
+                </span>
+              <% end %>
+            </div>
+            <% if other.note.to_s.present? %>
+              <p class="ck-calibration__others-note">"<%= other.note.to_s.truncate(140) %>"</p>
+            <% end %>
+          </li>
+        <% end %>
+      </ul>
+    </details>
+  <% end %>
+
   <% if active_verdict == "disagree" %>
     <% existing_score = (calibration&.corrected_score || review&.ai_score)&.round %>
     <%= form_with url: run_response_calibrations_path(run, response_row),
@@ -51,7 +88,7 @@
             <% radio_id = "ck-star-#{response_row.id}-#{metric.id}-#{n}" %>
             <input type="radio" name="corrected_score" id="<%= radio_id %>" value="<%= n %>" <%= "checked" if existing_score == n %> required>
             <label for="<%= radio_id %>" title="<%= pluralize(n, 'star') %>" aria-label="<%= pluralize(n, 'star') %>">
-              <svg viewBox="0 0 24 24" width="28" height="28" stroke-width="1.5" aria-hidden="true"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
+              <svg viewBox="0 0 24 24" stroke-width="1.5" aria-hidden="true"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
             </label>
           <% end %>
         </div>

data/app/views/completion_kit/calibrations/_trust_panel.html.erb

@@ -1,31 +1,30 @@
 <% stats = local_assigns[:stats] %>
-<div class="ck-trust-panel ck-trust-panel--<%= stats.gate %>">
-  <p class="ck-trust-panel__label" title="How often this metric's scores match the humans who reviewed them.">Trust level</p>
-  <% if stats.counter_only? %>
-    <div class="ck-trust-panel__body">
-      <span class="ck-trust-panel__counter"><%= stats.sample_size %><span class="ck-trust-panel__counter-of">/ <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %></span></span>
-      <span class="ck-trust-panel__hint">verdicts so far<% if stats.short_to_target > 0 %> · <%= stats.short_to_target %> more before we can score the judge<% end %></span>
-    </div>
+<% metric = local_assigns[:metric] %>
+<% anchor = metric&.name&.parameterize %>
+<% target_response = if (stats.sample_size.zero? || stats.counter_only?) && metric
+     created_by = CompletionKit.config.username.presence || "operator"
+     verdicted_ids = CompletionKit::Calibration.where(metric_id: metric.id, created_by: created_by).pluck(:response_id)
+     CompletionKit::Response.joins(:reviews)
+       .where(reviews: { metric_id: metric.id })
+       .where.not(reviews: { ai_score: nil })
+       .where.not(id: verdicted_ids)
+       .order(created_at: :desc).first
+   end %>
+
+<p class="ck-trust-line ck-trust-line--<%= stats.gate %>">
+  <span class="ck-trust-line__label"><%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true" %>Calibration</span>
+  <% if stats.sample_size.zero? %>
+    <span class="ck-trust-line__state">Not measured yet.</span>
+    <span class="ck-trust-line__hint">Needs <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts on the judge's scores.<% if target_response %>
+      <%= link_to "Give a verdict →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-trust-line__link" %>
+    <% else %> Run this metric on a dataset, then give a verdict.<% end %></span>
+  <% elsif stats.counter_only? %>
+    <span class="ck-trust-line__counter"><%= stats.sample_size %>/<%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %></span>
+    <span class="ck-trust-line__hint"><%= pluralize(stats.sample_size, "verdict") %><% if stats.short_to_target > 0 %> · <%= stats.short_to_target %> more before this can be measured<% end %><% if target_response %> · <%= link_to "Give another verdict →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-trust-line__link" %><% end %></span>
   <% else %>
-    <div class="ck-trust-panel__body">
-      <span class="ck-trust-panel__score"
-            title="Roughly how often the judge and the humans agreed.">~<%= (stats.agreement_point * 100).round %><span class="ck-trust-panel__score-pct">%</span></span>
-      <span class="ck-trust-panel__margin"
-            title="The range we're confident the true rate sits in, given how few verdicts we have so far.">±<%= (stats.margin * 100).round %> pt</span>
-      <span class="ck-trust-panel__gate"
-            title="<%= stats.firm? ? 'Enough verdicts for a reliable read.' : 'Early read. Keep giving verdicts to tighten the margin.' %>"><%= stats.firm? ? "settled" : "early read" %></span>
-    </div>
-    <div class="ck-trust-panel__details">
-      <span><%= pluralize(stats.sample_size, "verdict") %></span>
-      <% if stats.borderline_rate && stats.borderline_rate > 0 %>
-        <% level = if stats.borderline_rate > 0.30 then "danger"
-                   elsif stats.borderline_rate > 0.15 then "warning"
-                   else "ok" end %>
-        <span class="ck-trust-panel__borderline ck-trust-panel__borderline--<%= level %>"
-              title="<%= level == 'ok' ? 'Some reviewers said the rubric was unclear here.' : 'A lot of reviewers say the rubric is unclear here. Consider splitting the metric or rewriting the rubric.' %>">
-          <%= (stats.borderline_rate * 100).round %>% said "unclear"
-        </span>
-      <% end %>
-    </div>
+    <span class="ck-trust-line__score" title="Roughly how often the judge and the humans agreed.">~<%= (stats.agreement_point * 100).round %>%</span>
+    <span class="ck-trust-line__margin" title="The range we're confident the true rate sits in.">±<%= (stats.margin * 100).round %> pt</span>
+    <span class="ck-trust-line__gate" title="<%= stats.firm? ? 'Enough verdicts for a reliable read.' : 'Early read. Keep giving verdicts.' %>"><%= stats.firm? ? "settled" : "early" %></span>
+    <span class="ck-trust-line__hint"><%= pluralize(stats.sample_size, "verdict") %><% if stats.borderline_rate && stats.borderline_rate > 0 %><% level = stats.borderline_rate > 0.30 ? "danger" : stats.borderline_rate > 0.15 ? "warning" : "ok" %> · <span class="ck-trust-line__borderline ck-trust-line__borderline--<%= level %>" title="<%= level == 'ok' ? '' : 'Reviewers said the rubric was unclear here.' %>"><%= (stats.borderline_rate * 100).round %>% unclear</span><% end %></span>
   <% end %>
-</div>
+</p>