completion-kit 0.5.40 → 0.5.42

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8875fa22de9c8626e401706818b271f3cb26bffa7faae1f583b29503228cead
-  data.tar.gz: 5bf39fad883b2eed2f505b11403ab60fddf9efc5735073ff61c290d53f59a36d
+  metadata.gz: cb4e02cd47c184c237c34642f1ece8c44bef6a2e79cfeeb3da913a536940216e
+  data.tar.gz: 516720138ec5dd2eeecc2d28cd9e7830c8c2ab2b4a183733afe44a939ca4d9af
 SHA512:
-  metadata.gz: a8f2a7f14235c1214b567b891defaf523a645a21a2409ed81df964893a260cccb1fc9bf63903794f952c91bc9c91f3c1e3850db751a08ce0edc49b360ad9642d
-  data.tar.gz: e148b500e498a00dc370bf203fea3e2618b9f1c8fccd1dc5f220ae77a4988a8934198ad32bbf25a29bc84f948c687fab9a0ae16b9c03fe575fe9fca4cf98a0ea
+  metadata.gz: 0f3403ecf234b58f659f49406e0264384a5ccbda7206e3673c3b4d96704658b1efe5c2e74943fd6110477a8acffeca3600af655980e76a139319e6dc4376779f
+  data.tar.gz: da16ea0344f7a00f8366faabe4cd9d53ef487fc67a06f415b7772ff3631fcb3edca6f30be9a59dd3e4cc9e40795cdd4977028c5fd880ab9d2d49d3563babe5aa

data/app/assets/stylesheets/completion_kit/application.css

@@ -5413,9 +5413,11 @@ a.tag-mark {
 }
 
 .ck-metrics-table__trust {
+  margin: 4px 0 0;
   font-family: var(--ck-mono);
-  font-size: 0.78rem;
+  font-size: 0.72rem;
   letter-spacing: 0.03em;
+  color: var(--ck-dim);
 }
 .ck-metrics-table__trust-rate {
   font-weight: 600;
@@ -5485,3 +5487,122 @@ a.tag-mark {
 @keyframes ck-saved-flash {
   0% { background: var(--ck-success); border-color: var(--ck-success); }
 }
+
+.ck-disagreement-list {
+  list-style: none;
+  padding: 0;
+  margin: 12px 0 0;
+  display: flex;
+  flex-direction: column;
+  gap: 12px;
+}
+.ck-disagreement {
+  padding: 14px;
+  background: var(--ck-surface-soft);
+  border: 1px solid var(--ck-line);
+  border-radius: 6px;
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+}
+.ck-disagreement__head {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 12px;
+  flex-wrap: wrap;
+}
+.ck-disagreement__scores {
+  display: inline-flex;
+  align-items: center;
+  gap: 8px;
+  flex-wrap: wrap;
+}
+.ck-disagreement__scores-label {
+  font-family: var(--ck-mono);
+  font-size: 0.7rem;
+  letter-spacing: 0.08em;
+  text-transform: uppercase;
+  color: var(--ck-dim);
+}
+.ck-disagreement__scores-arrow {
+  color: var(--ck-dim);
+}
+.ck-disagreement__note {
+  margin: 0;
+  color: var(--ck-text);
+  font-size: 0.92rem;
+  line-height: 1.45;
+}
+.ck-disagreement__source {
+  margin: 0;
+  font-size: 0.78rem;
+}
+
+.ck-starter-row {
+  margin-top: 2rem;
+  padding-top: 1.5rem;
+  border-top: 1px solid var(--ck-line);
+}
+.ck-starter-row--empty-state {
+  margin-top: 0;
+  padding-top: 0;
+  border-top: 0;
+}
+.ck-starter-row .ck-kicker,
+.ck-starter-row .ck-title {
+  margin-bottom: 0.5rem;
+}
+.ck-starter-row .ck-lead,
+.ck-starter-row .ck-meta-copy {
+  margin-bottom: 1.25rem;
+}
+.ck-starter-grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fill, minmax(260px, 1fr));
+  gap: 12px;
+}
+.ck-starter-card {
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+  padding: 14px;
+  background: var(--ck-surface);
+  border: 1px solid var(--ck-line);
+  border-radius: var(--ck-radius-lg);
+  transition: border-color 0.12s, background 0.12s;
+}
+.ck-starter-card:hover {
+  border-color: rgba(6, 182, 212, 0.35);
+  background: var(--ck-surface-hover);
+}
+.ck-starter-card__name {
+  margin: 0;
+  font-size: 1rem;
+  font-weight: 600;
+}
+.ck-starter-card__desc {
+  margin: 0;
+  font-size: 0.85rem;
+  color: var(--ck-muted);
+  line-height: 1.45;
+  flex: 1;
+}
+.ck-starter-card__actions {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 10px;
+  margin-top: 4px;
+}
+.ck-starter-card__dismiss {
+  font-size: 0.75rem;
+  color: var(--ck-dim);
+}
+.ck-starter-card__dismiss:hover,
+.ck-starter-card__dismiss:focus-visible {
+  color: var(--ck-text);
+}
+.ck-actions--right {
+  justify-content: flex-end;
+}

data/app/controllers/completion_kit/metrics_controller.rb

@@ -5,6 +5,33 @@ module CompletionKit
 
     def index
       @metrics = apply_tag_filter(Metric.includes(:metric_groups, :tags).order(:name))
+      @available_starters = StarterMetrics.available
+    end
+
+    def starter_preview
+      @starter = StarterMetrics.find(params[:key])
+      return redirect_to(metrics_path, alert: "Unknown starter metric.") unless @starter
+    end
+
+    def adopt_starter
+      starter = StarterMetrics.find(params[:key])
+      return redirect_to(metrics_path, alert: "Unknown starter metric.") unless starter
+      if Metric.exists?(name: starter.name)
+        return redirect_to(metrics_path, alert: "A metric named \"#{starter.name}\" already exists.")
+      end
+      metric = Metric.create!(
+        name: starter.name,
+        instruction: starter.instruction,
+        rubric_bands: starter.rubric_bands
+      )
+      redirect_to metric_path(metric), notice: "Added the \"#{starter.name}\" starter. Tweak any band before you run a judge against it."
+    end
+
+    def dismiss_starter
+      starter = StarterMetrics.find(params[:key])
+      return redirect_to(metrics_path, alert: "Unknown starter metric.") unless starter
+      StarterMetricDismissal.find_or_create_by(starter_key: starter.key)
+      redirect_to metrics_path, notice: "Dismissed \"#{starter.name}\". It won't appear here again."
     end
 
     def show
@@ -109,7 +136,7 @@ module CompletionKit
         "added_at" => Time.current.utc.iso8601
       }
       @metric.update!(few_shot_examples: examples)
-      redirect_to metric_path(@metric), notice: "Saved as a teaching example. The judge will see it next time it grades."
+      redirect_to metric_path(@metric), notice: "Got it. The judge will remember this next time it grades."
     end
 
     private

data/app/models/completion_kit/starter_metric_dismissal.rb

@@ -0,0 +1,5 @@
+module CompletionKit
+  class StarterMetricDismissal < ApplicationRecord
+    validates :starter_key, presence: true, uniqueness: true
+  end
+end

data/app/services/completion_kit/starter_metrics.rb

@@ -0,0 +1,94 @@
+module CompletionKit
+  module StarterMetrics
+    Starter = Struct.new(:key, :name, :description, :catches, :instruction, :rubric_bands, keyword_init: true)
+
+    ALL = [
+      Starter.new(
+        key: "correctness",
+        name: "Correctness",
+        description: "Is the output factually right and free of made-up information?",
+        catches: "Hallucinations, wrong facts, subtle distortions. The most universally-asked question about an LLM's output.",
+        instruction: "Is the output factually right and free of made-up information? Penalise hallucinations and subtle factual distortions; reward outputs whose every claim checks out.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Every fact in the output checks out." },
+          { "stars" => 4, "description" => "Right in substance; minor imprecision or omission." },
+          { "stars" => 3, "description" => "Mostly right, one or two facts are off." },
+          { "stars" => 2, "description" => "Mostly wrong with a few right details." },
+          { "stars" => 1, "description" => "Wrong, misleading, or contains fabricated facts." }
+        ]
+      ),
+      Starter.new(
+        key: "instruction_following",
+        name: "Instruction following",
+        description: "Did the model do everything that was asked?",
+        catches: "The response is factually right but ignores \"answer in two sentences\", \"use bullet points\", \"do not include X\". Distinct from Correctness — a response can be right and still fail this.",
+        instruction: "Did the model do every concrete thing the prompt asked for? Score against the explicit requirements in the prompt (format constraints, count limits, exclusions, audience cues). Factual accuracy is a different dimension — score that elsewhere.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Followed every requirement in the prompt exactly." },
+          { "stars" => 4, "description" => "Followed every requirement with a small slip." },
+          { "stars" => 3, "description" => "Did the main thing, missed at least one explicit requirement." },
+          { "stars" => 2, "description" => "Did some of what was asked, missed the main requirement." },
+          { "stars" => 1, "description" => "Ignored the instructions or did something different." }
+        ]
+      ),
+      Starter.new(
+        key: "format_compliance",
+        name: "Format compliance",
+        description: "Does the output follow the required structure?",
+        catches: "Invalid JSON, missing schema fields, extra prose around a structured response, wrong casing on keys. Critical for any LLM wired into an API.",
+        instruction: "Does the output match the format the prompt asked for — JSON shape, schema, keys, casing, no stray prose? Score on whether a downstream parser would accept it without massaging.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Exact spec, ready to consume programmatically." },
+          { "stars" => 4, "description" => "Spec-compliant with one cosmetic issue." },
+          { "stars" => 3, "description" => "Right shape, minor deviations (extra commentary, casing, ordering)." },
+          { "stars" => 2, "description" => "Right format with substantive deviations (missing required fields, wrong types)." },
+          { "stars" => 1, "description" => "Wrong format or unparseable." }
+        ]
+      ),
+      Starter.new(
+        key: "tone",
+        name: "Tone",
+        description: "Does the voice fit the audience the prompt asked for?",
+        catches: "Rude, robotic, off-brand, too casual, too formal. The dimension hardest to eyeball at scale and the one most user-facing surfaces care about.",
+        instruction: "Does the voice match the audience and brand the prompt called for? Reward outputs that sound like the persona the prompt asked for. Penalise rude, robotic, off-brand, or wrong-register replies.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Sounds like the brand or persona the prompt asked for." },
+          { "stars" => 4, "description" => "Right tone with a slip or two." },
+          { "stars" => 3, "description" => "Acceptable, generic, no personality." },
+          { "stars" => 2, "description" => "Mismatched tone; sounds like a different audience." },
+          { "stars" => 1, "description" => "Off-tone in a way a user would notice (rude, condescending, jarring)." }
+        ]
+      ),
+      Starter.new(
+        key: "conciseness",
+        name: "Conciseness",
+        description: "Is it the right length — no padding, no missing detail?",
+        catches: "Rambling responses, repetitive caveats, over-hedging. LLMs default to verbose. Conciseness is the dimension where users most often see scores move after tuning.",
+        instruction: "Is the output the right length for the task — no padding, no missing detail, no hedging filler? Penalise rambling, repetition, over-caveating, and unnecessary preamble. Penalise too-short outputs that drop information.",
+        rubric_bands: [
+          { "stars" => 5, "description" => "Exactly as long as the task needs, no more, no less." },
+          { "stars" => 4, "description" => "Right length with a small redundancy." },
+          { "stars" => 3, "description" => "Acceptable; trims could happen or detail could be added." },
+          { "stars" => 2, "description" => "Noticeable filler or visible gaps." },
+          { "stars" => 1, "description" => "Padded, repetitive, or so short it loses information." }
+        ]
+      )
+    ].freeze
+
+    module_function
+
+    def find(key)
+      ALL.find { |s| s.key == key }
+    end
+
+    def available
+      adopted_names = Metric.where(name: ALL.map(&:name)).pluck(:name).to_set
+      dismissed_keys = StarterMetricDismissal.pluck(:starter_key).to_set
+      ALL.reject { |s| adopted_names.include?(s.name) || dismissed_keys.include?(s.key) }
+    end
+
+    def adopted?(starter)
+      Metric.exists?(name: starter.name)
+    end
+  end
+end

data/app/views/completion_kit/metrics/_starter_card.html.erb

@@ -0,0 +1,11 @@
+<article class="ck-starter-card">
+  <h3 class="ck-starter-card__name"><%= link_to starter.name, starter_preview_metrics_path(key: starter.key), class: "ck-link" %></h3>
+  <p class="ck-starter-card__desc"><%= starter.description %></p>
+  <div class="ck-starter-card__actions">
+    <%= link_to "Preview", starter_preview_metrics_path(key: starter.key), class: ck_button_classes(:dark) + " ck-button--sm" %>
+    <%= button_to "Don't show this one", dismiss_starter_metrics_path(key: starter.key),
+          method: :post, form_class: "inline-block",
+          class: "ck-link ck-starter-card__dismiss",
+          data: { turbo_confirm: "Hide \"#{starter.name}\" from this list?" } %>
+  </div>
+</article>

data/app/views/completion_kit/metrics/index.html.erb

@@ -19,7 +19,6 @@
       <tr>
         <th scope="col">Name</th>
         <th scope="col">Instruction</th>
-        <th scope="col" title="How often this metric's scores match the humans who reviewed them.">Trust level</th>
         <th scope="col">In groups</th>
         <th scope="col"></th>
       </tr>
@@ -29,6 +28,21 @@
         <tr onclick="window.location='<%= metric_path(metric) %>'" style="cursor: pointer;">
           <td>
             <%= link_to metric_path(metric), class: "ck-record-name" do %><strong><%= metric.name %></strong><% end %>
+            <% if CompletionKit.config.judge_calibration_enabled %>
+              <% s = CompletionKit::MetricCalibrationStats.for(metric) %>
+              <p class="ck-metrics-table__trust">
+                <% if s.counter_only? %>
+                  <% if s.sample_size.zero? %>
+                    No verdicts yet
+                  <% else %>
+                    <%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts
+                  <% end %>
+                <% else %>
+                  <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read — keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
+                  ±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %>
+                <% end %>
+              </p>
+            <% end %>
             <% if metric.tags.any? %>
               <div class="tag-marks-row">
                 <%= render "completion_kit/tags/marks", tags: metric.tags %>
@@ -36,23 +50,6 @@
             <% end %>
           </td>
           <td data-label="Instruction" class="ck-meta-copy"><div class="ck-clamp-2"><%= metric.instruction.presence || "—" %></div></td>
-          <td data-label="Trust level" class="ck-metrics-table__trust">
-            <% if CompletionKit.config.judge_calibration_enabled %>
-              <% s = CompletionKit::MetricCalibrationStats.for(metric) %>
-              <% if s.counter_only? %>
-                <% if s.sample_size.zero? %>
-                  <span class="ck-meta-copy">No verdicts yet</span>
-                <% else %>
-                  <span class="ck-meta-copy"><%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts</span>
-                <% end %>
-              <% else %>
-                <span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read — keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
-                <span class="ck-meta-copy">±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %></span>
-              <% end %>
-            <% else %>
-              <span class="ck-meta-copy">—</span>
-            <% end %>
-          </td>
           <td data-label="In groups">
             <% groups = metric.metric_groups %>
             <% if groups.any? %>
@@ -78,12 +75,36 @@
       Use the same metrics on multiple runs? <%= link_to "Group them →", metric_groups_path, class: "ck-link" %>
     </p>
   <% end %>
+
+  <% if @available_starters.any? %>
+    <section class="ck-starter-row">
+      <p class="ck-kicker">Add a starter metric</p>
+      <p class="ck-meta-copy">Pre-written rubrics for the dimensions most teams score against. Click a card to preview before it's created.</p>
+      <div class="ck-starter-grid">
+        <% @available_starters.each do |starter| %>
+          <%= render "starter_card", starter: starter %>
+        <% end %>
+      </div>
+    </section>
+  <% end %>
 <% elsif @selected_tags.any? %>
   <div class="ck-empty">
     <p>No metrics match these tags. <%= link_to "Clear filters", metrics_path, class: "ck-link" %>.</p>
   </div>
 <% else %>
-  <div class="ck-empty">
-    <p>No metrics yet. <%= link_to "Create your first metric", new_metric_path, class: "ck-link" %> to start scoring prompt outputs.</p>
-  </div>
+  <% if @available_starters.any? %>
+    <section class="ck-starter-row ck-starter-row--empty-state">
+      <h2 class="ck-title ck-title--sm">Start with a ready-made rubric</h2>
+      <p class="ck-lead">Pick one of the dimensions below to drop in a pre-written 1–5 rubric. You can edit anything after adding it. Or <%= link_to "write your own from scratch", new_metric_path, class: "ck-link" %>.</p>
+      <div class="ck-starter-grid">
+        <% @available_starters.each do |starter| %>
+          <%= render "starter_card", starter: starter %>
+        <% end %>
+      </div>
+    </section>
+  <% else %>
+    <div class="ck-empty">
+      <p>No metrics yet. <%= link_to "Create your first metric", new_metric_path, class: "ck-link" %> to start scoring prompt outputs.</p>
+    </div>
+  <% end %>
 <% end %>

data/app/views/completion_kit/metrics/show.html.erb

@@ -140,82 +140,69 @@
   </section>
 <% end %>
 
-<% if CompletionKit.config.judge_calibration_enabled %>
+<% if CompletionKit.config.judge_calibration_enabled && @disagreements.any? %>
   <section class="ck-card ck-card--spaced">
     <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Where the judge got it wrong</p>
-      <% if @disagreements.any? %>
-        <span class="ck-chip"><%= pluralize(@disagreements.size, "row") %></span>
-      <% end %>
+      <p class="ck-kicker">Cases to learn from</p>
+      <span class="ck-chip"><%= pluralize(@disagreements.size, "row") %></span>
     </div>
-    <% if @disagreements.empty? %>
-      <p class="ck-meta-copy">Nothing here yet. As people give a "disagree" verdict on response rows, those rows show up below so you can review the judge's misses and turn them into teaching examples.</p>
-    <% else %>
-      <p class="ck-meta-copy">Rows where a reviewer said the judge got it wrong. Save the best ones as teaching examples — the judge will see them next time it grades.</p>
-      <table class="ck-results-table ck-disagreements-table">
-        <thead>
-          <tr>
-            <th scope="col">Run · row</th>
-            <th scope="col">Judge</th>
-            <th scope="col">Human</th>
-            <th scope="col">Note</th>
-            <th scope="col"></th>
-          </tr>
-        </thead>
-        <tbody>
-          <% existing_ids = Array(@metric.few_shot_examples).map { |fs| fs["calibration_id"] } %>
-          <% @disagreements.each do |cal| %>
-            <% review = cal.response.reviews.find { |r| r.metric_id == @metric.id } %>
-            <% already = existing_ids.include?(cal.id) %>
-            <tr>
-              <td>
-                <%= link_to ck_run_path(cal.response.run), class: "ck-record-name" do %>
-                  <strong><%= cal.response.run.name.to_s.truncate(40) %></strong>
-                <% end %>
-                <span class="ck-meta-copy">· <%= link_to "row ##{cal.response.id}", run_response_path(cal.response.run, cal.response), class: "ck-link" %></span>
-              </td>
-              <td>
-                <% if review&.ai_score %>
-                  <span class="<%= ck_badge_classes(ck_score_kind(review.ai_score.to_f)) %>"><%= review.ai_score %></span>
-                <% else %>
-                  <span class="ck-meta-copy">—</span>
-                <% end %>
-              </td>
-              <td>
-                <% if cal.corrected_score %>
-                  <span class="<%= ck_badge_classes(ck_score_kind(cal.corrected_score.to_f)) %>"><%= cal.corrected_score %></span>
-                <% else %>
-                  <span class="ck-meta-copy">—</span>
-                <% end %>
-              </td>
-              <td class="ck-meta-copy"><%= cal.note.to_s.truncate(120) %></td>
-              <td>
-                <% if already %>
-                  <span class="ck-chip ck-chip--done">Saved as example</span>
-                <% else %>
-                  <%= button_to "Teach the judge",
-                        add_few_shot_metric_path(@metric, calibration_id: cal.id),
-                        method: :post,
-                        form_class: "inline-block",
-                        class: ck_button_classes(:light, variant: :outline),
-                        title: "Save this row as a teaching example. The judge will see it next time it grades." %>
-                <% end %>
-              </td>
-            </tr>
+    <p class="ck-meta-copy">Rows where a reviewer's score didn't match the judge's. Pin useful ones with <strong>Remember this</strong> — the judge sees them next time it grades.</p>
+    <% existing_ids = Array(@metric.few_shot_examples).map { |fs| fs["calibration_id"] } %>
+    <ul class="ck-disagreement-list">
+      <% @disagreements.each do |cal| %>
+        <% review = cal.response.reviews.find { |r| r.metric_id == @metric.id } %>
+        <% already = existing_ids.include?(cal.id) %>
+        <li class="ck-disagreement">
+          <div class="ck-disagreement__head">
+            <div class="ck-disagreement__scores">
+              <span class="ck-disagreement__scores-label">Judge</span>
+              <% if review&.ai_score %>
+                <span class="<%= ck_badge_classes(ck_score_kind(review.ai_score.to_f)) %>"><%= review.ai_score %></span>
+              <% else %>
+                <span class="ck-meta-copy">—</span>
+              <% end %>
+              <span class="ck-disagreement__scores-arrow">→</span>
+              <span class="ck-disagreement__scores-label">Human</span>
+              <% if cal.corrected_score %>
+                <span class="<%= ck_badge_classes(ck_score_kind(cal.corrected_score.to_f)) %>"><%= cal.corrected_score %></span>
+              <% else %>
+                <span class="ck-meta-copy">—</span>
+              <% end %>
+            </div>
+            <div class="ck-disagreement__action">
+              <% if already %>
+                <span class="ck-chip ck-chip--done">Remembered</span>
+              <% else %>
+                <%= button_to "Remember this",
+                      add_few_shot_metric_path(@metric, calibration_id: cal.id),
+                      method: :post,
+                      form_class: "inline-block",
+                      class: ck_button_classes(:light, variant: :outline),
+                      title: "Pin this row so the judge sees it next time it grades for this metric." %>
+              <% end %>
+            </div>
+          </div>
+          <% if cal.note.to_s.present? %>
+            <p class="ck-disagreement__note"><%= cal.note %></p>
           <% end %>
-        </tbody>
-      </table>
-    <% end %>
+          <p class="ck-disagreement__source ck-meta-copy">
+            <%= link_to cal.response.run.name.to_s.truncate(50), ck_run_path(cal.response.run), class: "ck-link" %>
+            ·
+            <%= link_to "row ##{cal.response.id}", run_response_path(cal.response.run, cal.response), class: "ck-link" %>
+          </p>
+        </li>
+      <% end %>
+    </ul>
   </section>
 
 
   <% if Array(@metric.few_shot_examples).any? %>
     <section class="ck-card ck-card--spaced">
       <div class="ck-prompt-preview__header">
-        <p class="ck-kicker">Teaching examples</p>
-        <span class="ck-chip"><%= pluralize(Array(@metric.few_shot_examples).size, "example") %></span>
+        <p class="ck-kicker">What the judge remembers</p>
+        <span class="ck-chip"><%= pluralize(Array(@metric.few_shot_examples).size, "case") %></span>
       </div>
-      <p class="ck-meta-copy">The judge sees these worked examples whenever it grades for this metric. Each shows what the judge gave and what a human said it should have been.</p>
+      <p class="ck-meta-copy">Rows you've pinned so the judge sees them next time it grades. Each one shows what the judge gave and what a human said it should have been.</p>
       <ol class="ck-few-shot-list">
         <% Array(@metric.few_shot_examples).each do |fs| %>
           <li class="ck-few-shot-item">

data/app/views/completion_kit/metrics/starter_preview.html.erb

@@ -0,0 +1,45 @@
+<section class="ck-page-header">
+  <div>
+    <p class="ck-kicker">Starter metric</p>
+    <h1 class="ck-title"><%= @starter.name %></h1>
+    <p class="ck-lead"><%= @starter.description %></p>
+  </div>
+  <div class="ck-actions">
+    <%= link_to "← Back to metrics", metrics_path, class: ck_button_classes(:light, variant: :outline) %>
+  </div>
+</section>
+
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">What this catches</p>
+  <p class="ck-copy"><%= @starter.catches %></p>
+</section>
+
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">Instruction the judge will see</p>
+  <p class="ck-copy"><%= @starter.instruction %></p>
+</section>
+
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">Rubric</p>
+  <div class="ck-rubric-display">
+    <% @starter.rubric_bands.sort_by { |b| -b["stars"] }.each do |band| %>
+      <div class="ck-rubric-row ck-rubric-row--display">
+        <div class="ck-rubric-row__stars">
+          <% 5.times do |i| %>
+            <svg viewBox="0 0 24 24" width="16" height="16" stroke-width="1.75" class="ck-star <%= i < band["stars"] ? "ck-star--filled" : "ck-star--empty" %>"><polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/></svg>
+          <% end %>
+        </div>
+        <div class="ck-rubric-row__fields">
+          <p class="ck-copy"><%= band["description"] %></p>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</section>
+
+<div class="ck-actions ck-actions--right">
+  <%= link_to "Cancel", metrics_path, class: ck_button_classes(:light, variant: :outline) %>
+  <%= button_to "Add this metric", adopt_starter_metrics_path(key: @starter.key),
+        method: :post, form_class: "inline-block",
+        class: ck_button_classes(:dark) %>
+</div>

data/config/routes.rb

@@ -13,6 +13,11 @@ CompletionKit::Engine.routes.draw do
 
   resources :datasets
   resources :metrics do
+    collection do
+      get  "starters/:key", to: "metrics#starter_preview",  as: :starter_preview
+      post "starters/:key", to: "metrics#adopt_starter",    as: :adopt_starter
+      post "starters/:key/dismiss", to: "metrics#dismiss_starter", as: :dismiss_starter
+    end
     member do
       post :add_few_shot
       post :publish_draft

data/db/migrate/20260524000001_create_completion_kit_starter_metric_dismissals.rb

@@ -0,0 +1,12 @@
+class CreateCompletionKitStarterMetricDismissals < ActiveRecord::Migration[8.1]
+  def change
+    create_table :completion_kit_starter_metric_dismissals do |t|
+      t.string :starter_key, null: false
+      t.timestamps
+    end
+
+    add_index :completion_kit_starter_metric_dismissals, :starter_key,
+              unique: true,
+              name: "index_ck_starter_dismissals_on_key"
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.40"
+  VERSION = "0.5.42"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.40
+  version: 0.5.42
 platform: ruby
 authors:
 - Damien Bastin
@@ -284,6 +284,7 @@ files:
 - app/models/completion_kit/review.rb
 - app/models/completion_kit/run.rb
 - app/models/completion_kit/run_metric.rb
+- app/models/completion_kit/starter_metric_dismissal.rb
 - app/models/completion_kit/suggestion.rb
 - app/models/completion_kit/tag.rb
 - app/models/completion_kit/tagging.rb
@@ -318,6 +319,7 @@ files:
 - app/services/completion_kit/open_router_client.rb
 - app/services/completion_kit/prompt_improvement_service.rb
 - app/services/completion_kit/provider_endpoint.rb
+- app/services/completion_kit/starter_metrics.rb
 - app/services/completion_kit/worker_health.rb
 - app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb
 - app/views/completion_kit/api_reference/_authentication.html.erb
@@ -345,10 +347,12 @@ files:
 - app/views/completion_kit/metric_groups/new.html.erb
 - app/views/completion_kit/metric_groups/show.html.erb
 - app/views/completion_kit/metrics/_form.html.erb
+- app/views/completion_kit/metrics/_starter_card.html.erb
 - app/views/completion_kit/metrics/edit.html.erb
 - app/views/completion_kit/metrics/index.html.erb
 - app/views/completion_kit/metrics/new.html.erb
 - app/views/completion_kit/metrics/show.html.erb
+- app/views/completion_kit/metrics/starter_preview.html.erb
 - app/views/completion_kit/onboarding/_concept.html.erb
 - app/views/completion_kit/onboarding/show.html.erb
 - app/views/completion_kit/prompts/_form.html.erb
@@ -414,6 +418,7 @@ files:
 - db/migrate/20260522000002_create_completion_kit_calibrations.rb
 - db/migrate/20260523000001_add_few_shot_examples_to_completion_kit_metrics.rb
 - db/migrate/20260523000002_add_state_to_completion_kit_judge_versions.rb
+- db/migrate/20260524000001_create_completion_kit_starter_metric_dismissals.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb