completion-kit 0.5.44 → 0.7.0

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -17,17 +17,19 @@
   <input type="radio" name="ck-api-tab" id="ck-tab-datasets" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-metrics" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-metric-groups" class="ck-api-tabs__radio">
+  <input type="radio" name="ck-api-tab" id="ck-tab-calibrations" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-tags" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-providers" class="ck-api-tabs__radio">
 
   <nav class="ck-api-tabs__nav">
-    <label for="ck-tab-mcp" class="ck-api-tabs__label">MCP <span class="ck-api-tabs__count">35</span></label>
+    <label for="ck-tab-mcp" class="ck-api-tabs__label">MCP <span class="ck-api-tabs__count"><%= CompletionKit::McpDispatcher.tool_definitions.size %></span></label>
     <label for="ck-tab-prompts" class="ck-api-tabs__label">Prompts <span class="ck-api-tabs__count">6</span></label>
-    <label for="ck-tab-runs" class="ck-api-tabs__label">Runs <span class="ck-api-tabs__count">7</span></label>
+    <label for="ck-tab-runs" class="ck-api-tabs__label">Runs <span class="ck-api-tabs__count">10</span></label>
     <label for="ck-tab-responses" class="ck-api-tabs__label">Responses <span class="ck-api-tabs__count">2</span></label>
     <label for="ck-tab-datasets" class="ck-api-tabs__label">Datasets <span class="ck-api-tabs__count">5</span></label>
-    <label for="ck-tab-metrics" class="ck-api-tabs__label">Metrics <span class="ck-api-tabs__count">5</span></label>
+    <label for="ck-tab-metrics" class="ck-api-tabs__label">Metrics <span class="ck-api-tabs__count">12</span></label>
     <label for="ck-tab-metric-groups" class="ck-api-tabs__label">Metric Groups <span class="ck-api-tabs__count">5</span></label>
+    <label for="ck-tab-calibrations" class="ck-api-tabs__label">Calibrations <span class="ck-api-tabs__count">3</span></label>
     <label for="ck-tab-tags" class="ck-api-tabs__label">Tags <span class="ck-api-tabs__count">5</span></label>
     <label for="ck-tab-providers" class="ck-api-tabs__label">Providers <span class="ck-api-tabs__count">5</span></label>
   </nav>
@@ -36,7 +38,7 @@
 
     <div class="ck-api-tabs__panel">
       <h2 class="ck-section-title">MCP Server</h2>
-      <p class="ck-copy">Connect Claude Code, Cursor, or any <a href="https://modelcontextprotocol.io" class="ck-link">MCP</a> client to manage prompts, runs, datasets, and metrics conversationally. 35 tools over streamable HTTP.</p>
+      <p class="ck-copy">Connect Claude Code, Cursor, or any <a href="https://modelcontextprotocol.io" class="ck-link">MCP</a> client to manage prompts, runs, datasets, and metrics conversationally. <%= CompletionKit::McpDispatcher.tool_definitions.size %> tools over streamable HTTP.</p>
 
       <div class="ck-mcp-install-grid">
         <div class="ck-mcp-install-card">
@@ -116,7 +118,8 @@
       <p class="ck-copy">Create runs, generate LLM responses, and judge them with metrics.</p>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs</p>
-        <p class="ck-meta-copy">List all runs with response counts and average scores.</p>
+        <p class="ck-meta-copy">List runs with response counts and average scores. Supports pagination (<code>limit</code>, <code>offset</code>) and the following filters.</p>
+        <p class="ck-api-params"><strong>Optional filters:</strong>&ensp;<code>status</code> (<code>pending</code>, <code>running</code>, <code>completed</code>, <code>failed</code>), <code>prompt_id</code>, <code>dataset_id</code>, <code>tag[]</code></p>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs</p>
@@ -139,6 +142,24 @@
         <p class="ck-meta-copy">Start generating responses. Returns 202 Accepted. Poll the run to check progress.</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs/1/generate \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:id/retry_failures</p>
+        <p class="ck-meta-copy">Re-queue any responses that failed during generation. Returns 202 Accepted.</p>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:id/rerun</p>
+        <p class="ck-meta-copy">Clone the run and start generating responses on the copy against the current prompt and metric versions. Returns the new run with 201 Created. Useful for capturing a fresh baseline after metric edits.</p>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs/1/rerun \\\n  -H \"Authorization: Bearer #{token}\"" %>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:id/regrade</p>
+        <p class="ck-meta-copy">Re-judge the existing successful responses against the current metric versions without regenerating model output. Returns 202 Accepted, or 422 if no responses are eligible.</p>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:id/compare?with=:other_id</p>
+        <p class="ck-meta-copy">Side-by-side comparison against another run. Returns <code>{rows: [...], metric_ids: [...]}</code> with one row per input case, per-metric scores on both sides, and the delta. Cases that exist on only one side are still returned with the missing side nulled out.</p>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl \"#{base_url}/api/v1/runs/1/compare?with=2\" \\\n  -H \"Authorization: Bearer #{token}\"" %>
+      </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">PATCH</span> /api/v1/runs/:id</p>
         <p class="ck-meta-copy">Update a run. Accepts same params as create.</p>
@@ -154,7 +175,8 @@
       <p class="ck-copy">Read-only access to generated responses and their review scores. Nested under runs.</p>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:run_id/responses</p>
-        <p class="ck-meta-copy">List all responses for a run, including nested review scores.</p>
+        <p class="ck-meta-copy">List responses for a run, including nested review scores.</p>
+        <p class="ck-api-params"><strong>Optional filters:</strong>&ensp;<code>status</code> (<code>pending</code>, <code>succeeded</code>, <code>failed</code>), plus <code>limit</code> and <code>offset</code></p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl #{base_url}/api/v1/runs/1/responses \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <div class="ck-api-endpoint">
@@ -214,6 +236,48 @@
               { name: m.name, subtitle: m.instruction.presence&.truncate(100),
                 url: "#{base_url}/api/v1/metrics/#{m.id}", dom_id: "metric_ep_#{m.id}" }
             } %>
+
+      <div class="ck-api-endpoint" style="padding-top: 1.5rem;">
+        <p class="ck-kicker" style="margin-bottom: 0.5rem;">Calibration loop</p>
+        <p class="ck-meta-copy">Drive metric improvement from disagree-flagged calibrations: ask the model for variants, then pin individual cases as few-shot examples on the metric.</p>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/metrics/:id/suggest_variants</p>
+        <p class="ck-meta-copy">Generate draft metric versions from the current disagreements. Returns 201 with the new draft versions, 422 if no disagreements exist or the model produced nothing usable.</p>
+        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>count</code>, <code>model</code></p>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/metrics/1/suggest_variants \\\n  -H \"Authorization: Bearer #{token}\"" %>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/metrics/:id/add_few_shot</p>
+        <p class="ck-meta-copy">Pin a disagree calibration as a few-shot example on the metric. Returns the updated metric, or 404 if the calibration is not a disagree on this metric.</p>
+        <p class="ck-api-params"><strong>Required:</strong>&ensp;<code>calibration_id</code></p>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip" style="color: var(--ck-danger);">DELETE</span> /api/v1/metrics/:id/remove_few_shot</p>
+        <p class="ck-meta-copy">Drop the pinned few-shot example by calibration ID.</p>
+        <p class="ck-api-params"><strong>Required:</strong>&ensp;<code>calibration_id</code></p>
+      </div>
+
+      <div class="ck-api-endpoint" style="padding-top: 1.5rem;">
+        <p class="ck-kicker" style="margin-bottom: 0.5rem;">Metric versions</p>
+        <p class="ck-meta-copy">Every metric carries a history of versions (the current published one, prior published ones, and any draft suggestions). Reviews and calibrations record the version they ran against, so the API can surface stale state and let you revert.</p>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/metrics/:metric_id/metric_versions</p>
+        <p class="ck-meta-copy">List every version for the metric, newest version_number first.</p>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/metrics/:metric_id/metric_versions/:id</p>
+        <p class="ck-meta-copy">Get a single version with its instruction, rubric bands, state, and source.</p>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/metrics/:metric_id/metric_versions/:id/publish</p>
+        <p class="ck-meta-copy">Publish the version as current. Works for a draft (promote) or a superseded published version (revert). Copies the version's instruction and rubric back onto the metric.</p>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip" style="color: var(--ck-danger);">DELETE</span> /api/v1/metrics/:metric_id/metric_versions/:id</p>
+        <p class="ck-meta-copy">Dismiss a draft version. Returns 204 No Content, or 409 Conflict if the version is published (published versions are immutable history).</p>
+      </div>
     </div>
 
     <div class="ck-api-tabs__panel">
@@ -239,6 +303,27 @@
             } %>
     </div>
 
+    <div class="ck-api-tabs__panel">
+      <h2 class="ck-section-title">Calibrations</h2>
+      <p class="ck-copy">Per-verdict feedback events on a response/metric pair: agree, disagree (with a corrected score and note), or borderline. Calibrations capture the metric version that was current when the verdict was cast, which is what drives the trust signal and the "stale" indicators across the rest of the API.</p>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/calibrations</p>
+        <p class="ck-meta-copy">List calibrations across all runs. Supports filtering by any combination of the query params below.</p>
+        <p class="ck-api-params"><strong>Optional filters:</strong>&ensp;<code>run_id</code>, <code>response_id</code>, <code>metric_id</code>, <code>metric_version_id</code>, <code>created_by</code>, <code>verdict</code> (<code>agree</code>, <code>disagree</code>, or <code>borderline</code>)</p>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl \"#{base_url}/api/v1/calibrations?metric_id=1&verdict=disagree\" \\\n  -H \"Authorization: Bearer #{token}\"" %>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:run_id/responses/:response_id/metrics/:metric_id/calibrations</p>
+        <p class="ck-meta-copy">Cast a calibration on a specific response/metric pair. The metric version on the record is set automatically from the run's review.</p>
+        <p class="ck-api-params"><strong>Required:</strong>&ensp;<code>verdict</code>, <code>created_by</code>&emsp;<strong>Optional:</strong>&ensp;<code>corrected_score</code>, <code>note</code></p>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs/1/responses/42/metrics/3/calibrations \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"verdict\": \"disagree\", \"corrected_score\": 3, \"note\": \"too generous\", \"created_by\": \"alice\"}'" %>
+      </div>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip" style="color: var(--ck-danger);">DELETE</span> /api/v1/calibrations/:id</p>
+        <p class="ck-meta-copy">Delete a calibration. Returns 204 No Content.</p>
+      </div>
+    </div>
+
     <div class="ck-api-tabs__panel">
       <h2 class="ck-section-title">Tags</h2>
       <p class="ck-copy">Domain labels you can attach to metrics, prompts, runs, and datasets. Tags are auto-assigned a color from a 10-color palette. Each index page can be filtered by one or more tags using <code>?tag[]=name</code> query params (OR semantics).</p>

data/app/views/completion_kit/api_reference/index.html.erb

@@ -12,6 +12,14 @@
         <p class="ck-kicker">Authentication</p>
         <%= render CompletionKit.config.api_reference_authentication_partial, token: @token %>
       </div>
+      <div>
+        <p class="ck-kicker">Pagination</p>
+        <p class="ck-meta-copy">Every index endpoint accepts <code>?limit=</code> and <code>?offset=</code> (default limit 50, max 500). The server returns <code>X-Total-Count</code>, <code>X-Limit</code>, and <code>X-Offset</code> headers so the caller can build cursors without re-counting.</p>
+      </div>
+      <div>
+        <p class="ck-kicker">Tag filtering</p>
+        <p class="ck-meta-copy">Prompts, runs, metrics, datasets, and metric groups accept <code>?tag[]=name</code> (repeat for OR semantics).</p>
+      </div>
     </div>
   </div>
 </div>

data/app/views/completion_kit/calibrations/_trust_panel.html.erb

@@ -15,12 +15,17 @@
        .where.not(id: verdicted_ids)
        .order(created_at: :desc).first
    end %>
+<% prior_version_verdicts = if stats.sample_size.zero? && metric && current_metric_version
+     CompletionKit::Calibration.where(metric_id: metric.id).where.not(metric_version_id: current_metric_version.id).count
+   else
+     0
+   end %>
 
 <p class="ck-trust-line ck-trust-line--<%= stats.gate %>">
   <span class="ck-trust-line__label"><%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true" %>Calibration</span>
   <% if stats.sample_size.zero? %>
     <span class="ck-trust-line__state">Not measured yet.</span>
-    <span class="ck-trust-line__hint">Needs <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts on the judge's scores.<% if target_response %>
+    <span class="ck-trust-line__hint">Needs <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts on the judge's scores.<% if prior_version_verdicts > 0 %> (<%= pluralize(prior_version_verdicts, "verdict") %> on prior versions, tied to that version's history.)<% end %><% if target_response %>
       <%= link_to "Give a verdict →", run_response_path(target_response.run, target_response, anchor: anchor), class: "ck-trust-line__link" %>
     <% else %> Run this metric on a dataset, then give a verdict.<% end %></span>
   <% elsif stats.counter_only? %>

data/app/views/completion_kit/metrics/index.html.erb

@@ -80,7 +80,7 @@
 
   <% if @available_starters.any? %>
     <section class="ck-starter-row">
-      <p class="ck-kicker">Add a starter metric</p>
+      <p class="ck-kicker">Skip the blank page</p>
       <p class="ck-meta-copy">Pre-written rubrics for the dimensions most teams score against. Click a card to preview before it's created.</p>
       <div class="ck-starter-grid">
         <% @available_starters.each do |starter| %>
@@ -96,8 +96,8 @@
 <% else %>
   <% if @available_starters.any? %>
     <section class="ck-starter-row ck-starter-row--empty-state">
-      <h2 class="ck-title ck-title--sm">Start with a ready-made rubric</h2>
-      <p class="ck-lead">Pick one of the dimensions below to drop in a pre-written 1–5 rubric. You can edit anything after adding it. Or <%= link_to "write your own from scratch", new_metric_path, class: "ck-link" %>.</p>
+      <h2 class="ck-title ck-title--sm">Skip the blank page</h2>
+      <p class="ck-lead">Five rubrics we've worked through for common evaluation dimensions. Adopt one to drop in a pre-written 1&ndash;5 scale, edit anything after. Or <%= link_to "write your own from scratch", new_metric_path, class: "ck-link" %>.</p>
       <div class="ck-starter-grid">
         <% @available_starters.each do |starter| %>
           <%= render "starter_card", starter: starter %>

data/app/views/completion_kit/metrics/show.html.erb

@@ -109,6 +109,7 @@
               <% source_label, source_class = case v.source
                                               when "suggestion" then ["AI suggestion", "ck-source-chip ck-source-chip--ai"]
                                               when "edit" then ["Manual edit", "ck-source-chip ck-source-chip--manual"]
+                                              when "revert" then ["Reverted", "ck-source-chip ck-source-chip--revert"]
                                               else ["Original", "ck-source-chip ck-source-chip--initial"]
                                               end %>
               <span class="<%= source_class %>"><%= source_label %></span>
@@ -165,7 +166,7 @@
       <p class="ck-kicker">Cases to learn from</p>
       <span class="ck-chip"><%= pluralize(@disagreements.size, "case") %></span>
     </div>
-    <% mixed_versions = @disagreements.map(&:metric_version_id).uniq.size > 1 %>
+    <% mixed_versions = @disagreements.any? { |c| c.metric_version_id != @published_metric_version.id } %>
     <p class="ck-meta-copy">Cases where a reviewer's score didn't match the judge's. Pin useful ones with <strong>Remember this</strong> so the judge sees them next time it grades<%= " (pins flow into the current judge regardless of which version produced the verdict)" if mixed_versions %>.</p>
     <% existing_ids = Array(@metric.few_shot_examples).map { |fs| fs["calibration_id"] } %>
     <ul class="ck-disagreement-list">

data/app/views/completion_kit/runs/_actions.html.erb

@@ -11,6 +11,7 @@
       <%= button_to "Retry", generate_run_path(run), method: :post, class: ck_button_classes(:light, variant: :outline), form_class: "inline-block" %>
       <%= button_to "Re-run as new", rerun_run_path(run), method: :post, class: ck_button_classes(:dark), form_class: "inline-block" %>
     <% elsif run.status == "completed" %>
+      <%= link_to "Compare", compare_run_path(run), class: ck_button_classes(:light, variant: :outline) %>
       <%= button_to "Re-run", rerun_run_path(run), method: :post, class: ck_button_classes(:dark), form_class: "inline-block" %>
     <% end %>
   <% end %>

data/app/views/completion_kit/runs/compare.html.erb

@@ -0,0 +1,85 @@
+<ol class="ck-breadcrumb">
+  <li><%= link_to "Runs", runs_path %></li>
+  <li><%= link_to @run.name, run_path(@run) %></li>
+  <li>vs <%= @other_run.name %></li>
+</ol>
+
+<section class="ck-page-header">
+  <div>
+    <h1 class="ck-title">Comparing runs</h1>
+    <p class="ck-meta-copy"><strong>A</strong>: <%= link_to @run.name, run_path(@run), class: "ck-link" %> &middot; <strong>B</strong>: <%= link_to @other_run.name, run_path(@other_run), class: "ck-link" %></p>
+  </div>
+  <div class="ck-actions">
+    <%= link_to "Pick another", compare_run_path(@run), class: ck_button_classes(:light, variant: :outline) %>
+  </div>
+</section>
+
+<% if @comparison[:rows].empty? %>
+  <div class="ck-empty">
+    <p>No responses to compare yet.</p>
+  </div>
+<% else %>
+  <table class="ck-results-table ck-run-compare-table">
+    <thead>
+      <tr>
+        <th scope="col">Case</th>
+        <th scope="col">Metric</th>
+        <th scope="col">A score</th>
+        <th scope="col">B score</th>
+        <th scope="col">Δ</th>
+        <th scope="col">A version</th>
+        <th scope="col">B version</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @comparison[:rows].each do |row| %>
+        <% case_label = ((row[:left_response].row_index || 0) + 1).to_s %>
+        <% row[:per_metric].each_with_index do |pm, idx| %>
+          <tr>
+            <% if idx == 0 %>
+              <td rowspan="<%= row[:per_metric].size %>">
+                <%= link_to case_label, run_response_path(@run, row[:left_response]), class: "ck-link" %>
+                <% if row[:right_response] %>
+                  <span class="ck-meta-copy">/ <%= link_to "B", run_response_path(@other_run, row[:right_response]), class: "ck-link" %></span>
+                <% end %>
+              </td>
+            <% end %>
+            <td><%= pm[:metric_name] %></td>
+            <td>
+              <% if pm[:left_score] %>
+                <span class="<%= ck_badge_classes(ck_score_kind(pm[:left_score].to_f)) %>"><%= pm[:left_score] %></span>
+              <% else %>
+                <span class="ck-meta-copy">—</span>
+              <% end %>
+            </td>
+            <td>
+              <% if pm[:right_score] %>
+                <span class="<%= ck_badge_classes(ck_score_kind(pm[:right_score].to_f)) %>"><%= pm[:right_score] %></span>
+              <% else %>
+                <span class="ck-meta-copy">—</span>
+              <% end %>
+            </td>
+            <td>
+              <% if pm[:delta] %>
+                <% delta_class = pm[:delta] > 0 ? "ck-delta--positive" : pm[:delta] < 0 ? "ck-delta--negative" : "ck-delta--zero" %>
+                <span class="ck-delta <%= delta_class %>"><%= pm[:delta].positive? ? "+#{pm[:delta]}" : pm[:delta].to_s %></span>
+              <% else %>
+                <span class="ck-meta-copy">—</span>
+              <% end %>
+            </td>
+            <td>
+              <% if pm[:left_version_label] %>
+                <span class="ck-source-chip ck-source-chip--current"><%= pm[:left_version_label] %></span>
+              <% end %>
+            </td>
+            <td>
+              <% if pm[:right_version_label] %>
+                <span class="ck-source-chip ck-source-chip--current"><%= pm[:right_version_label] %></span>
+              <% end %>
+            </td>
+          </tr>
+        <% end %>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>

data/app/views/completion_kit/runs/compare_picker.html.erb

@@ -0,0 +1,39 @@
+<ol class="ck-breadcrumb">
+  <li><%= link_to "Runs", runs_path %></li>
+  <li><%= link_to @run.name, run_path(@run) %></li>
+  <li>Compare</li>
+</ol>
+
+<section class="ck-page-header">
+  <div>
+    <h1 class="ck-title">Compare with another run</h1>
+    <p class="ck-lead">Pick a run on the same dataset and prompt to see per-case score deltas side by side.</p>
+  </div>
+</section>
+
+<% if @other_runs.any? %>
+  <table class="ck-results-table">
+    <thead>
+      <tr>
+        <th scope="col">Run</th>
+        <th scope="col">Judge</th>
+        <th scope="col">Created</th>
+        <th scope="col"></th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @other_runs.each do |other| %>
+        <tr>
+          <td><%= link_to other.name, run_path(other), class: "ck-link" %></td>
+          <td class="ck-meta-copy"><%= other.judge_model %></td>
+          <td class="ck-meta-copy"><time datetime="<%= other.created_at.utc.iso8601 %>"><%= time_ago_in_words(other.created_at) %> ago</time></td>
+          <td class="ck-results-table__arrow"><%= link_to "Compare →", compare_run_path(@run, with: other.id), class: "ck-link" %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% else %>
+  <div class="ck-empty">
+    <p>No other runs on this dataset + prompt combination yet. <%= link_to "Re-run from this one", rerun_run_path(@run), method: :post, class: "ck-link" %> to create one.</p>
+  </div>
+<% end %>

data/app/views/completion_kit/runs/show.html.erb

@@ -32,10 +32,16 @@
         </p>
       </div>
       <% if @run.status == "completed" %>
-        <%= button_to "Re-run with current judge",
+        <%= button_to "Re-run from scratch",
               rerun_run_path(@run), method: :post,
+              class: ck_button_classes(:light, variant: :outline), form_class: "inline-block",
+              title: "Create a new run that regenerates responses and grades them with the current judge.",
+              data: { turbo_confirm: "Create a new run with fresh responses and the current judge? The original run stays as a record." } %>
+        <%= button_to "Re-grade with current judge",
+              regrade_run_path(@run), method: :post,
               class: ck_button_classes(:dark), form_class: "inline-block",
-              data: { turbo_confirm: "Create a new run with the current metric versions? The original run stays as a record." } %>
+              title: "Re-judge this run's existing responses against the current judge. Faster and cheaper than re-running.",
+              data: { turbo_confirm: "Re-judge this run's existing responses against the current judge?" } %>
       <% end %>
     </div>
   <% end %>

data/config/routes.rb

@@ -37,7 +37,9 @@ CompletionKit::Engine.routes.draw do
       post :suggest
       post :retry_failures
       post :rerun
+      post :regrade
       get :refresh_status
+      get :compare
     end
     resources :responses, only: [:show] do
       resources :calibrations, only: [:create]
@@ -68,6 +70,9 @@ CompletionKit::Engine.routes.draw do
         member do
           post :generate
           post :retry_failures
+          post :rerun
+          post :regrade
+          get :compare
         end
         resources :responses, only: [:index, :show] do
           resources :metrics, only: [] do
@@ -76,10 +81,22 @@ CompletionKit::Engine.routes.draw do
         end
       end
       resources :datasets
-      resources :metrics
+      resources :metrics do
+        resources :metric_versions, only: [:index, :show, :destroy] do
+          member do
+            post :publish
+          end
+        end
+        member do
+          post :suggest_variants
+          post :add_few_shot
+          delete :remove_few_shot
+        end
+      end
       resources :metric_groups
       resources :tags
       resources :provider_credentials
+      resources :calibrations, only: [:index, :destroy]
     end
   end
 

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.44"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.44
+  version: 0.7.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -238,6 +238,7 @@ files:
 - app/controllers/completion_kit/api/v1/calibrations_controller.rb
 - app/controllers/completion_kit/api/v1/datasets_controller.rb
 - app/controllers/completion_kit/api/v1/metric_groups_controller.rb
+- app/controllers/completion_kit/api/v1/metric_versions_controller.rb
 - app/controllers/completion_kit/api/v1/metrics_controller.rb
 - app/controllers/completion_kit/api/v1/prompts_controller.rb
 - app/controllers/completion_kit/api/v1/provider_credentials_controller.rb
@@ -288,6 +289,7 @@ files:
 - app/models/completion_kit/suggestion.rb
 - app/models/completion_kit/tag.rb
 - app/models/completion_kit/tagging.rb
+- app/models/concerns/completion_kit/has_job_status.rb
 - app/models/concerns/completion_kit/taggable.rb
 - app/services/completion_kit/anthropic_client.rb
 - app/services/completion_kit/api_config.rb
@@ -302,6 +304,7 @@ files:
 - app/services/completion_kit/mcp_tools/datasets.rb
 - app/services/completion_kit/mcp_tools/judges.rb
 - app/services/completion_kit/mcp_tools/metric_groups.rb
+- app/services/completion_kit/mcp_tools/metric_versions.rb
 - app/services/completion_kit/mcp_tools/metrics.rb
 - app/services/completion_kit/mcp_tools/prompts.rb
 - app/services/completion_kit/mcp_tools/provider_credentials.rb
@@ -377,6 +380,8 @@ files:
 - app/views/completion_kit/runs/_status_header.html.erb
 - app/views/completion_kit/runs/_status_panel.html.erb
 - app/views/completion_kit/runs/_table.html.erb
+- app/views/completion_kit/runs/compare.html.erb
+- app/views/completion_kit/runs/compare_picker.html.erb
 - app/views/completion_kit/runs/edit.html.erb
 - app/views/completion_kit/runs/index.html.erb
 - app/views/completion_kit/runs/new.html.erb