completion-kit 0.28.24 → 0.28.34

data/app/views/completion_kit/responses/show.html.erb

@@ -95,11 +95,12 @@
 <% end %>
 
 <% if @reviews.any? %>
+  <% review_header = ck_review_section_header(@reviews, @run.judge_model) %>
   <section class="ck-card--spaced">
-    <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Review</p>
-      <% if @run.judge_model.present? %>
-        <span class="ck-chip ck-chip--soft" style="text-transform: none;"><%= @run.judge_model %></span>
+    <div class="ck-prompt-preview__header" data-ck-review-header>
+      <p class="ck-kicker"><%= review_header[:heading] %></p>
+      <% if review_header[:judge_model] %>
+        <span class="ck-chip ck-chip--soft" style="text-transform: none;"><%= review_header[:judge_model] %></span>
       <% end %>
     </div>
 
@@ -126,7 +127,7 @@
               <% elsif review.check? && !review.passed.nil? %>
                 <%= ck_check_badge(review.passed) %>
               <% elsif review.status == "failed" %>
-                <span class="<%= ck_badge_classes(:failed) %>" title="<%= review.error_message.presence || "Judge failed" %>">Failed</span>
+                <span class="<%= ck_badge_classes(:failed) %>" title="<%= review.error_message.presence || (review.check? ? "Check failed" : "Judge failed") %>">Failed</span>
               <% else %>
                 <span class="<%= ck_badge_classes(:pending) %>">Pending</span>
               <% end %>

data/app/views/completion_kit/runs/_form.html.erb

@@ -93,6 +93,14 @@
       </div>
     </div>
 
+    <div class="ck-field">
+      <label class="ck-label" for="run_max_tokens" style="position: relative;">
+        Max tokens<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">The ceiling on how many tokens the model may generate per row. Leave it blank to use the provider's default. Set it when your prompt produces long output: if generation hits the cap the response is cut off mid-sentence, and the judge scores the truncation rather than the prompt. Match whatever your production caller uses so the evaluation reflects reality.</span>
+      </label>
+      <%= form.number_field :max_tokens, min: 1, step: 1, class: "ck-input", id: "run_max_tokens", placeholder: "Provider default", **ck_field_aria(form, :max_tokens) %>
+      <%= ck_field_error(form, :max_tokens) %>
+    </div>
+
     <div class="ck-field" id="judge-field">
       <label class="ck-label" for="run_judge_model" style="position: relative;">
         Judge model<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">Judge models score generated responses against your metrics. Pick one when configuring a run.</span>
@@ -118,6 +126,15 @@
       <% end %>
     </div>
 
+    <div class="ck-field" id="judge-temperature-field">
+      <label class="ck-label" for="run_judge_temperature" style="position: relative;">
+        Judge temperature<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">How random the judge is when it scores. This is separate from the generation temperature above. Keep it at 0 so re-judging the same output gives the same score, which is what makes one run comparable to another. Raise it only when you are deliberately measuring how much the judge disagrees with itself.</span>
+      </label>
+      <%= form.number_field :judge_temperature, in: 0..1, step: 0.1, class: "ck-input", id: "run_judge_temperature", **ck_field_aria(form, :judge_temperature) %>
+      <%= ck_field_error(form, :judge_temperature) %>
+      <p class="ck-field-hint" id="judge-temperature-hint">0 is deterministic and recommended. Above 0, the same output can score differently on a re-judge.</p>
+    </div>
+
     <div class="ck-field" id="metrics-field">
       <p class="ck-label" id="run_metrics_label">Metrics</p>
       <% if @all_metrics.empty? %>

data/app/views/completion_kit/runs/_responses_region.html.erb

@@ -0,0 +1,20 @@
+<div id="run_responses_region">
+  <p class="ck-kicker" style="margin-top: 1.5rem;<%= ' display:none;' unless responses.any? %>" id="responses_kicker">Responses</p>
+  <table class="ck-results-table ck-responses-table"<%= ' hidden'.html_safe unless responses.any? %> id="responses_table">
+    <thead>
+      <tr>
+        <th scope="col">#</th>
+        <th scope="col">Response</th>
+        <th scope="col">Metrics</th>
+        <th scope="col">Avg score</th>
+        <th scope="col">Status</th>
+        <th scope="col"></th>
+      </tr>
+    </thead>
+    <tbody id="run_responses">
+      <% responses.each_with_index do |response, idx| %>
+        <%= render "response_row", run: run, response: response, index: responses_offset + idx + 1 %>
+      <% end %>
+    </tbody>
+  </table>
+</div>

data/app/views/completion_kit/runs/_status_header.html.erb

@@ -12,6 +12,28 @@
     </div>
   <% end %>
 
+  <% if run.scores_at_ceiling? %>
+    <div class="ck-score-ceiling-banner" role="status">
+      <div class="ck-score-ceiling-banner__body">
+        <p class="ck-kicker">Scores are clustered near the top</p>
+        <p class="ck-meta-copy">
+          A near-perfect average usually means the judge is not separating good output from bad, not that every
+          output is good. Check it against your own reading before you trust the number.
+        </p>
+      </div>
+      <div class="ck-score-ceiling-banner__actions">
+        <%= link_to "Worst first", run_path(run, sort: "score_asc"),
+              class: ck_button_classes(:light, variant: :outline) + " ck-button--sm",
+              title: "Sort the responses lowest score first to see whether the judge ever penalises anything." %>
+        <% if CompletionKit.config.judge_agreement_enabled && run.calibratable_metric %>
+          <%= link_to "Check the judge", metric_path(run.calibratable_metric, anchor: "agreement"),
+                class: ck_button_classes(:dark) + " ck-button--sm",
+                title: "Score a few outputs yourself and see how often the judge agrees with you." %>
+        <% end %>
+      </div>
+    </div>
+  <% end %>
+
   <section class="ck-page-header">
     <div>
       <span class="ck-status-badge ck-status-badge--<%= run.status %>">

data/app/views/completion_kit/runs/_status_panel.html.erb

@@ -77,6 +77,24 @@
             </div>
           <% end %>
 
+          <% if has_llm_metric && CompletionKit.config.judge_agreement_enabled %>
+            <% agreement = run.judge_agreement %>
+            <div class="ck-run-status__cell">
+              <p class="ck-run-status__metric-label">Judge agreement</p>
+              <div class="ck-run-status__cell-value <%= agreement ? "ck-run-status__cell-value--split" : "ck-run-status__cell-value--stack" %>">
+                <% if agreement %>
+                  <span class="<%= ck_badge_classes(ck_pass_rate_kind(agreement[:rate])) %> ck-badge--lg"><%= (agreement[:rate] * 100).round %>%</span>
+                  <span class="ck-run-status__summary-text"><%= pluralize(agreement[:sample_size], "label") %></span>
+                <% else %>
+                  <span class="ck-run-status__cell-unverified">Unverified</span>
+                  <% if run.calibratable_metric %>
+                    <%= link_to "Check the judge &rarr;".html_safe, metric_path(run.calibratable_metric, anchor: "agreement"), class: "ck-cal-link" %>
+                  <% end %>
+                <% end %>
+              </div>
+            </div>
+          <% end %>
+
           <% if has_checks %>
             <div class="ck-run-status__cell">
               <p class="ck-run-status__metric-label">Checks passed</p>

data/app/views/completion_kit/runs/show.html.erb

@@ -25,20 +25,26 @@
           This run was scored against metric versions that are no longer live.
           <% stale_summary.values.each_with_index do |s, i| %>
             <%= ", " if i > 0 %><strong><%= s[:metric_name] %></strong> (scored by <%= s[:scored_labels].join(", ") %>; live is <%= s[:current_label] %>)<% end %>.
-          Re-run to refresh the scores with the current metrics.
+          Re-grade to refresh the scores with the current metrics.
         </p>
       </div>
       <% if @run.status == "completed" %>
-        <%= button_to "Re-run from scratch",
-              rerun_run_path(@run), method: :post,
-              class: ck_button_classes(:light, variant: :outline), form_class: "inline-block",
-              title: "Create a new run that regenerates responses and grades them with the current metrics.",
-              data: { turbo_confirm: "Create a new run with fresh responses and the current metrics? The original run stays as a record." } %>
-        <%= button_to "Re-grade with current metrics",
-              regrade_run_path(@run), method: :post,
-              class: ck_button_classes(:dark), form_class: "inline-block",
-              title: "Re-grade this run's existing responses against the current metrics. Faster and cheaper than re-running.",
-              data: { turbo_confirm: "Re-grade this run's existing responses against the current metrics?" } %>
+        <div class="ck-banner-actions">
+          <div class="ck-banner-action">
+            <%= button_to "Re-grade with current metrics",
+                  regrade_run_path(@run), method: :post,
+                  class: ck_button_classes(:dark), form_class: "inline-block",
+                  data: { turbo_confirm: "Re-grade this run's existing responses against the current metrics?" } %>
+            <p class="ck-banner-action__note">Re-scores the responses you already have. No generation cost.</p>
+          </div>
+          <div class="ck-banner-action">
+            <%= button_to "Re-run from scratch",
+                  rerun_run_path(@run), method: :post,
+                  class: ck_button_classes(:light, variant: :outline), form_class: "inline-block",
+                  data: { turbo_confirm: "Create a new run with fresh responses and the current metrics? The original run stays as a record." } %>
+            <p class="ck-banner-action__note">Generates fresh responses first. Costs a full run.</p>
+          </div>
+        </div>
       <% end %>
     </div>
   <% end %>
@@ -83,6 +89,17 @@
       <span class="ck-run-config__warn" style="color: var(--ck-dim);" title="The model rejected the temperature parameter, so CompletionKit re-sent the request without it.">ignored by model</span>
     <% end %>
   </div>
+  <div class="ck-run-config__row">
+    <span class="ck-run-config__key">Max tokens</span>
+    <span><%= @run.max_tokens || "Provider default" %></span>
+  </div>
+  <div class="ck-run-config__row">
+    <span class="ck-run-config__key">Judge temperature</span>
+    <span><%= @run.judge_temperature %></span>
+    <% if @run.nondeterministic_judge? %>
+      <span class="ck-run-config__warn" style="color: var(--ck-warning);" title="Judging above temperature 0 makes scores irreproducible. The same output can get a different score on a re-judge.">scores not reproducible</span>
+    <% end %>
+  </div>
 </div>
 
 <% if @run.prompt %>
@@ -109,7 +126,7 @@
     <div class="ck-prompt-preview__header">
       <p class="ck-kicker">Output source</p>
     </div>
-    <p class="ck-prompt-preview__text">Dataset column <code><%= @run.output_column.presence || "actual_output" %></code> — no prompt generated these outputs.</p>
+    <p class="ck-prompt-preview__text">Dataset column <code><%= @run.output_column.presence || "actual_output" %></code>. No prompt generated these outputs.</p>
   </div>
 <% end %>
 
@@ -186,24 +203,7 @@
 
 <%= render "sort_toolbar", run: @run %>
 
-<p class="ck-kicker" style="margin-top: 1.5rem;<%= ' display:none;' unless @responses.any? %>" id="responses_kicker">Responses</p>
-<table class="ck-results-table ck-responses-table"<%= ' hidden'.html_safe unless @responses.any? %> id="responses_table">
-  <thead>
-    <tr>
-      <th scope="col">#</th>
-      <th scope="col">Response</th>
-      <th scope="col">Metrics</th>
-      <th scope="col">Avg score</th>
-      <th scope="col">Status</th>
-      <th scope="col"></th>
-    </tr>
-  </thead>
-  <tbody id="run_responses">
-    <% @responses.each_with_index do |response, idx| %>
-      <%= render "response_row", run: @run, response: response, index: @responses_offset + idx + 1 %>
-    <% end %>
-  </tbody>
-</table>
+<%= render "responses_region", run: @run, responses: @responses, responses_offset: @responses_offset %>
 
 <% if @responses_total_pages > 1 %>
   <nav class="ck-pagination" aria-label="Response pages">

data/db/migrate/20260728000001_add_max_tokens_to_completion_kit_runs.rb

@@ -0,0 +1,5 @@
+class AddMaxTokensToCompletionKitRuns < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_runs, :max_tokens, :integer
+  end
+end

data/db/migrate/20260730000001_add_judge_temperature_to_completion_kit_runs.rb

@@ -0,0 +1,5 @@
+class AddJudgeTemperatureToCompletionKitRuns < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_runs, :judge_temperature, :float, default: 0.0
+  end
+end

data/db/migrate/20260730000002_create_completion_kit_prompt_serves.rb

@@ -0,0 +1,17 @@
+class CreateCompletionKitPromptServes < ActiveRecord::Migration[8.1]
+  def change
+    create_table :completion_kit_prompt_serves do |t|
+      t.integer :prompt_id
+      t.string :family_key, null: false
+      t.date :served_on, null: false
+      t.integer :serve_count, null: false, default: 0
+      t.datetime :last_served_at
+      t.timestamps
+    end
+
+    add_index :completion_kit_prompt_serves, [:prompt_id, :served_on],
+              unique: true, name: "index_ck_prompt_serves_on_prompt_and_day"
+    add_index :completion_kit_prompt_serves, [:family_key, :served_on],
+              name: "index_ck_prompt_serves_on_family_and_day"
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.24"
+  VERSION = "0.28.34"
 end

data/lib/completion_kit.rb

@@ -53,8 +53,11 @@ module CompletionKit
       yield(config) if block_given?
     end
 
+    # current_prompt_payload and render_current_prompt both come through here,
+    # so this is the one place a library-side fetch is counted. The REST show
+    # action records its own, since it can also resolve by numeric id.
     def current_prompt(identifier)
-      Prompt.current_for(identifier)
+      Prompt.current_for(identifier).tap { |prompt| PromptServe.record!(prompt) }
     end
 
     def current_prompt_payload(identifier)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.24
+  version: 0.28.34
 platform: ruby
 authors:
 - Damien Bastin
@@ -173,7 +173,7 @@ description: CompletionKit is a prompt testing platform that runs as a Rails eng
   or a standalone app. Run prompts against real datasets, score every output with
   an LLM judge against criteria you define, track prompt versions, and get AI-generated
   improvement suggestions grounded in your actual results. Includes a web UI, REST
-  API, and a built-in MCP server with 34 tools.
+  API, and a built-in MCP server with 53 tools.
 email:
 - damien@homemade.software
 executables: []
@@ -232,6 +232,7 @@ files:
 - app/jobs/completion_kit/model_discovery_job.rb
 - app/jobs/completion_kit/prompt_suggestion_job.rb
 - app/jobs/completion_kit/run_completion_check_job.rb
+- app/jobs/completion_kit/start_run_job.rb
 - app/mailers/completion_kit/application_mailer.rb
 - app/models/completion_kit/agreement.rb
 - app/models/completion_kit/application_record.rb
@@ -244,6 +245,7 @@ files:
 - app/models/completion_kit/metric_version.rb
 - app/models/completion_kit/model.rb
 - app/models/completion_kit/prompt.rb
+- app/models/completion_kit/prompt_serve.rb
 - app/models/completion_kit/provider_credential.rb
 - app/models/completion_kit/response.rb
 - app/models/completion_kit/review.rb
@@ -303,6 +305,7 @@ files:
 - app/services/completion_kit/prompt_improvement_validator.rb
 - app/services/completion_kit/promptfoo_importer.rb
 - app/services/completion_kit/provider_endpoint.rb
+- app/services/completion_kit/response_query.rb
 - app/services/completion_kit/run_comparison.rb
 - app/services/completion_kit/starter_metrics.rb
 - app/services/completion_kit/worker_health.rb
@@ -368,6 +371,7 @@ files:
 - app/views/completion_kit/runs/_compare_score.html.erb
 - app/views/completion_kit/runs/_form.html.erb
 - app/views/completion_kit/runs/_response_row.html.erb
+- app/views/completion_kit/runs/_responses_region.html.erb
 - app/views/completion_kit/runs/_row.html.erb
 - app/views/completion_kit/runs/_sort_toolbar.html.erb
 - app/views/completion_kit/runs/_status_header.html.erb
@@ -440,6 +444,9 @@ files:
 - db/migrate/20260708000001_add_api_version_to_completion_kit_provider_credentials.rb
 - db/migrate/20260713000001_add_catalog_model_count_to_completion_kit_provider_credentials.rb
 - db/migrate/20260715000001_dedupe_and_unique_index_metric_versions.rb
+- db/migrate/20260728000001_add_max_tokens_to_completion_kit_runs.rb
+- db/migrate/20260730000001_add_judge_temperature_to_completion_kit_runs.rb
+- db/migrate/20260730000002_create_completion_kit_prompt_serves.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb