completion-kit 0.28.24 → 0.28.32

data/app/views/completion_kit/runs/_status_panel.html.erb

@@ -77,6 +77,24 @@
             </div>
           <% end %>
 
+          <% if has_llm_metric && CompletionKit.config.judge_agreement_enabled %>
+            <% agreement = run.judge_agreement %>
+            <div class="ck-run-status__cell">
+              <p class="ck-run-status__metric-label">Judge agreement</p>
+              <div class="ck-run-status__cell-value <%= agreement ? "ck-run-status__cell-value--split" : "ck-run-status__cell-value--stack" %>">
+                <% if agreement %>
+                  <span class="<%= ck_badge_classes(ck_pass_rate_kind(agreement[:rate])) %> ck-badge--lg"><%= (agreement[:rate] * 100).round %>%</span>
+                  <span class="ck-run-status__summary-text"><%= pluralize(agreement[:sample_size], "label") %></span>
+                <% else %>
+                  <span class="ck-run-status__cell-unverified">Unverified</span>
+                  <% if run.calibratable_metric %>
+                    <%= link_to "Check the judge &rarr;".html_safe, metric_path(run.calibratable_metric, anchor: "agreement"), class: "ck-cal-link" %>
+                  <% end %>
+                <% end %>
+              </div>
+            </div>
+          <% end %>
+
           <% if has_checks %>
             <div class="ck-run-status__cell">
               <p class="ck-run-status__metric-label">Checks passed</p>

data/app/views/completion_kit/runs/show.html.erb

@@ -25,20 +25,26 @@
           This run was scored against metric versions that are no longer live.
           <% stale_summary.values.each_with_index do |s, i| %>
             <%= ", " if i > 0 %><strong><%= s[:metric_name] %></strong> (scored by <%= s[:scored_labels].join(", ") %>; live is <%= s[:current_label] %>)<% end %>.
-          Re-run to refresh the scores with the current metrics.
+          Re-grade to refresh the scores with the current metrics.
         </p>
       </div>
       <% if @run.status == "completed" %>
-        <%= button_to "Re-run from scratch",
-              rerun_run_path(@run), method: :post,
-              class: ck_button_classes(:light, variant: :outline), form_class: "inline-block",
-              title: "Create a new run that regenerates responses and grades them with the current metrics.",
-              data: { turbo_confirm: "Create a new run with fresh responses and the current metrics? The original run stays as a record." } %>
-        <%= button_to "Re-grade with current metrics",
-              regrade_run_path(@run), method: :post,
-              class: ck_button_classes(:dark), form_class: "inline-block",
-              title: "Re-grade this run's existing responses against the current metrics. Faster and cheaper than re-running.",
-              data: { turbo_confirm: "Re-grade this run's existing responses against the current metrics?" } %>
+        <div class="ck-banner-actions">
+          <div class="ck-banner-action">
+            <%= button_to "Re-grade with current metrics",
+                  regrade_run_path(@run), method: :post,
+                  class: ck_button_classes(:dark), form_class: "inline-block",
+                  data: { turbo_confirm: "Re-grade this run's existing responses against the current metrics?" } %>
+            <p class="ck-banner-action__note">Re-scores the responses you already have. No generation cost.</p>
+          </div>
+          <div class="ck-banner-action">
+            <%= button_to "Re-run from scratch",
+                  rerun_run_path(@run), method: :post,
+                  class: ck_button_classes(:light, variant: :outline), form_class: "inline-block",
+                  data: { turbo_confirm: "Create a new run with fresh responses and the current metrics? The original run stays as a record." } %>
+            <p class="ck-banner-action__note">Generates fresh responses first. Costs a full run.</p>
+          </div>
+        </div>
       <% end %>
     </div>
   <% end %>
@@ -83,6 +89,17 @@
       <span class="ck-run-config__warn" style="color: var(--ck-dim);" title="The model rejected the temperature parameter, so CompletionKit re-sent the request without it.">ignored by model</span>
     <% end %>
   </div>
+  <div class="ck-run-config__row">
+    <span class="ck-run-config__key">Max tokens</span>
+    <span><%= @run.max_tokens || "Provider default" %></span>
+  </div>
+  <div class="ck-run-config__row">
+    <span class="ck-run-config__key">Judge temperature</span>
+    <span><%= @run.judge_temperature %></span>
+    <% if @run.nondeterministic_judge? %>
+      <span class="ck-run-config__warn" style="color: var(--ck-warning);" title="Judging above temperature 0 makes scores irreproducible. The same output can get a different score on a re-judge.">scores not reproducible</span>
+    <% end %>
+  </div>
 </div>
 
 <% if @run.prompt %>
@@ -109,7 +126,7 @@
     <div class="ck-prompt-preview__header">
       <p class="ck-kicker">Output source</p>
     </div>
-    <p class="ck-prompt-preview__text">Dataset column <code><%= @run.output_column.presence || "actual_output" %></code> — no prompt generated these outputs.</p>
+    <p class="ck-prompt-preview__text">Dataset column <code><%= @run.output_column.presence || "actual_output" %></code>. No prompt generated these outputs.</p>
   </div>
 <% end %>
 
@@ -186,24 +203,7 @@
 
 <%= render "sort_toolbar", run: @run %>
 
-<p class="ck-kicker" style="margin-top: 1.5rem;<%= ' display:none;' unless @responses.any? %>" id="responses_kicker">Responses</p>
-<table class="ck-results-table ck-responses-table"<%= ' hidden'.html_safe unless @responses.any? %> id="responses_table">
-  <thead>
-    <tr>
-      <th scope="col">#</th>
-      <th scope="col">Response</th>
-      <th scope="col">Metrics</th>
-      <th scope="col">Avg score</th>
-      <th scope="col">Status</th>
-      <th scope="col"></th>
-    </tr>
-  </thead>
-  <tbody id="run_responses">
-    <% @responses.each_with_index do |response, idx| %>
-      <%= render "response_row", run: @run, response: response, index: @responses_offset + idx + 1 %>
-    <% end %>
-  </tbody>
-</table>
+<%= render "responses_region", run: @run, responses: @responses, responses_offset: @responses_offset %>
 
 <% if @responses_total_pages > 1 %>
   <nav class="ck-pagination" aria-label="Response pages">

data/db/migrate/20260728000001_add_max_tokens_to_completion_kit_runs.rb

@@ -0,0 +1,5 @@
+class AddMaxTokensToCompletionKitRuns < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_runs, :max_tokens, :integer
+  end
+end

data/db/migrate/20260730000001_add_judge_temperature_to_completion_kit_runs.rb

@@ -0,0 +1,5 @@
+class AddJudgeTemperatureToCompletionKitRuns < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_runs, :judge_temperature, :float, default: 0.0
+  end
+end

data/db/migrate/20260730000002_create_completion_kit_prompt_serves.rb

@@ -0,0 +1,17 @@
+class CreateCompletionKitPromptServes < ActiveRecord::Migration[8.1]
+  def change
+    create_table :completion_kit_prompt_serves do |t|
+      t.integer :prompt_id
+      t.string :family_key, null: false
+      t.date :served_on, null: false
+      t.integer :serve_count, null: false, default: 0
+      t.datetime :last_served_at
+      t.timestamps
+    end
+
+    add_index :completion_kit_prompt_serves, [:prompt_id, :served_on],
+              unique: true, name: "index_ck_prompt_serves_on_prompt_and_day"
+    add_index :completion_kit_prompt_serves, [:family_key, :served_on],
+              name: "index_ck_prompt_serves_on_family_and_day"
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.24"
+  VERSION = "0.28.32"
 end

data/lib/completion_kit.rb

@@ -53,8 +53,11 @@ module CompletionKit
       yield(config) if block_given?
     end
 
+    # current_prompt_payload and render_current_prompt both come through here,
+    # so this is the one place a library-side fetch is counted. The REST show
+    # action records its own, since it can also resolve by numeric id.
     def current_prompt(identifier)
-      Prompt.current_for(identifier)
+      Prompt.current_for(identifier).tap { |prompt| PromptServe.record!(prompt) }
     end
 
     def current_prompt_payload(identifier)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.24
+  version: 0.28.32
 platform: ruby
 authors:
 - Damien Bastin
@@ -232,6 +232,7 @@ files:
 - app/jobs/completion_kit/model_discovery_job.rb
 - app/jobs/completion_kit/prompt_suggestion_job.rb
 - app/jobs/completion_kit/run_completion_check_job.rb
+- app/jobs/completion_kit/start_run_job.rb
 - app/mailers/completion_kit/application_mailer.rb
 - app/models/completion_kit/agreement.rb
 - app/models/completion_kit/application_record.rb
@@ -244,6 +245,7 @@ files:
 - app/models/completion_kit/metric_version.rb
 - app/models/completion_kit/model.rb
 - app/models/completion_kit/prompt.rb
+- app/models/completion_kit/prompt_serve.rb
 - app/models/completion_kit/provider_credential.rb
 - app/models/completion_kit/response.rb
 - app/models/completion_kit/review.rb
@@ -303,6 +305,7 @@ files:
 - app/services/completion_kit/prompt_improvement_validator.rb
 - app/services/completion_kit/promptfoo_importer.rb
 - app/services/completion_kit/provider_endpoint.rb
+- app/services/completion_kit/response_query.rb
 - app/services/completion_kit/run_comparison.rb
 - app/services/completion_kit/starter_metrics.rb
 - app/services/completion_kit/worker_health.rb
@@ -368,6 +371,7 @@ files:
 - app/views/completion_kit/runs/_compare_score.html.erb
 - app/views/completion_kit/runs/_form.html.erb
 - app/views/completion_kit/runs/_response_row.html.erb
+- app/views/completion_kit/runs/_responses_region.html.erb
 - app/views/completion_kit/runs/_row.html.erb
 - app/views/completion_kit/runs/_sort_toolbar.html.erb
 - app/views/completion_kit/runs/_status_header.html.erb
@@ -440,6 +444,9 @@ files:
 - db/migrate/20260708000001_add_api_version_to_completion_kit_provider_credentials.rb
 - db/migrate/20260713000001_add_catalog_model_count_to_completion_kit_provider_credentials.rb
 - db/migrate/20260715000001_dedupe_and_unique_index_metric_versions.rb
+- db/migrate/20260728000001_add_max_tokens_to_completion_kit_runs.rb
+- db/migrate/20260730000001_add_judge_temperature_to_completion_kit_runs.rb
+- db/migrate/20260730000002_create_completion_kit_prompt_serves.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb