completion-kit 0.28.24 → 0.28.32

data/app/services/completion_kit/mcp_tools/base.rb

@@ -1,6 +1,18 @@
 module CompletionKit
   module McpTools
     module Base
+      DEFAULT_PAGE_LIMIT = 50
+      MAX_PAGE_LIMIT = 500
+
+      def page_bounds(args)
+        limit = args["limit"].to_i
+        limit = DEFAULT_PAGE_LIMIT if limit <= 0
+        limit = MAX_PAGE_LIMIT if limit > MAX_PAGE_LIMIT
+        offset = args["offset"].to_i
+        offset = 0 if offset < 0
+        [limit, offset]
+      end
+
       def definitions
         self::TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
       end

data/app/services/completion_kit/mcp_tools/responses.rb

@@ -3,10 +3,32 @@ module CompletionKit
     module Responses
       extend Base
 
+      FIELDS_DESCRIPTION = "Only return these keys, keeping the payload small. Response keys: id, run_id, " \
+                           "input_data, response_text, expected_output, created_at, score, reviewed, reviews, " \
+                           "status, attempts, row_index, error. Prefix with \"reviews.\" to trim each review, " \
+                           "e.g. [\"score\", \"reviews.metric_name\", \"reviews.ai_score\"]. id is always included.".freeze
+
       TOOLS = {
         "responses_list" => {
-          description: "List responses for a run",
-          inputSchema: {type: "object", properties: {run_id: {type: "integer"}}, required: ["run_id"]},
+          description: "List responses for a run, in row order. Returns " \
+                       "{total, limit, offset, returned, responses}. Defaults to #{Base::DEFAULT_PAGE_LIMIT} rows " \
+                       "because full payloads are large: use \"fields\" to drop the bodies, \"min_score\"/\"max_score\" " \
+                       "to isolate low scorers, and sort \"score_asc\" to read the worst rows first. For per-metric " \
+                       "averages of the whole run use runs_get instead of aggregating here.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              run_id: {type: "integer"},
+              limit: {type: "integer", description: "Rows to return; defaults to #{Base::DEFAULT_PAGE_LIMIT}, capped at #{Base::MAX_PAGE_LIMIT}."},
+              offset: {type: "integer", description: "Rows to skip before returning results."},
+              status: {type: "string", description: "Filter by row status: pending, retrying, succeeded or failed."},
+              min_score: {type: "number", description: "Only rows whose average judge score is at least this."},
+              max_score: {type: "number", description: "Only rows whose average judge score is at most this. Use with sort \"score_asc\" for failure-mode analysis."},
+              sort: {type: "string", enum: ResponseQuery::SORTS, description: "Row order; defaults to \"id\"."},
+              fields: {type: "array", items: {type: "string"}, description: FIELDS_DESCRIPTION}
+            },
+            required: ["run_id"]
+          },
           handler: :list
         },
         "responses_get" => {
@@ -22,7 +44,20 @@ module CompletionKit
 
       def self.list(args)
         run = Run.find(args["run_id"])
-        text_result(run.responses.includes(:reviews).map(&:as_json))
+        query = ResponseQuery.new(
+          run,
+          status: args["status"], min_score: args["min_score"], max_score: args["max_score"],
+          sort: args["sort"], fields: args["fields"]
+        )
+        scope = query.relation
+        total = scope.count
+        limit, offset = page_bounds(args)
+        rows = scope.limit(limit).offset(offset).to_a
+
+        text_result({
+          total: total, limit: limit, offset: offset, returned: rows.length,
+          responses: rows.map { |response| query.serialize(response) }
+        })
       end
 
       def self.get(args)

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -3,6 +3,19 @@ module CompletionKit
     module Runs
       extend Base
 
+      TEMPERATURE_DESCRIPTION = "Sampling temperature for generation, 0 to 1. Defaults to the column default. " \
+                                "Reasoning models ignore it and the run is flagged temperature_ignored.".freeze
+
+      MAX_TOKENS_DESCRIPTION = "Cap on generated tokens per row. Leave unset to use the provider client's default, " \
+                               "which is what silently truncates long outputs and makes the judge score malformed " \
+                               "JSON. Set it to whatever the prompt uses in production so the eval matches.".freeze
+
+      JUDGE_TEMPERATURE_DESCRIPTION = "Sampling temperature for the judge, 0 to 1. Defaults to 0 so re-judging the " \
+                                      "same output gives the same score. Raise it only to measure judge variance " \
+                                      "on purpose; any value above 0 makes the run's scores irreproducible.".freeze
+
+      GENERATION_FIELDS = %w[temperature max_tokens judge_temperature].freeze
+
       TOOLS = {
         "runs_list" => {
           description: "List all runs",
@@ -10,7 +23,9 @@ module CompletionKit
           handler: :list
         },
         "runs_get" => {
-          description: "Get a run by ID",
+          description: "Get a run by ID, including \"metric_averages\": a per-metric breakdown with each metric's " \
+                       "average score (or pass rate for checks), how many rows it graded, and how many scored low. " \
+                       "Use this to find the metric dragging a prompt down without listing responses.",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :get
         },
@@ -21,6 +36,9 @@ module CompletionKit
             properties: {
               name: {type: "string"}, prompt_id: {type: "integer"},
               dataset_id: {type: "integer"}, judge_model: {type: "string"},
+              temperature: {type: "number", description: TEMPERATURE_DESCRIPTION},
+              max_tokens: {type: "integer", description: MAX_TOKENS_DESCRIPTION},
+              judge_temperature: {type: "number", description: JUDGE_TEMPERATURE_DESCRIPTION},
               output_column: {type: "string", description: "Dataset column to grade when prompt_id is omitted; defaults to \"actual_output\"."},
               expected_column: {type: "string", description: "Dataset column holding each row's answer key / ground truth, graded by checks with compare_to \"expected\" and passed to the judge; defaults to \"expected_output\"."},
               metric_ids: {type: "array", items: {type: "integer"}},
@@ -38,6 +56,9 @@ module CompletionKit
             properties: {
               id: {type: "integer"}, name: {type: "string"},
               dataset_id: {type: "integer"}, judge_model: {type: "string"},
+              temperature: {type: "number", description: TEMPERATURE_DESCRIPTION},
+              max_tokens: {type: "integer", description: MAX_TOKENS_DESCRIPTION},
+              judge_temperature: {type: "number", description: JUDGE_TEMPERATURE_DESCRIPTION},
               output_column: {type: "string"},
               expected_column: {type: "string"},
               metric_ids: {type: "array", items: {type: "integer"}},
@@ -84,7 +105,7 @@ module CompletionKit
       end
 
       def self.create(args)
-        run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column", "expected_column"))
+        run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column", "expected_column", *GENERATION_FIELDS))
         if run.save
           run.replace_metrics!(resolve_metric_ids(args))
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
@@ -96,7 +117,7 @@ module CompletionKit
 
       def self.update(args)
         run = Run.find(args["id"])
-        if run.update(args.except("id", "metric_ids", "metric_group_id", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column"))
+        if run.update(args.except("id", "metric_ids", "metric_group_id", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column", *GENERATION_FIELDS))
           run.replace_metrics!(resolve_metric_ids(args)) if args.key?("metric_ids") || args["metric_group_id"].present?
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
           text_result(run_payload(run.reload))
@@ -156,9 +177,16 @@ module CompletionKit
 
       def self.run_payload(run)
         json = run.as_json
-        return json unless run.metric_ids.empty?
+        warnings = []
+        if run.metric_ids.empty?
+          warnings << "No metrics are attached, so this run judges nothing. Attach metric_ids or a metric_group_id before generating."
+        end
+        if run.nondeterministic_judge?
+          warnings << "Judge temperature is #{run.judge_temperature}. Judging above 0 makes scores irreproducible: the same output can get a different score on a re-judge. Set judge_temperature to 0 unless you are deliberately measuring judge variance."
+        end
+        return json if warnings.empty?
 
-        json.merge("warning" => "No metrics are attached, so this run judges nothing. Attach metric_ids or a metric_group_id before generating.")
+        json.merge("warning" => warnings.join(" "))
       end
     end
   end

data/app/services/completion_kit/metric_improvement_validator.rb

@@ -87,9 +87,8 @@ module CompletionKit
 
     def rescore(response, candidate)
       run = response.run
-      config = ApiConfig.for_model(run.judge_model).merge(judge_model: run.judge_model)
       rubric_text = Metric.rubric_text_for(Metric.normalize_rubric_bands(candidate.rubric_bands))
-      result = JudgeService.new(config).evaluate(
+      result = JudgeService.new(run.judge_config).evaluate(
         response.response_text,
         response.expected_output,
         run.prompt&.template,

data/app/services/completion_kit/prompt_improvement_validator.rb

@@ -77,15 +77,14 @@ module CompletionKit
       client = LlmClient.for_model(model, ApiConfig.for_model(model))
       raise CompletionKit::ConfigurationError, client.configuration_errors.join(", ") unless client.configured?
 
-      text = client.generate_completion(rendered, model: model, temperature: @run.temperature)
+      text = client.generate_completion(rendered, **@run.generation_options(@run.prompt))
       raise StandardError, text if text.to_s.start_with?("Error:")
 
       text
     end
 
     def judge_score(response, new_text)
-      config = ApiConfig.for_model(@run.judge_model).merge(judge_model: @run.judge_model)
-      judge = JudgeService.new(config)
+      judge = JudgeService.new(@run.judge_config)
       scores = @run.metrics.select(&:llm_judge?).filter_map do |metric|
         judge.evaluate(
           new_text, response.expected_output, @candidate,

data/app/services/completion_kit/response_query.rb

@@ -0,0 +1,94 @@
+module CompletionKit
+  # Filtering, score-ordering and field projection for a run's responses,
+  # shared by the REST endpoint and the MCP tool so both surfaces answer
+  # "show me the worst rows, without the full bodies" the same way.
+  class ResponseQuery
+    SORTS = %w[id score_asc score_desc].freeze
+    REVIEW_FIELD_PREFIX = "reviews.".freeze
+
+    def initialize(run, status: nil, min_score: nil, max_score: nil, sort: nil, fields: nil)
+      @run = run
+      @status = status.presence
+      @min_score = min_score.presence&.to_f
+      @max_score = max_score.presence&.to_f
+      @sort = SORTS.include?(sort.to_s) ? sort.to_s : "id"
+      @fields = parse_fields(fields)
+    end
+
+    def relation
+      return base.order(:id) unless score_scoped?
+
+      ids = ordered_ids
+      base.where(id: ids).in_order_of(:id, ids)
+    end
+
+    def serialize(response)
+      project(response.as_json)
+    end
+
+    private
+
+    def base
+      scope = @run.responses.includes(:reviews)
+      @status ? scope.where(status: @status) : scope
+    end
+
+    def score_scoped?
+      filtering? || @sort != "id"
+    end
+
+    def filtering?
+      !@min_score.nil? || !@max_score.nil?
+    end
+
+    def ordered_ids
+      averages = score_averages
+      scored, unscored = base.order(:id).pluck(:id).partition { |id| averages.key?(id) }
+      kept = sort_by_score(scored.select { |id| in_range?(averages[id]) }, averages)
+      filtering? ? kept : kept + unscored
+    end
+
+    # Mirrors Response#score (mean of the row's judge scores, rounded to 2) so a
+    # min_score filter keeps exactly the rows whose reported score qualifies.
+    def score_averages
+      Review.where(response_id: base.select(:id))
+            .where.not(ai_score: nil)
+            .group(:response_id)
+            .pluck(:response_id, Arel.sql("AVG(ai_score)"))
+            .to_h { |id, avg| [id, avg.to_f.round(2)] }
+    end
+
+    def in_range?(average)
+      return false if @min_score && average < @min_score
+      return false if @max_score && average > @max_score
+
+      true
+    end
+
+    def sort_by_score(ids, averages)
+      return ids if @sort == "id"
+
+      sorted = ids.sort_by { |id| averages[id] }
+      @sort == "score_desc" ? sorted.reverse : sorted
+    end
+
+    def parse_fields(raw)
+      names = Array(raw).flat_map { |value| value.to_s.split(",") }.map(&:strip).reject(&:empty?)
+      {
+        top: names.reject { |name| name.start_with?(REVIEW_FIELD_PREFIX) }.map(&:to_sym),
+        reviews: names.select { |name| name.start_with?(REVIEW_FIELD_PREFIX) }
+                      .map { |name| name.delete_prefix(REVIEW_FIELD_PREFIX).to_sym }
+      }
+    end
+
+    def project(json)
+      return json if @fields[:top].empty? && @fields[:reviews].empty?
+
+      projected = json.slice(:id, *@fields[:top])
+      if @fields[:reviews].any?
+        projected[:reviews] = json[:reviews].map { |review| review.slice(*@fields[:reviews]) }
+      end
+      projected
+    end
+  end
+end

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -129,12 +129,13 @@
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs</p>
         <p class="ck-meta-copy">Create a new run.</p>
-        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>name</code>, <code>prompt_id</code>, <code>dataset_id</code>, <code>metric_ids</code>, <code>judge_model</code>, <code>temperature</code>, <code>output_column</code> (score existing outputs: omit <code>prompt_id</code> and grade a dataset column instead, default <code>actual_output</code>), <code>expected_column</code> (dataset column holding each row's answer key, given to the judge and to checks that compare against the row's expected value, default <code>expected_output</code>)</p>
+        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>name</code>, <code>prompt_id</code>, <code>dataset_id</code>, <code>metric_ids</code>, <code>judge_model</code>, <code>temperature</code>, <code>judge_temperature</code> (defaults to 0 so re-judging is reproducible; above 0 the same output can score differently), <code>max_tokens</code> (cap on generated tokens per row; leave unset for the provider default, set it to match production when your prompt's output is long enough to truncate), <code>output_column</code> (score existing outputs: omit <code>prompt_id</code> and grade a dataset column instead, default <code>actual_output</code>), <code>expected_column</code> (dataset column holding each row's answer key, given to the judge and to checks that compare against the row's expected value, default <code>expected_output</code>)</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"prompt_id\": 1, \"dataset_id\": 1, \"metric_ids\": [1, 2]}'" %>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:id</p>
         <p class="ck-meta-copy">Get a run with status, progress, response count, and average score.</p>
+        <p class="ck-api-params"><strong>Also returns:</strong>&ensp;<code>metric_averages</code>, a per-metric breakdown with each metric's average score (<code>pass_rate</code> for checks), the number of rows it graded, and <code>low_count</code>, how many scored below the medium quality threshold</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl #{base_url}/api/v1/runs/1 \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <%= render "completion_kit/api_reference/resource_list", title: "Your recent runs",
@@ -185,8 +186,9 @@
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:run_id/responses</p>
         <p class="ck-meta-copy">List responses for a run, including nested review scores.</p>
-        <p class="ck-api-params"><strong>Optional filters:</strong>&ensp;<code>status</code> (<code>pending</code>, <code>retrying</code>, <code>succeeded</code>, <code>failed</code>), plus <code>limit</code> and <code>offset</code></p>
-        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl #{base_url}/api/v1/runs/1/responses \\\n  -H \"Authorization: Bearer #{token}\"" %>
+        <p class="ck-api-params"><strong>Optional filters:</strong>&ensp;<code>status</code> (<code>pending</code>, <code>retrying</code>, <code>succeeded</code>, <code>failed</code>), <code>min_score</code>, <code>max_score</code>, <code>sort</code> (<code>id</code>, <code>score_asc</code>, <code>score_desc</code>), plus <code>limit</code> and <code>offset</code></p>
+        <p class="ck-api-params"><strong>Trim the payload:</strong>&ensp;<code>fields</code> takes a comma-separated list of response keys and always returns <code>id</code>. Prefix with <code>reviews.</code> to trim each nested review, e.g. <code>fields=score,reviews.metric_name,reviews.ai_score</code></p>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl \"#{base_url}/api/v1/runs/1/responses?sort=score_asc&limit=10&fields=id,score,reviews.metric_name,reviews.ai_score\" \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:run_id/responses/:id</p>

data/app/views/completion_kit/dashboard/_failing_checks_card.html.erb

@@ -1,19 +1,48 @@
+<% resolved = check_activity.sum { |day| day[:resolved] } %>
+<% passed = check_activity.sum { |day| day[:passed] } %>
 <div class="ck-card ck-stat-card ck-rise" id="ck-failing-checks-card" style="--rise-delay: 200ms;">
-  <p class="ck-kicker">Failing checks · last 7 days</p>
+  <p class="ck-kicker">Checks · 14D</p>
   <div class="ck-stat-card__body">
-    <span class="ck-stat-card__count<%= failing_checks[:count].positive? ? ' is-danger' : ' is-clean' %>"><%= failing_checks[:count] %></span>
+    <% if resolved.positive? %>
+      <% rate = passed.to_f / resolved %>
+      <span class="ck-stat-card__count is-<%= ck_pass_rate_kind(rate) %>"><%= (rate * 100).round %>%</span>
+      <span class="ck-stat-card__unit">passing</span>
+    <% else %>
+      <span class="ck-stat-card__metric ck-stat-card__metric--empty">Not run yet</span>
+    <% end %>
+  </div>
+
+  <div class="ck-sparkline ck-sparkline--compact" role="img"
+       aria-label="Daily check pass rate over the last 14 days">
+    <% check_activity.each do |day| %>
+      <% kind = day[:rate] && ck_pass_rate_kind(day[:rate]) %>
+      <span class="ck-sparkline__bar<%= " is-#{kind}" if kind && kind != :high %>"
+            style="height: <%= day[:rate] ? [(day[:rate] * 100).round, 6].max : 0 %>%;"
+            title="<%= day[:date].strftime('%b %-d') %>: <%= day[:rate] ? "#{(day[:rate] * 100).round}% of #{day[:resolved]}" : 'no checks' %>"></span>
+    <% end %>
   </div>
 
   <% if failing_checks[:items].any? %>
     <ul class="ck-failure-list">
-      <% failing_checks[:items].first(5).each do |item| %>
+      <% failing_checks[:items].first(2).each do |item| %>
         <li class="ck-failure-list__item">
-          <span class="ck-failure-list__surface ck-failure-list__surface--check">check</span>
-          <%= link_to item[:metric_name], completion_kit.run_path(item[:run]), class: "ck-link ck-failure-list__cause" %>
+          <span class="ck-failure-list__surface ck-failure-list__surface--check">fail</span>
+          <%= link_to item[:metric_name], completion_kit.run_path(item[:run]),
+                class: "ck-link ck-failure-list__cause", title: "Failed in #{item[:run].name}" %>
         </li>
       <% end %>
     </ul>
-  <% else %>
-    <div class="ck-stat-card__foot"><span>No failing checks this week.</span></div>
   <% end %>
+
+  <div class="ck-stat-card__foot">
+    <% if failing_checks[:count] > 2 %>
+      <span><%= failing_checks[:count] - 2 %> more failing.</span>
+    <% elsif failing_checks[:count].positive? %>
+      <span><%= pluralize(failing_checks[:count], "check") %> failing in the window.</span>
+    <% elsif resolved.positive? %>
+      <span>Every check passed in the window.</span>
+    <% else %>
+      <span>Add a check metric to a run to populate this.</span>
+    <% end %>
+  </div>
 </div>

data/app/views/completion_kit/dashboard/_failures_card.html.erb

@@ -1,5 +1,5 @@
 <div class="ck-card ck-stat-card ck-rise" id="ck-failures-card" style="--rise-delay: 180ms;">
-  <p class="ck-kicker">Failures · last 7 days</p>
+  <p class="ck-kicker">Failures · 7D</p>
   <div class="ck-stat-card__body">
     <span class="ck-stat-card__count<%= failures[:count].positive? ? ' is-danger' : ' is-clean' %>"><%= failures[:count] %></span>
   </div>
@@ -8,11 +8,12 @@
     <ul class="ck-failure-list">
       <% failures[:items].each do |item| %>
         <li class="ck-failure-list__item">
-          <span class="ck-failure-list__surface ck-failure-list__surface--<%= item[:surface] %>"><%= item[:surface] %></span>
+          <span class="ck-failure-list__surface ck-failure-list__surface--<%= item[:surface] %>"><%= item[:surface] == "generation" ? "gen" : item[:surface] %></span>
           <% if item[:run] %>
-            <%= link_to item[:cause], completion_kit.run_path(item[:run]), class: "ck-link ck-failure-list__cause" %>
+            <%= link_to item[:run].name, completion_kit.run_path(item[:run]),
+                  class: "ck-link ck-failure-list__cause", title: item[:cause] %>
           <% else %>
-            <span class="ck-failure-list__cause"><%= item[:cause] %></span>
+            <span class="ck-failure-list__cause" title="<%= item[:cause] %>">Deleted run</span>
           <% end %>
           <%= button_to completion_kit.dashboard_dismissals_path,
                 params: { dashboard_dismissal: { dismissable_type: item[:record].class.name,
@@ -26,7 +27,7 @@
 
   <% if failures[:items].empty? || ignored_failures.any? %>
     <div class="ck-stat-card__foot ck-stat-card__foot--split">
-      <span><% if failures[:items].empty? %>All clear — nothing failed this week.<% end %></span>
+      <span><% if failures[:items].empty? %>All clear. Nothing failed this week.<% end %></span>
       <% if ignored_failures.any? %>
         <details class="ck-flyout">
           <summary class="ck-flyout__toggle"><%= ignored_failures.size %> ignored</summary>

data/app/views/completion_kit/dashboard/_worst_metric_card.html.erb

@@ -1,5 +1,5 @@
 <div class="ck-card ck-stat-card ck-rise" id="ck-worst-metric-card" style="--rise-delay: 120ms;">
-  <p class="ck-kicker">Worst metric · last 7 days</p>
+  <p class="ck-kicker">Worst metric · 7D</p>
   <div class="ck-stat-card__body">
     <% if worst_metric %>
       <span class="ck-stat-card__metric"><%= worst_metric[:name] %></span>

data/app/views/completion_kit/dashboard/show.html.erb

@@ -28,20 +28,30 @@
 </nav>
 
 <% if @activity %>
-  <div class="ck-grid ck-grid--cards ck-grid--cards-4 ck-pulse-grid">
+  <div class="ck-grid ck-grid--cards <%= @failing_checks ? "ck-grid--cards-4" : "ck-grid--cards-3" %> ck-pulse-grid">
     <div class="ck-card ck-stat-card ck-rise" style="--rise-delay: 60ms;">
-      <p class="ck-kicker">Activity · last 14 days</p>
+      <p class="ck-kicker">Activity · 14D</p>
+      <% activity_total = @activity.sum { |d| d[:count] } %>
       <% activity_max = @activity.map { |d| d[:count] }.max %>
-      <div class="ck-sparkline" role="img" aria-label="<%= @activity.sum { |d| d[:count] } %> runs over the last 14 days">
+      <div class="ck-stat-card__body">
+        <span class="ck-stat-card__count"><%= activity_total %></span>
+        <span class="ck-stat-card__unit"><%= "run".pluralize(activity_total) %></span>
+      </div>
+      <div class="ck-sparkline ck-sparkline--compact" role="img" aria-label="<%= activity_total %> runs over the last 14 days">
         <% @activity.each do |day| %>
           <span class="ck-sparkline__bar<%= ' is-peak' if activity_max.to_i.positive? && day[:count] == activity_max %>"
-                style="height: <%= activity_max.to_i.zero? ? 0 : (day[:count] * 100.0 / activity_max).round %>%"
+                style="height: <%= activity_max.to_i.zero? ? 0 : [(day[:count] * 100.0 / activity_max).round, 6].max %>%"
                 title="<%= day[:date].strftime('%b %-d') %>: <%= day[:count] %> run<%= 's' unless day[:count] == 1 %>"></span>
         <% end %>
       </div>
-      <p class="ck-stat-card__foot">
-        <span class="ck-stat-card__figure"><%= @activity.sum { |d| d[:count] } %></span> runs in the window
-      </p>
+      <div class="ck-stat-card__foot">
+        <% if activity_total.positive? %>
+          <% peak = @activity.max_by { |d| d[:count] } %>
+          <span>Busiest day <%= peak[:date].strftime("%b %-d") %> · <%= peak[:count] %></span>
+        <% else %>
+          <span>No runs in the window.</span>
+        <% end %>
+      </div>
     </div>
 
     <%= render "completion_kit/dashboard/worst_metric_card",
@@ -50,8 +60,10 @@
     <%= render "completion_kit/dashboard/failures_card",
                failures: @failures, ignored_failures: @ignored_failures %>
 
-    <%= render "completion_kit/dashboard/failing_checks_card",
-               failing_checks: @failing_checks %>
+    <% if @failing_checks %>
+      <%= render "completion_kit/dashboard/failing_checks_card",
+                 failing_checks: @failing_checks, check_activity: @check_activity %>
+    <% end %>
   </div>
 
   <div class="ck-card ck-card--spaced ck-rise" style="--rise-delay: 240ms;">
@@ -80,6 +92,33 @@
   </div>
 <% end %>
 
+<div class="ck-card ck-card--spaced ck-rise" id="top_served_card" style="--rise-delay: 280ms;">
+  <p class="ck-kicker">Prompts served · 7D</p>
+  <% if @top_served.any? %>
+    <% serve_max = @serve_activity.map { |d| d[:count] }.max %>
+    <div class="ck-sparkline" role="img" aria-label="<%= @serve_activity.sum { |d| d[:count] } %> prompt fetches over the last 14 days">
+      <% @serve_activity.each do |day| %>
+        <span class="ck-sparkline__bar<%= ' is-peak' if serve_max.positive? && day[:count] == serve_max %>"
+              style="height: <%= serve_max.positive? ? [(day[:count] * 100.0 / serve_max).round, 4].max : 4 %>%;"
+              title="<%= day[:date].strftime('%b %-d') %>: <%= day[:count] %>"></span>
+      <% end %>
+    </div>
+    <ul class="ck-improvements">
+      <% @top_served.each do |row| %>
+        <li class="ck-improvement">
+          <%= link_to row[:prompt].name, prompt_path(row[:prompt]), class: "ck-improvement__name ck-link" %>
+          <span class="ck-improvement__versions"><%= row[:prompt].version_label %></span>
+          <span class="ck-improvement__scores"><span class="ck-serve-figure"><%= number_with_delimiter(row[:count]) %></span> fetches</span>
+        </li>
+      <% end %>
+    </ul>
+  <% else %>
+    <p class="ck-improvements__empty">
+      No prompts fetched in the last 7 days. Once your application reads a published prompt from the API, the prompts it uses most show up here.
+    </p>
+  <% end %>
+</div>
+
 <section class="ck-card--spaced">
   <div class="ck-split">
     <h2 class="ck-section-title">Recent runs</h2>

data/app/views/completion_kit/prompts/show.html.erb

@@ -18,6 +18,19 @@
       <code class="ck-endpoint__url" id="prompt_endpoint"><%= request.base_url %><%= api_v1_prompt_path(@prompt.slug) %></code>
       <button type="button" class="ck-icon-btn" title="Copy endpoint" aria-label="Copy endpoint" onclick="navigator.clipboard.writeText(document.getElementById('prompt_endpoint').textContent)"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="14" height="14" aria-hidden="true"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"/><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"/></svg></button>
     </div>
+    <p class="ck-meta-copy" id="prompt_serves">
+      <% if @serve_summary[:total].positive? %>
+        Served <span class="ck-serve-figure"><%= number_with_delimiter(@serve_summary[:total]) %></span>
+        <%= "time".pluralize(@serve_summary[:total]) %><% if @serve_summary[7] < @serve_summary[:total] %>,
+        <%= number_with_delimiter(@serve_summary[7]) %> in the last 7 days<% end %>.
+        <% if @serve_summary[:last_served_at] %>
+          Last fetched <time data-relative-time datetime="<%= @serve_summary[:last_served_at].utc.iso8601 %>"><%= time_ago_in_words(@serve_summary[:last_served_at]) %> ago</time>.
+        <% end %>
+      <% else %>
+        Not fetched yet. Counts appear here once something requests this prompt from the URL above.
+      <% end %>
+    </p>
+
     <% if @prompt.tags.any? %>
       <div class="tag-marks-row">
         <%= render "completion_kit/tags/marks", tags: @prompt.tags %>

data/app/views/completion_kit/responses/show.html.erb

@@ -97,7 +97,7 @@
 <% if @reviews.any? %>
   <section class="ck-card--spaced">
     <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Review</p>
+      <p class="ck-kicker">Judge's review</p>
       <% if @run.judge_model.present? %>
         <span class="ck-chip ck-chip--soft" style="text-transform: none;"><%= @run.judge_model %></span>
       <% end %>

data/app/views/completion_kit/runs/_form.html.erb

@@ -93,6 +93,14 @@
       </div>
     </div>
 
+    <div class="ck-field">
+      <label class="ck-label" for="run_max_tokens" style="position: relative;">
+        Max tokens<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">The ceiling on how many tokens the model may generate per row. Leave it blank to use the provider's default. Set it when your prompt produces long output: if generation hits the cap the response is cut off mid-sentence, and the judge scores the truncation rather than the prompt. Match whatever your production caller uses so the evaluation reflects reality.</span>
+      </label>
+      <%= form.number_field :max_tokens, min: 1, step: 1, class: "ck-input", id: "run_max_tokens", placeholder: "Provider default", **ck_field_aria(form, :max_tokens) %>
+      <%= ck_field_error(form, :max_tokens) %>
+    </div>
+
     <div class="ck-field" id="judge-field">
       <label class="ck-label" for="run_judge_model" style="position: relative;">
         Judge model<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">Judge models score generated responses against your metrics. Pick one when configuring a run.</span>
@@ -118,6 +126,15 @@
       <% end %>
     </div>
 
+    <div class="ck-field" id="judge-temperature-field">
+      <label class="ck-label" for="run_judge_temperature" style="position: relative;">
+        Judge temperature<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">How random the judge is when it scores. This is separate from the generation temperature above. Keep it at 0 so re-judging the same output gives the same score, which is what makes one run comparable to another. Raise it only when you are deliberately measuring how much the judge disagrees with itself.</span>
+      </label>
+      <%= form.number_field :judge_temperature, in: 0..1, step: 0.1, class: "ck-input", id: "run_judge_temperature", **ck_field_aria(form, :judge_temperature) %>
+      <%= ck_field_error(form, :judge_temperature) %>
+      <p class="ck-field-hint" id="judge-temperature-hint">0 is deterministic and recommended. Above 0, the same output can score differently on a re-judge.</p>
+    </div>
+
     <div class="ck-field" id="metrics-field">
       <p class="ck-label" id="run_metrics_label">Metrics</p>
       <% if @all_metrics.empty? %>

data/app/views/completion_kit/runs/_responses_region.html.erb

@@ -0,0 +1,20 @@
+<div id="run_responses_region">
+  <p class="ck-kicker" style="margin-top: 1.5rem;<%= ' display:none;' unless responses.any? %>" id="responses_kicker">Responses</p>
+  <table class="ck-results-table ck-responses-table"<%= ' hidden'.html_safe unless responses.any? %> id="responses_table">
+    <thead>
+      <tr>
+        <th scope="col">#</th>
+        <th scope="col">Response</th>
+        <th scope="col">Metrics</th>
+        <th scope="col">Avg score</th>
+        <th scope="col">Status</th>
+        <th scope="col"></th>
+      </tr>
+    </thead>
+    <tbody id="run_responses">
+      <% responses.each_with_index do |response, idx| %>
+        <%= render "response_row", run: run, response: response, index: responses_offset + idx + 1 %>
+      <% end %>
+    </tbody>
+  </table>
+</div>

data/app/views/completion_kit/runs/_status_header.html.erb

@@ -12,6 +12,28 @@
     </div>
   <% end %>
 
+  <% if run.scores_at_ceiling? %>
+    <div class="ck-score-ceiling-banner" role="status">
+      <div class="ck-score-ceiling-banner__body">
+        <p class="ck-kicker">Scores are clustered near the top</p>
+        <p class="ck-meta-copy">
+          A near-perfect average usually means the judge is not separating good output from bad, not that every
+          output is good. Check it against your own reading before you trust the number.
+        </p>
+      </div>
+      <div class="ck-score-ceiling-banner__actions">
+        <%= link_to "Worst first", run_path(run, sort: "score_asc"),
+              class: ck_button_classes(:light, variant: :outline) + " ck-button--sm",
+              title: "Sort the responses lowest score first to see whether the judge ever penalises anything." %>
+        <% if CompletionKit.config.judge_agreement_enabled && run.calibratable_metric %>
+          <%= link_to "Check the judge", metric_path(run.calibratable_metric, anchor: "agreement"),
+                class: ck_button_classes(:dark) + " ck-button--sm",
+                title: "Score a few outputs yourself and see how often the judge agrees with you." %>
+        <% end %>
+      </div>
+    </div>
+  <% end %>
+
   <section class="ck-page-header">
     <div>
       <span class="ck-status-badge ck-status-badge--<%= run.status %>">