completion-kit 0.28.23 → 0.28.32

data/app/services/completion_kit/dashboard_stats.rb

@@ -16,6 +16,35 @@ module CompletionKit
       end
     end
 
+    # Prompts fetched most often by consumers in the window, with the daily
+    # counts behind each so the caller can draw a trend. Serving is independent
+    # of evaluating, so this deliberately does not go through display_scoped.
+    def self.top_served(since:, limit: 5)
+      rows = PromptServe.where("served_on >= ?", since.to_date)
+                        .group(:prompt_id)
+                        .pluck(Arel.sql("prompt_id"), Arel.sql("SUM(serve_count)"), Arel.sql("MAX(last_served_at)"))
+      return [] if rows.empty?
+
+      prompts = Prompt.where(id: rows.map(&:first)).index_by(&:id)
+      rows.filter_map do |prompt_id, total, last_at|
+        prompt = prompts[prompt_id]
+        next unless prompt
+
+        { prompt: prompt, count: total.to_i, last_served_at: last_at }
+      end.sort_by { |row| -row[:count] }.first(limit)
+    end
+
+    # One entry per day, zero-filled, matching the shape `activity` returns so
+    # the same sparkline markup renders it.
+    def self.serve_activity(days: 14)
+      since = (days - 1).days.ago.to_date
+      counts = PromptServe.where("served_on >= ?", since).group(:served_on).sum(:serve_count)
+      (0...days).map do |offset|
+        date = since + offset
+        { date: date, count: counts[date] || counts[date.to_s] || 0 }
+      end
+    end
+
     # The metric with the lowest average judge score across succeeded reviews
     # in the window — the prompt-engineering target. Dismissed metrics are
     # skipped while their average holds at or above the score snapshotted when
@@ -70,6 +99,31 @@ module CompletionKit
       (resolved.where(passed: true).count.to_f / total).round(2)
     end
 
+    # Daily pass rate for deterministic checks across the trailing window,
+    # zero-filled and oldest first. `rate` is nil on days nothing resolved, so
+    # a quiet day stays distinguishable from a day everything failed.
+    def self.check_activity(days: 14)
+      since = (days - 1).days.ago.to_date
+      counts = Review.joins(:response)
+                     .where.not(passed: nil)
+                     .where("completion_kit_reviews.created_at >= ?", since.beginning_of_day)
+                     .where(completion_kit_responses: { run_id: Run.visible_run_ids })
+                     .group(Arel.sql("DATE(completion_kit_reviews.created_at)"), :passed)
+                     .count
+      by_day = counts.each_with_object({}) do |((day, passed), total), acc|
+        bucket = acc[day.to_s] ||= { passed: 0, resolved: 0 }
+        bucket[:passed] += total if passed
+        bucket[:resolved] += total
+      end
+
+      (0...days).map do |offset|
+        date = since + offset
+        bucket = by_day[date.to_s] || { passed: 0, resolved: 0 }
+        rate = bucket[:resolved].zero? ? nil : (bucket[:passed].to_f / bucket[:resolved]).round(2)
+        { date: date, resolved: bucket[:resolved], passed: bucket[:passed], rate: rate }
+      end
+    end
+
     def self.failing_checks(since:)
       reviews = Review.where(passed: false)
                       .where("completion_kit_reviews.created_at >= ?", since)

data/app/services/completion_kit/judge_service.rb

@@ -4,9 +4,12 @@ module CompletionKit
   class JudgeParseError < StandardError; end
 
   class JudgeService
+    DEFAULT_TEMPERATURE = 0.0
+
     def initialize(config = {})
       @config = config
       @judge_model = config[:judge_model].presence || ApiConfig.default_judge_model
+      @judge_temperature = config[:judge_temperature] || DEFAULT_TEMPERATURE
       @judge_client = LlmClient.for_model(@judge_model, ApiConfig.for_model(@judge_model))
     end
 
@@ -19,7 +22,7 @@ module CompletionKit
         input_data: input_data,
         human_examples: human_examples)
 
-      response = @judge_client.generate_completion(judge_prompt, model: @judge_model)
+      response = @judge_client.generate_completion(judge_prompt, model: @judge_model, temperature: @judge_temperature)
       raise CompletionKit::ProviderError.from_client_error(response) if response.start_with?("Error:")
       parse_judge_response(response)
     end

data/app/services/completion_kit/mcp_tools/base.rb

@@ -1,6 +1,18 @@
 module CompletionKit
   module McpTools
     module Base
+      DEFAULT_PAGE_LIMIT = 50
+      MAX_PAGE_LIMIT = 500
+
+      def page_bounds(args)
+        limit = args["limit"].to_i
+        limit = DEFAULT_PAGE_LIMIT if limit <= 0
+        limit = MAX_PAGE_LIMIT if limit > MAX_PAGE_LIMIT
+        offset = args["offset"].to_i
+        offset = 0 if offset < 0
+        [limit, offset]
+      end
+
       def definitions
         self::TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
       end

data/app/services/completion_kit/mcp_tools/responses.rb

@@ -3,10 +3,32 @@ module CompletionKit
     module Responses
       extend Base
 
+      FIELDS_DESCRIPTION = "Only return these keys, keeping the payload small. Response keys: id, run_id, " \
+                           "input_data, response_text, expected_output, created_at, score, reviewed, reviews, " \
+                           "status, attempts, row_index, error. Prefix with \"reviews.\" to trim each review, " \
+                           "e.g. [\"score\", \"reviews.metric_name\", \"reviews.ai_score\"]. id is always included.".freeze
+
       TOOLS = {
         "responses_list" => {
-          description: "List responses for a run",
-          inputSchema: {type: "object", properties: {run_id: {type: "integer"}}, required: ["run_id"]},
+          description: "List responses for a run, in row order. Returns " \
+                       "{total, limit, offset, returned, responses}. Defaults to #{Base::DEFAULT_PAGE_LIMIT} rows " \
+                       "because full payloads are large: use \"fields\" to drop the bodies, \"min_score\"/\"max_score\" " \
+                       "to isolate low scorers, and sort \"score_asc\" to read the worst rows first. For per-metric " \
+                       "averages of the whole run use runs_get instead of aggregating here.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              run_id: {type: "integer"},
+              limit: {type: "integer", description: "Rows to return; defaults to #{Base::DEFAULT_PAGE_LIMIT}, capped at #{Base::MAX_PAGE_LIMIT}."},
+              offset: {type: "integer", description: "Rows to skip before returning results."},
+              status: {type: "string", description: "Filter by row status: pending, retrying, succeeded or failed."},
+              min_score: {type: "number", description: "Only rows whose average judge score is at least this."},
+              max_score: {type: "number", description: "Only rows whose average judge score is at most this. Use with sort \"score_asc\" for failure-mode analysis."},
+              sort: {type: "string", enum: ResponseQuery::SORTS, description: "Row order; defaults to \"id\"."},
+              fields: {type: "array", items: {type: "string"}, description: FIELDS_DESCRIPTION}
+            },
+            required: ["run_id"]
+          },
           handler: :list
         },
         "responses_get" => {
@@ -22,7 +44,20 @@ module CompletionKit
 
       def self.list(args)
         run = Run.find(args["run_id"])
-        text_result(run.responses.includes(:reviews).map(&:as_json))
+        query = ResponseQuery.new(
+          run,
+          status: args["status"], min_score: args["min_score"], max_score: args["max_score"],
+          sort: args["sort"], fields: args["fields"]
+        )
+        scope = query.relation
+        total = scope.count
+        limit, offset = page_bounds(args)
+        rows = scope.limit(limit).offset(offset).to_a
+
+        text_result({
+          total: total, limit: limit, offset: offset, returned: rows.length,
+          responses: rows.map { |response| query.serialize(response) }
+        })
       end
 
       def self.get(args)

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -3,6 +3,19 @@ module CompletionKit
     module Runs
       extend Base
 
+      TEMPERATURE_DESCRIPTION = "Sampling temperature for generation, 0 to 1. Defaults to the column default. " \
+                                "Reasoning models ignore it and the run is flagged temperature_ignored.".freeze
+
+      MAX_TOKENS_DESCRIPTION = "Cap on generated tokens per row. Leave unset to use the provider client's default, " \
+                               "which is what silently truncates long outputs and makes the judge score malformed " \
+                               "JSON. Set it to whatever the prompt uses in production so the eval matches.".freeze
+
+      JUDGE_TEMPERATURE_DESCRIPTION = "Sampling temperature for the judge, 0 to 1. Defaults to 0 so re-judging the " \
+                                      "same output gives the same score. Raise it only to measure judge variance " \
+                                      "on purpose; any value above 0 makes the run's scores irreproducible.".freeze
+
+      GENERATION_FIELDS = %w[temperature max_tokens judge_temperature].freeze
+
       TOOLS = {
         "runs_list" => {
           description: "List all runs",
@@ -10,7 +23,9 @@ module CompletionKit
           handler: :list
         },
         "runs_get" => {
-          description: "Get a run by ID",
+          description: "Get a run by ID, including \"metric_averages\": a per-metric breakdown with each metric's " \
+                       "average score (or pass rate for checks), how many rows it graded, and how many scored low. " \
+                       "Use this to find the metric dragging a prompt down without listing responses.",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :get
         },
@@ -21,6 +36,9 @@ module CompletionKit
             properties: {
               name: {type: "string"}, prompt_id: {type: "integer"},
               dataset_id: {type: "integer"}, judge_model: {type: "string"},
+              temperature: {type: "number", description: TEMPERATURE_DESCRIPTION},
+              max_tokens: {type: "integer", description: MAX_TOKENS_DESCRIPTION},
+              judge_temperature: {type: "number", description: JUDGE_TEMPERATURE_DESCRIPTION},
               output_column: {type: "string", description: "Dataset column to grade when prompt_id is omitted; defaults to \"actual_output\"."},
               expected_column: {type: "string", description: "Dataset column holding each row's answer key / ground truth, graded by checks with compare_to \"expected\" and passed to the judge; defaults to \"expected_output\"."},
               metric_ids: {type: "array", items: {type: "integer"}},
@@ -38,6 +56,9 @@ module CompletionKit
             properties: {
               id: {type: "integer"}, name: {type: "string"},
               dataset_id: {type: "integer"}, judge_model: {type: "string"},
+              temperature: {type: "number", description: TEMPERATURE_DESCRIPTION},
+              max_tokens: {type: "integer", description: MAX_TOKENS_DESCRIPTION},
+              judge_temperature: {type: "number", description: JUDGE_TEMPERATURE_DESCRIPTION},
               output_column: {type: "string"},
               expected_column: {type: "string"},
               metric_ids: {type: "array", items: {type: "integer"}},
@@ -84,7 +105,7 @@ module CompletionKit
       end
 
       def self.create(args)
-        run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column", "expected_column"))
+        run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column", "expected_column", *GENERATION_FIELDS))
         if run.save
           run.replace_metrics!(resolve_metric_ids(args))
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
@@ -96,7 +117,7 @@ module CompletionKit
 
       def self.update(args)
         run = Run.find(args["id"])
-        if run.update(args.except("id", "metric_ids", "metric_group_id", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column"))
+        if run.update(args.except("id", "metric_ids", "metric_group_id", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column", *GENERATION_FIELDS))
           run.replace_metrics!(resolve_metric_ids(args)) if args.key?("metric_ids") || args["metric_group_id"].present?
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
           text_result(run_payload(run.reload))
@@ -156,9 +177,16 @@ module CompletionKit
 
       def self.run_payload(run)
         json = run.as_json
-        return json unless run.metric_ids.empty?
+        warnings = []
+        if run.metric_ids.empty?
+          warnings << "No metrics are attached, so this run judges nothing. Attach metric_ids or a metric_group_id before generating."
+        end
+        if run.nondeterministic_judge?
+          warnings << "Judge temperature is #{run.judge_temperature}. Judging above 0 makes scores irreproducible: the same output can get a different score on a re-judge. Set judge_temperature to 0 unless you are deliberately measuring judge variance."
+        end
+        return json if warnings.empty?
 
-        json.merge("warning" => "No metrics are attached, so this run judges nothing. Attach metric_ids or a metric_group_id before generating.")
+        json.merge("warning" => warnings.join(" "))
       end
     end
   end

data/app/services/completion_kit/metric_improvement_validator.rb

@@ -87,9 +87,8 @@ module CompletionKit
 
     def rescore(response, candidate)
       run = response.run
-      config = ApiConfig.for_model(run.judge_model).merge(judge_model: run.judge_model)
       rubric_text = Metric.rubric_text_for(Metric.normalize_rubric_bands(candidate.rubric_bands))
-      result = JudgeService.new(config).evaluate(
+      result = JudgeService.new(run.judge_config).evaluate(
         response.response_text,
         response.expected_output,
         run.prompt&.template,

data/app/services/completion_kit/prompt_improvement_validator.rb

@@ -77,15 +77,14 @@ module CompletionKit
       client = LlmClient.for_model(model, ApiConfig.for_model(model))
       raise CompletionKit::ConfigurationError, client.configuration_errors.join(", ") unless client.configured?
 
-      text = client.generate_completion(rendered, model: model, temperature: @run.temperature)
+      text = client.generate_completion(rendered, **@run.generation_options(@run.prompt))
       raise StandardError, text if text.to_s.start_with?("Error:")
 
       text
     end
 
     def judge_score(response, new_text)
-      config = ApiConfig.for_model(@run.judge_model).merge(judge_model: @run.judge_model)
-      judge = JudgeService.new(config)
+      judge = JudgeService.new(@run.judge_config)
       scores = @run.metrics.select(&:llm_judge?).filter_map do |metric|
         judge.evaluate(
           new_text, response.expected_output, @candidate,

data/app/services/completion_kit/response_query.rb

@@ -0,0 +1,94 @@
+module CompletionKit
+  # Filtering, score-ordering and field projection for a run's responses,
+  # shared by the REST endpoint and the MCP tool so both surfaces answer
+  # "show me the worst rows, without the full bodies" the same way.
+  class ResponseQuery
+    SORTS = %w[id score_asc score_desc].freeze
+    REVIEW_FIELD_PREFIX = "reviews.".freeze
+
+    def initialize(run, status: nil, min_score: nil, max_score: nil, sort: nil, fields: nil)
+      @run = run
+      @status = status.presence
+      @min_score = min_score.presence&.to_f
+      @max_score = max_score.presence&.to_f
+      @sort = SORTS.include?(sort.to_s) ? sort.to_s : "id"
+      @fields = parse_fields(fields)
+    end
+
+    def relation
+      return base.order(:id) unless score_scoped?
+
+      ids = ordered_ids
+      base.where(id: ids).in_order_of(:id, ids)
+    end
+
+    def serialize(response)
+      project(response.as_json)
+    end
+
+    private
+
+    def base
+      scope = @run.responses.includes(:reviews)
+      @status ? scope.where(status: @status) : scope
+    end
+
+    def score_scoped?
+      filtering? || @sort != "id"
+    end
+
+    def filtering?
+      !@min_score.nil? || !@max_score.nil?
+    end
+
+    def ordered_ids
+      averages = score_averages
+      scored, unscored = base.order(:id).pluck(:id).partition { |id| averages.key?(id) }
+      kept = sort_by_score(scored.select { |id| in_range?(averages[id]) }, averages)
+      filtering? ? kept : kept + unscored
+    end
+
+    # Mirrors Response#score (mean of the row's judge scores, rounded to 2) so a
+    # min_score filter keeps exactly the rows whose reported score qualifies.
+    def score_averages
+      Review.where(response_id: base.select(:id))
+            .where.not(ai_score: nil)
+            .group(:response_id)
+            .pluck(:response_id, Arel.sql("AVG(ai_score)"))
+            .to_h { |id, avg| [id, avg.to_f.round(2)] }
+    end
+
+    def in_range?(average)
+      return false if @min_score && average < @min_score
+      return false if @max_score && average > @max_score
+
+      true
+    end
+
+    def sort_by_score(ids, averages)
+      return ids if @sort == "id"
+
+      sorted = ids.sort_by { |id| averages[id] }
+      @sort == "score_desc" ? sorted.reverse : sorted
+    end
+
+    def parse_fields(raw)
+      names = Array(raw).flat_map { |value| value.to_s.split(",") }.map(&:strip).reject(&:empty?)
+      {
+        top: names.reject { |name| name.start_with?(REVIEW_FIELD_PREFIX) }.map(&:to_sym),
+        reviews: names.select { |name| name.start_with?(REVIEW_FIELD_PREFIX) }
+                      .map { |name| name.delete_prefix(REVIEW_FIELD_PREFIX).to_sym }
+      }
+    end
+
+    def project(json)
+      return json if @fields[:top].empty? && @fields[:reviews].empty?
+
+      projected = json.slice(:id, *@fields[:top])
+      if @fields[:reviews].any?
+        projected[:reviews] = json[:reviews].map { |review| review.slice(*@fields[:reviews]) }
+      end
+      projected
+    end
+  end
+end

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -129,12 +129,13 @@
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs</p>
         <p class="ck-meta-copy">Create a new run.</p>
-        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>name</code>, <code>prompt_id</code>, <code>dataset_id</code>, <code>metric_ids</code>, <code>judge_model</code>, <code>temperature</code>, <code>output_column</code> (score existing outputs: omit <code>prompt_id</code> and grade a dataset column instead, default <code>actual_output</code>), <code>expected_column</code> (dataset column holding each row's answer key, given to the judge and to checks that compare against the row's expected value, default <code>expected_output</code>)</p>
+        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>name</code>, <code>prompt_id</code>, <code>dataset_id</code>, <code>metric_ids</code>, <code>judge_model</code>, <code>temperature</code>, <code>judge_temperature</code> (defaults to 0 so re-judging is reproducible; above 0 the same output can score differently), <code>max_tokens</code> (cap on generated tokens per row; leave unset for the provider default, set it to match production when your prompt's output is long enough to truncate), <code>output_column</code> (score existing outputs: omit <code>prompt_id</code> and grade a dataset column instead, default <code>actual_output</code>), <code>expected_column</code> (dataset column holding each row's answer key, given to the judge and to checks that compare against the row's expected value, default <code>expected_output</code>)</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"prompt_id\": 1, \"dataset_id\": 1, \"metric_ids\": [1, 2]}'" %>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:id</p>
         <p class="ck-meta-copy">Get a run with status, progress, response count, and average score.</p>
+        <p class="ck-api-params"><strong>Also returns:</strong>&ensp;<code>metric_averages</code>, a per-metric breakdown with each metric's average score (<code>pass_rate</code> for checks), the number of rows it graded, and <code>low_count</code>, how many scored below the medium quality threshold</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl #{base_url}/api/v1/runs/1 \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <%= render "completion_kit/api_reference/resource_list", title: "Your recent runs",
@@ -185,8 +186,9 @@
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:run_id/responses</p>
         <p class="ck-meta-copy">List responses for a run, including nested review scores.</p>
-        <p class="ck-api-params"><strong>Optional filters:</strong>&ensp;<code>status</code> (<code>pending</code>, <code>retrying</code>, <code>succeeded</code>, <code>failed</code>), plus <code>limit</code> and <code>offset</code></p>
-        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl #{base_url}/api/v1/runs/1/responses \\\n  -H \"Authorization: Bearer #{token}\"" %>
+        <p class="ck-api-params"><strong>Optional filters:</strong>&ensp;<code>status</code> (<code>pending</code>, <code>retrying</code>, <code>succeeded</code>, <code>failed</code>), <code>min_score</code>, <code>max_score</code>, <code>sort</code> (<code>id</code>, <code>score_asc</code>, <code>score_desc</code>), plus <code>limit</code> and <code>offset</code></p>
+        <p class="ck-api-params"><strong>Trim the payload:</strong>&ensp;<code>fields</code> takes a comma-separated list of response keys and always returns <code>id</code>. Prefix with <code>reviews.</code> to trim each nested review, e.g. <code>fields=score,reviews.metric_name,reviews.ai_score</code></p>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl \"#{base_url}/api/v1/runs/1/responses?sort=score_asc&limit=10&fields=id,score,reviews.metric_name,reviews.ai_score\" \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:run_id/responses/:id</p>

data/app/views/completion_kit/dashboard/_failing_checks_card.html.erb

@@ -1,19 +1,48 @@
+<% resolved = check_activity.sum { |day| day[:resolved] } %>
+<% passed = check_activity.sum { |day| day[:passed] } %>
 <div class="ck-card ck-stat-card ck-rise" id="ck-failing-checks-card" style="--rise-delay: 200ms;">
-  <p class="ck-kicker">Failing checks · last 7 days</p>
+  <p class="ck-kicker">Checks · 14D</p>
   <div class="ck-stat-card__body">
-    <span class="ck-stat-card__count<%= failing_checks[:count].positive? ? ' is-danger' : ' is-clean' %>"><%= failing_checks[:count] %></span>
+    <% if resolved.positive? %>
+      <% rate = passed.to_f / resolved %>
+      <span class="ck-stat-card__count is-<%= ck_pass_rate_kind(rate) %>"><%= (rate * 100).round %>%</span>
+      <span class="ck-stat-card__unit">passing</span>
+    <% else %>
+      <span class="ck-stat-card__metric ck-stat-card__metric--empty">Not run yet</span>
+    <% end %>
+  </div>
+
+  <div class="ck-sparkline ck-sparkline--compact" role="img"
+       aria-label="Daily check pass rate over the last 14 days">
+    <% check_activity.each do |day| %>
+      <% kind = day[:rate] && ck_pass_rate_kind(day[:rate]) %>
+      <span class="ck-sparkline__bar<%= " is-#{kind}" if kind && kind != :high %>"
+            style="height: <%= day[:rate] ? [(day[:rate] * 100).round, 6].max : 0 %>%;"
+            title="<%= day[:date].strftime('%b %-d') %>: <%= day[:rate] ? "#{(day[:rate] * 100).round}% of #{day[:resolved]}" : 'no checks' %>"></span>
+    <% end %>
   </div>
 
   <% if failing_checks[:items].any? %>
     <ul class="ck-failure-list">
-      <% failing_checks[:items].first(5).each do |item| %>
+      <% failing_checks[:items].first(2).each do |item| %>
         <li class="ck-failure-list__item">
-          <span class="ck-failure-list__surface ck-failure-list__surface--check">check</span>
-          <%= link_to item[:metric_name], completion_kit.run_path(item[:run]), class: "ck-link ck-failure-list__cause" %>
+          <span class="ck-failure-list__surface ck-failure-list__surface--check">fail</span>
+          <%= link_to item[:metric_name], completion_kit.run_path(item[:run]),
+                class: "ck-link ck-failure-list__cause", title: "Failed in #{item[:run].name}" %>
         </li>
       <% end %>
     </ul>
-  <% else %>
-    <div class="ck-stat-card__foot"><span>No failing checks this week.</span></div>
   <% end %>
+
+  <div class="ck-stat-card__foot">
+    <% if failing_checks[:count] > 2 %>
+      <span><%= failing_checks[:count] - 2 %> more failing.</span>
+    <% elsif failing_checks[:count].positive? %>
+      <span><%= pluralize(failing_checks[:count], "check") %> failing in the window.</span>
+    <% elsif resolved.positive? %>
+      <span>Every check passed in the window.</span>
+    <% else %>
+      <span>Add a check metric to a run to populate this.</span>
+    <% end %>
+  </div>
 </div>

data/app/views/completion_kit/dashboard/_failures_card.html.erb

@@ -1,5 +1,5 @@
 <div class="ck-card ck-stat-card ck-rise" id="ck-failures-card" style="--rise-delay: 180ms;">
-  <p class="ck-kicker">Failures · last 7 days</p>
+  <p class="ck-kicker">Failures · 7D</p>
   <div class="ck-stat-card__body">
     <span class="ck-stat-card__count<%= failures[:count].positive? ? ' is-danger' : ' is-clean' %>"><%= failures[:count] %></span>
   </div>
@@ -8,11 +8,12 @@
     <ul class="ck-failure-list">
       <% failures[:items].each do |item| %>
         <li class="ck-failure-list__item">
-          <span class="ck-failure-list__surface ck-failure-list__surface--<%= item[:surface] %>"><%= item[:surface] %></span>
+          <span class="ck-failure-list__surface ck-failure-list__surface--<%= item[:surface] %>"><%= item[:surface] == "generation" ? "gen" : item[:surface] %></span>
           <% if item[:run] %>
-            <%= link_to item[:cause], completion_kit.run_path(item[:run]), class: "ck-link ck-failure-list__cause" %>
+            <%= link_to item[:run].name, completion_kit.run_path(item[:run]),
+                  class: "ck-link ck-failure-list__cause", title: item[:cause] %>
           <% else %>
-            <span class="ck-failure-list__cause"><%= item[:cause] %></span>
+            <span class="ck-failure-list__cause" title="<%= item[:cause] %>">Deleted run</span>
           <% end %>
           <%= button_to completion_kit.dashboard_dismissals_path,
                 params: { dashboard_dismissal: { dismissable_type: item[:record].class.name,
@@ -26,7 +27,7 @@
 
   <% if failures[:items].empty? || ignored_failures.any? %>
     <div class="ck-stat-card__foot ck-stat-card__foot--split">
-      <span><% if failures[:items].empty? %>All clear — nothing failed this week.<% end %></span>
+      <span><% if failures[:items].empty? %>All clear. Nothing failed this week.<% end %></span>
       <% if ignored_failures.any? %>
         <details class="ck-flyout">
           <summary class="ck-flyout__toggle"><%= ignored_failures.size %> ignored</summary>

data/app/views/completion_kit/dashboard/_worst_metric_card.html.erb

@@ -1,5 +1,5 @@
 <div class="ck-card ck-stat-card ck-rise" id="ck-worst-metric-card" style="--rise-delay: 120ms;">
-  <p class="ck-kicker">Worst metric · last 7 days</p>
+  <p class="ck-kicker">Worst metric · 7D</p>
   <div class="ck-stat-card__body">
     <% if worst_metric %>
       <span class="ck-stat-card__metric"><%= worst_metric[:name] %></span>

data/app/views/completion_kit/dashboard/show.html.erb

@@ -28,20 +28,30 @@
 </nav>
 
 <% if @activity %>
-  <div class="ck-grid ck-grid--cards ck-grid--cards-4 ck-pulse-grid">
+  <div class="ck-grid ck-grid--cards <%= @failing_checks ? "ck-grid--cards-4" : "ck-grid--cards-3" %> ck-pulse-grid">
     <div class="ck-card ck-stat-card ck-rise" style="--rise-delay: 60ms;">
-      <p class="ck-kicker">Activity · last 14 days</p>
+      <p class="ck-kicker">Activity · 14D</p>
+      <% activity_total = @activity.sum { |d| d[:count] } %>
       <% activity_max = @activity.map { |d| d[:count] }.max %>
-      <div class="ck-sparkline" role="img" aria-label="<%= @activity.sum { |d| d[:count] } %> runs over the last 14 days">
+      <div class="ck-stat-card__body">
+        <span class="ck-stat-card__count"><%= activity_total %></span>
+        <span class="ck-stat-card__unit"><%= "run".pluralize(activity_total) %></span>
+      </div>
+      <div class="ck-sparkline ck-sparkline--compact" role="img" aria-label="<%= activity_total %> runs over the last 14 days">
         <% @activity.each do |day| %>
           <span class="ck-sparkline__bar<%= ' is-peak' if activity_max.to_i.positive? && day[:count] == activity_max %>"
-                style="height: <%= activity_max.to_i.zero? ? 0 : (day[:count] * 100.0 / activity_max).round %>%"
+                style="height: <%= activity_max.to_i.zero? ? 0 : [(day[:count] * 100.0 / activity_max).round, 6].max %>%"
                 title="<%= day[:date].strftime('%b %-d') %>: <%= day[:count] %> run<%= 's' unless day[:count] == 1 %>"></span>
         <% end %>
       </div>
-      <p class="ck-stat-card__foot">
-        <span class="ck-stat-card__figure"><%= @activity.sum { |d| d[:count] } %></span> runs in the window
-      </p>
+      <div class="ck-stat-card__foot">
+        <% if activity_total.positive? %>
+          <% peak = @activity.max_by { |d| d[:count] } %>
+          <span>Busiest day <%= peak[:date].strftime("%b %-d") %> · <%= peak[:count] %></span>
+        <% else %>
+          <span>No runs in the window.</span>
+        <% end %>
+      </div>
     </div>
 
     <%= render "completion_kit/dashboard/worst_metric_card",
@@ -50,8 +60,10 @@
     <%= render "completion_kit/dashboard/failures_card",
                failures: @failures, ignored_failures: @ignored_failures %>
 
-    <%= render "completion_kit/dashboard/failing_checks_card",
-               failing_checks: @failing_checks %>
+    <% if @failing_checks %>
+      <%= render "completion_kit/dashboard/failing_checks_card",
+                 failing_checks: @failing_checks, check_activity: @check_activity %>
+    <% end %>
   </div>
 
   <div class="ck-card ck-card--spaced ck-rise" style="--rise-delay: 240ms;">
@@ -80,6 +92,33 @@
   </div>
 <% end %>
 
+<div class="ck-card ck-card--spaced ck-rise" id="top_served_card" style="--rise-delay: 280ms;">
+  <p class="ck-kicker">Prompts served · 7D</p>
+  <% if @top_served.any? %>
+    <% serve_max = @serve_activity.map { |d| d[:count] }.max %>
+    <div class="ck-sparkline" role="img" aria-label="<%= @serve_activity.sum { |d| d[:count] } %> prompt fetches over the last 14 days">
+      <% @serve_activity.each do |day| %>
+        <span class="ck-sparkline__bar<%= ' is-peak' if serve_max.positive? && day[:count] == serve_max %>"
+              style="height: <%= serve_max.positive? ? [(day[:count] * 100.0 / serve_max).round, 4].max : 4 %>%;"
+              title="<%= day[:date].strftime('%b %-d') %>: <%= day[:count] %>"></span>
+      <% end %>
+    </div>
+    <ul class="ck-improvements">
+      <% @top_served.each do |row| %>
+        <li class="ck-improvement">
+          <%= link_to row[:prompt].name, prompt_path(row[:prompt]), class: "ck-improvement__name ck-link" %>
+          <span class="ck-improvement__versions"><%= row[:prompt].version_label %></span>
+          <span class="ck-improvement__scores"><span class="ck-serve-figure"><%= number_with_delimiter(row[:count]) %></span> fetches</span>
+        </li>
+      <% end %>
+    </ul>
+  <% else %>
+    <p class="ck-improvements__empty">
+      No prompts fetched in the last 7 days. Once your application reads a published prompt from the API, the prompts it uses most show up here.
+    </p>
+  <% end %>
+</div>
+
 <section class="ck-card--spaced">
   <div class="ck-split">
     <h2 class="ck-section-title">Recent runs</h2>

data/app/views/completion_kit/prompts/show.html.erb

@@ -18,6 +18,19 @@
       <code class="ck-endpoint__url" id="prompt_endpoint"><%= request.base_url %><%= api_v1_prompt_path(@prompt.slug) %></code>
       <button type="button" class="ck-icon-btn" title="Copy endpoint" aria-label="Copy endpoint" onclick="navigator.clipboard.writeText(document.getElementById('prompt_endpoint').textContent)"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="14" height="14" aria-hidden="true"><path d="M0 6.75C0 5.784.784 5 1.75 5h1.5a.75.75 0 0 1 0 1.5h-1.5a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-1.5a.75.75 0 0 1 1.5 0v1.5A1.75 1.75 0 0 1 9.25 16h-7.5A1.75 1.75 0 0 1 0 14.25Z"/><path d="M5 1.75C5 .784 5.784 0 6.75 0h7.5C15.216 0 16 .784 16 1.75v7.5A1.75 1.75 0 0 1 14.25 11h-7.5A1.75 1.75 0 0 1 5 9.25Zm1.75-.25a.25.25 0 0 0-.25.25v7.5c0 .138.112.25.25.25h7.5a.25.25 0 0 0 .25-.25v-7.5a.25.25 0 0 0-.25-.25Z"/></svg></button>
     </div>
+    <p class="ck-meta-copy" id="prompt_serves">
+      <% if @serve_summary[:total].positive? %>
+        Served <span class="ck-serve-figure"><%= number_with_delimiter(@serve_summary[:total]) %></span>
+        <%= "time".pluralize(@serve_summary[:total]) %><% if @serve_summary[7] < @serve_summary[:total] %>,
+        <%= number_with_delimiter(@serve_summary[7]) %> in the last 7 days<% end %>.
+        <% if @serve_summary[:last_served_at] %>
+          Last fetched <time data-relative-time datetime="<%= @serve_summary[:last_served_at].utc.iso8601 %>"><%= time_ago_in_words(@serve_summary[:last_served_at]) %> ago</time>.
+        <% end %>
+      <% else %>
+        Not fetched yet. Counts appear here once something requests this prompt from the URL above.
+      <% end %>
+    </p>
+
     <% if @prompt.tags.any? %>
       <div class="tag-marks-row">
         <%= render "completion_kit/tags/marks", tags: @prompt.tags %>

data/app/views/completion_kit/responses/show.html.erb

@@ -97,7 +97,7 @@
 <% if @reviews.any? %>
   <section class="ck-card--spaced">
     <div class="ck-prompt-preview__header">
-      <p class="ck-kicker">Review</p>
+      <p class="ck-kicker">Judge's review</p>
       <% if @run.judge_model.present? %>
         <span class="ck-chip ck-chip--soft" style="text-transform: none;"><%= @run.judge_model %></span>
       <% end %>