completion-kit 0.28.23 → 0.28.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c5b5a45e20e1884194db7473fa6bc44e75b33b919d1790dfa240f2e3902bd48
-  data.tar.gz: c6eeabc0e477591231895f42868514d50363b34834967d0088593e360ccaf04b
+  metadata.gz: 58dfff9f577e36bd13a32f6670ad1c095c23492238fd50043bd2ccceca648e98
+  data.tar.gz: 0c8a0d5e4c96e8bccff0a93ede04aef40c650255f1271162537a8a63008314a6
 SHA512:
-  metadata.gz: b604a731fd81ba9c6aadd87f00e5a0b7665685df4651a3fca27e2995375cfcac7a48ef8860ba573abd3af612fd4cdee1224ec804a0fbd57a7bdbee50bd996684
-  data.tar.gz: 566e496833459b9f16e8c19251e4847d71c917e14bdb408fee2f5d08252aa542a554a4578fbcc9dfbc62e53e89c288745eda4a1cbadd78310b86d10c70b96b85
+  metadata.gz: 15b9d07c9bd067885ab2d47ed580d7237dd49e1f6e0b64874a46664db8f02d5c0f1cdde7b183431c9244769a6b96bbb4279d8c5062fc890755bf9b43542ca4cb
+  data.tar.gz: '08fd16f17461b4960aecffdcc350347120d2a5d786327d982adfab7f3b0cfcdaa1e011eb7e81ec5773c39390baa48f3477bed115358e2d4a043205acd33a58f9'

data/app/assets/stylesheets/completion_kit/application.css

@@ -519,8 +519,16 @@ form.button_to {
   font-size: 2.6rem;
   line-height: 1;
 }
-.ck-stat-card__count.is-clean { color: var(--ck-success, #4ade80); }
-.ck-stat-card__count.is-danger { color: var(--ck-danger, #f87171); }
+.ck-stat-card__count.is-clean,
+.ck-stat-card__count.is-high { color: var(--ck-success, #4ade80); }
+.ck-stat-card__count.is-medium { color: var(--ck-warning); }
+.ck-stat-card__count.is-danger,
+.ck-stat-card__count.is-low { color: var(--ck-danger, #f87171); }
+.ck-stat-card__unit {
+  font-family: var(--ck-mono);
+  font-size: 0.78rem;
+  color: var(--ck-muted);
+}
 
 /* Dashboard activity sparkline — fixed-height row of bars, height set
    inline per day. Bars hug the bottom; quiet days collapse to a sliver.
@@ -542,8 +550,17 @@ form.button_to {
   transition: background 0.15s ease;
 }
 .ck-sparkline__bar.is-peak { background: var(--ck-accent); }
+.ck-sparkline__bar.is-medium { background: var(--ck-warning); }
+.ck-sparkline__bar.is-low { background: var(--ck-danger); }
 .ck-sparkline__bar:hover { background: var(--ck-accent-hover); }
 
+/* The number carries the headline; the bars are supporting texture, so they
+   take about half the height of a standalone sparkline. */
+.ck-sparkline--compact {
+  height: 1.9rem;
+  margin: 0.55rem 0 0;
+}
+
 /* One orchestrated page-load reveal — the dashboard "boots up". */
 @keyframes ck-rise {
   from { opacity: 0; transform: translateY(8px); }
@@ -1295,6 +1312,25 @@ tr:hover .ck-chip--publish {
   margin: 0 0.15rem;
 }
 
+.ck-run-status__cell-value--split {
+  gap: 0.5rem;
+  flex-wrap: wrap;
+}
+
+.ck-run-status__cell-value--stack {
+  flex-direction: column;
+  align-items: flex-start;
+  justify-content: center;
+  gap: 0.15rem;
+}
+
+.ck-run-status__cell-unverified {
+  font-family: var(--ck-mono);
+  font-size: 0.95rem;
+  color: var(--ck-warning);
+  letter-spacing: 0.02em;
+}
+
 .ck-run-status__cell-empty {
   font-family: var(--ck-mono);
   font-size: 1.4rem;
@@ -3184,7 +3220,8 @@ select.ck-input {
   line-height: 1.55;
 }
 
-.ck-stale-versions-banner {
+.ck-stale-versions-banner,
+.ck-score-ceiling-banner {
   margin: 0 0 1rem;
   padding: 0.9rem 1rem;
   border: 1px solid rgba(224, 164, 88, 0.4);
@@ -3196,8 +3233,33 @@ select.ck-input {
   gap: 1rem;
   flex-wrap: wrap;
 }
-.ck-stale-versions-banner__body { min-width: 0; flex: 1 1 320px; }
-.ck-stale-versions-banner .ck-kicker { color: var(--ck-warning); }
+.ck-stale-versions-banner__body,
+.ck-score-ceiling-banner__body { min-width: 0; flex: 1 1 320px; }
+.ck-stale-versions-banner .ck-kicker,
+.ck-score-ceiling-banner .ck-kicker { color: var(--ck-warning); }
+.ck-score-ceiling-banner__actions { display: flex; gap: 0.5rem; flex-wrap: wrap; }
+
+/* Each action carries its own consequence line, because the difference
+   between re-grading and re-running is the whole decision. */
+.ck-banner-actions {
+  display: flex;
+  gap: 1.25rem;
+  flex-wrap: wrap;
+  align-items: flex-start;
+}
+.ck-banner-action {
+  display: flex;
+  flex-direction: column;
+  gap: 0.35rem;
+  max-width: 16rem;
+}
+.ck-banner-action__note {
+  margin: 0;
+  font-family: var(--ck-mono);
+  font-size: 0.7rem;
+  line-height: 1.45;
+  color: var(--ck-muted);
+}
 
 .ck-delta {
   font-family: var(--ck-mono);
@@ -6557,3 +6619,9 @@ a.tag-mark {
   outline: 2px solid var(--ck-accent);
   outline-offset: 2px;
 }
+
+.ck-serve-figure {
+  font-family: var(--ck-mono);
+  font-weight: 700;
+  color: var(--ck-text);
+}

data/app/controllers/completion_kit/api/v1/prompts_controller.rb

@@ -11,6 +11,7 @@ module CompletionKit
         end
 
         def show
+          PromptServe.record!(@prompt)
           render json: @prompt
         end
 

data/app/controllers/completion_kit/api/v1/responses_controller.rb

@@ -6,9 +6,12 @@ module CompletionKit
         before_action :set_response, only: [:show]
 
         def index
-          scope = @run.responses.includes(:reviews)
-          scope = scope.where(status: params[:status]) if params[:status].present?
-          render json: paginate(scope.order(:id))
+          query = ResponseQuery.new(
+            @run,
+            status: params[:status], min_score: params[:min_score], max_score: params[:max_score],
+            sort: params[:sort], fields: params[:fields]
+          )
+          render json: paginate(query.relation).map { |response| query.serialize(response) }
         end
 
         def show

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -130,7 +130,7 @@ module CompletionKit
         end
 
         def run_params
-          params.permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :output_column, :expected_column,
+          params.permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :max_tokens, :judge_temperature, :output_column, :expected_column,
             metric_ids: [], tag_names: [])
         end
       end

data/app/controllers/completion_kit/dashboard_controller.rb

@@ -9,12 +9,23 @@ module CompletionKit
       @metric_count = Metric.count
       @recent_runs = Run.display_scoped.order(created_at: :desc).limit(5)
 
+      # Serving is independent of evaluating: a workspace that only fetches
+      # prompts over the API and never runs an eval is exactly the audience for
+      # this, so it sits outside the run-count gate below.
+      @top_served = DashboardStats.top_served(since: 7.days.ago)
+      @serve_activity = DashboardStats.serve_activity
+
       return unless Run.count > 5
 
       @activity = DashboardStats.activity
       @worst_metric = DashboardStats.worst_metric(since: 7.days.ago)
       @failures = DashboardStats.failures(since: 7.days.ago)
-      @failing_checks = DashboardStats.failing_checks(since: 7.days.ago)
+      # A workspace with no check metrics can never populate this card, so it
+      # would sit at a permanent zero and read as filler.
+      if Metric.exists?(metric_type: "check")
+        @failing_checks = DashboardStats.failing_checks(since: 14.days.ago)
+        @check_activity = DashboardStats.check_activity
+      end
       @ignored_metrics = DashboardDismissal.metrics
       @ignored_failures = DashboardDismissal.failures
       @prompt_changes = DashboardStats.prompt_changes

data/app/controllers/completion_kit/prompts_controller.rb

@@ -8,6 +8,7 @@ module CompletionKit
     end
     
     def show
+      @serve_summary = PromptServe.summary_for(@prompt)
       @runs = Run.where(prompt_id: @prompt.family_versions.select(:id))
                  .includes(:prompt, :dataset, :tags, responses: :reviews)
                  .order(created_at: :desc)

data/app/controllers/completion_kit/runs_controller.rb

@@ -119,14 +119,21 @@ module CompletionKit
       end
     end
 
+    # The web process drives the live run page, because worker-side Turbo
+    # broadcasts do not reliably publish in production. Since a run is now
+    # started asynchronously, this has to carry the rows and the progress panel
+    # too, not just the status badge, or the page sits empty until a reload.
     def refresh_status
+      show
+
       respond_to do |format|
         format.turbo_stream do
-          render turbo_stream: turbo_stream.replace(
-            "run_status_header",
-            partial: "completion_kit/runs/status_header",
-            locals: { run: @run }
-          )
+          render turbo_stream: [
+            turbo_stream.replace("run_status_header", partial: "completion_kit/runs/status_header", locals: { run: @run }),
+            turbo_stream.replace("run_status_panel", partial: "completion_kit/runs/status_panel", locals: { run: @run }),
+            turbo_stream.replace("run_responses_region", partial: "completion_kit/runs/responses_region",
+                                 locals: { run: @run, responses: @responses, responses_offset: @responses_offset })
+          ]
         end
       end
     end
@@ -215,13 +222,13 @@ module CompletionKit
     end
 
     def run_params
-      params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :output_column, :expected_column, metric_ids: [], tag_names: [])
+      params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :max_tokens, :judge_temperature, :output_column, :expected_column, metric_ids: [], tag_names: [])
     end
 
     # Editing a run that already has results forks a new run — but only when a
     # field that affects generation or judging changed. Renaming or retagging is
     # pure metadata and updates the run in place.
-    GENERATION_RUN_FIELDS = %i[prompt_id dataset_id judge_model temperature output_column expected_column].freeze
+    GENERATION_RUN_FIELDS = %i[prompt_id dataset_id judge_model temperature max_tokens judge_temperature output_column expected_column].freeze
 
     def run_generation_changed?
       GENERATION_RUN_FIELDS.each do |field|
@@ -236,8 +243,8 @@ module CompletionKit
       s = value.to_s.strip
       return nil if s.empty?
       case field
-      when :temperature then s.to_f
-      when :prompt_id, :dataset_id then s.to_i
+      when :temperature, :judge_temperature then s.to_f
+      when :prompt_id, :dataset_id, :max_tokens then s.to_i
       else s
       end
     end

data/app/jobs/completion_kit/generate_row_job.rb

@@ -58,7 +58,7 @@ module CompletionKit
 
       raise ConfigurationError, client.configuration_errors.join(", ") unless client.configured?
 
-      text = client.generate_completion(rendered, model: prompt.llm_model, temperature: run.temperature)
+      text = client.generate_completion(rendered, **run.generation_options(prompt))
 
       if text.to_s.start_with?("Error:")
         record_terminal_failure!(ProviderError.from_client_error(text))

data/app/jobs/completion_kit/judge_review_job.rb

@@ -59,8 +59,7 @@ module CompletionKit
       metric = Metric.find(metric_id)
       run = response.run
 
-      config = ApiConfig.for_model(run.judge_model).merge(judge_model: run.judge_model)
-      judge = JudgeService.new(config)
+      judge = JudgeService.new(run.judge_config)
 
       begin
         evaluation = judge.evaluate(

data/app/jobs/completion_kit/run_completion_check_job.rb

@@ -10,6 +10,10 @@ module CompletionKit
       run = Run.find_by(id: run_id)
       return unless run
       return unless run.status == "running"
+      # A run that has been claimed but whose rows are still being inserted by
+      # StartRunJob has no outstanding work yet, and would otherwise complete
+      # itself with zero responses.
+      return if run.progress_total.zero?
       return unless run.outstanding_work_zero?
 
       run.mark_completed!

data/app/jobs/completion_kit/start_run_job.rb

@@ -0,0 +1,29 @@
+module CompletionKit
+  class StartRunJob < ApplicationJob
+    queue_as :default
+
+    limits_concurrency to: 1,
+                       key: ->(run_id, _defaults = {}) { "run:#{run_id}:start" },
+                       duration: 10.minutes
+
+    rescue_from(StandardError) do |error|
+      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id })
+      run = Run.find_by(id: @run_id)
+      run&.fail_to_start!(error.message)
+    end
+
+    # scope_defaults carries the host's tenant columns, resolved in the web
+    # request where the tenant context exists. A worker has no such context, so
+    # without them the inserted responses would be stamped with nothing.
+    def perform(run_id, scope_defaults = {})
+      @run_id = run_id
+
+      run = Run.find_by(id: run_id)
+      return unless run
+      return unless run.status == "running"
+      return if run.responses.exists?
+
+      run.execute_start!(scope_defaults.symbolize_keys)
+    end
+  end
+end

data/app/models/completion_kit/metric.rb

@@ -25,6 +25,7 @@ module CompletionKit
     validates :name, presence: true
     validates :key, tenant_scoped_uniqueness: { allow_nil: true }
     validates :metric_type, inclusion: { in: METRIC_TYPES }
+    validates :instruction, presence: { message: "can't be blank. An LLM judge needs a rubric to score against." }, if: :llm_judge?
     validate :validate_check_config, if: :check?
     validate :metric_type_immutable_once_in_use, on: :update
 

data/app/models/completion_kit/prompt_serve.rb

@@ -0,0 +1,41 @@
+module CompletionKit
+  # One row per prompt per day, counting how often that prompt was fetched by a
+  # consumer. Rolled up daily rather than stored per request so the table stays
+  # bounded by (prompts x days) instead of growing with traffic.
+  #
+  # family_key is denormalised so a family's serving history survives deleting
+  # an individual version, which prompt_id alone would not.
+  class PromptServe < ApplicationRecord
+    self.table_name = "completion_kit_prompt_serves"
+
+    belongs_to :prompt, optional: true
+
+    def self.record!(prompt)
+      today = Date.current
+      now = Time.current
+      scope = where(prompt_id: prompt.id, served_on: today)
+
+      return if scope.update_all(["serve_count = serve_count + 1, last_served_at = ?", now]).positive?
+
+      create!(prompt_id: prompt.id, family_key: prompt.family_key, served_on: today,
+              serve_count: 1, last_served_at: now)
+    rescue ActiveRecord::RecordNotUnique
+      scope.update_all(["serve_count = serve_count + 1, last_served_at = ?", now])
+    end
+
+    # Totals for the whole prompt family, so publishing a new version does not
+    # reset the number a user is watching.
+    def self.summary_for(prompt, windows: [7, 30])
+      rows = where(family_key: prompt.family_key)
+             .pluck(:served_on, :serve_count, :last_served_at)
+      return { total: 0, last_served_at: nil }.merge(windows.index_with { 0 }) if rows.empty?
+
+      summary = { total: rows.sum { |_, count, _| count },
+                  last_served_at: rows.filter_map { |_, _, at| at }.max }
+      windows.each_with_object(summary) do |days, acc|
+        cutoff = Date.current - (days - 1)
+        acc[days] = rows.sum { |on, count, _| on >= cutoff ? count : 0 }
+      end
+    end
+  end
+end

data/app/models/completion_kit/run.rb

@@ -18,6 +18,8 @@ module CompletionKit
 
     validates :name, presence: true
     validates :status, inclusion: { in: STATUSES }
+    validates :max_tokens, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true
+    validates :judge_temperature, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true
     validate :dataset_supplies_prompt_variables
     validate :judge_only_run_supplies_output_column
     validate :dataset_supplies_expected_column
@@ -65,6 +67,7 @@ module CompletionKit
         :metric_name,
         Arel.sql("AVG(ai_score)"),
         Arel.sql("COUNT(ai_score)"),
+        Arel.sql("SUM(CASE WHEN ai_score < #{low_score_ceiling} THEN 1 ELSE 0 END)"),
         Arel.sql("COUNT(passed)"),
         Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
       )
@@ -77,17 +80,61 @@ module CompletionKit
         run.avg_score = stats && stats[:avg] ? stats[:avg].to_f.round(2) : nil
         run.check_pass_rate = stats && stats[:resolved] > 0 ? (stats[:passed].to_f / stats[:resolved]).round(2) : nil
 
-        run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, resolved, passed)|
+        run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed)|
           if scored.to_i > 0
-            {name: name, avg: avg.to_f.round(1)}
+            {name: name, avg: avg.to_f.round(1), count: scored.to_i, low_count: low.to_i}
           elsif resolved.to_i > 0
-            {name: name, kind: "check", pass_rate: (passed.to_i.to_f / resolved.to_i).round(2)}
+            {name: name, kind: "check", pass_rate: (passed.to_i.to_f / resolved.to_i).round(2),
+             count: resolved.to_i, low_count: resolved.to_i - passed.to_i}
           end
         end
       end
       runs
     end
 
+    # Scores below this are the ones worth reading: `low_count` on each metric
+    # average counts them, so a caller can spot the dragging metric without
+    # pulling every review.
+    def self.low_score_ceiling
+      CompletionKit.config.medium_quality_threshold.to_f
+    end
+
+    TOP_SCORE = 5.0
+    CEILING_MEAN = 4.8
+    CEILING_TOP_SHARE = 0.9
+    CEILING_MIN_REVIEWS = 10
+
+    # A judge that scores almost everything at the top is usually failing to
+    # separate good output from bad, which reads as success to anyone who has
+    # not calibrated it. Detected either as a near-max mean or as nearly every
+    # score landing on the top band, and only once there are enough scores for
+    # the shape to mean anything.
+    def scores_at_ceiling?
+      return false unless status == "completed"
+
+      scores = reviews_for_summary.filter_map { |review| review.ai_score&.to_f }
+      return false if scores.length < CEILING_MIN_REVIEWS
+      return true if (scores.sum / scores.length) >= CEILING_MEAN
+
+      (scores.count { |score| score >= TOP_SCORE }.to_f / scores.length) >= CEILING_TOP_SHARE
+    end
+
+    def calibratable_metric
+      llm_metrics.first
+    end
+
+    # How often a human agreed with the judge on this run's own responses.
+    # Scoped to the run rather than to a metric's current version, so the
+    # figure describes the scores actually shown on this page. Uses the same
+    # Wilson point as MetricAgreementStats so the two surfaces never disagree.
+    def judge_agreement
+      verdicts = Agreement.where(run_id: id).pluck(:verdict)
+      return nil if verdicts.empty?
+
+      point = AgreementMath.wilson_interval(successes: verdicts.count { |v| v == "agree" }, n: verdicts.length)[:point]
+      { rate: point, sample_size: verdicts.length }
+    end
+
     # A scoring-only run grades a pre-existing column on the dataset instead of
     # generating new outputs. No prompt is attached; the response text is read
     # from row[output_column]; no LLM generation happens.
@@ -105,8 +152,12 @@ module CompletionKit
     end
 
     def mark_completed!
-      update!(status: "completed")
-      broadcast_ui
+      if all_responses_failed?
+        fail_with_summary!(all_failed_summary)
+      else
+        update!(status: "completed")
+        broadcast_ui
+      end
     end
 
     def gradable_metric_ids
@@ -198,17 +249,20 @@ module CompletionKit
     def metric_averages
       return @metric_averages if defined?(@metric_averages)
 
+      ceiling = self.class.low_score_ceiling
       reviews_for_summary.group_by(&:metric_name).filter_map do |name, reviews|
         scored = reviews.select { |r| r.ai_score.present? }
         if scored.any?
           scores = scored.map { |r| r.ai_score.to_f }
-          { name: name, avg: (scores.sum / scores.length).round(1) }
+          { name: name, avg: (scores.sum / scores.length).round(1), count: scores.length,
+            low_count: scores.count { |score| score < ceiling } }
         else
           resolved = reviews.reject { |r| r.passed.nil? }
           next if resolved.empty?
 
           passed = resolved.count { |r| r.passed == true }
-          { name: name, kind: "check", pass_rate: (passed.to_f / resolved.length).round(2) }
+          { name: name, kind: "check", pass_rate: (passed.to_f / resolved.length).round(2),
+            count: resolved.length, low_count: resolved.length - passed }
         end
       end
     end
@@ -251,18 +305,17 @@ module CompletionKit
       summary
     end
 
+    # Validates what can be checked cheaply, claims the run, and hands the
+    # expensive part (parsing the dataset, inserting a response per row, and
+    # enqueueing a job per row) to StartRunJob. Holding all that open inside the
+    # caller's request is what made runs_generate look like it had timed out
+    # when the run had in fact started.
     def start!
       unless %w[pending failed].include?(status)
         return fail_with_summary!("Cannot start a run in state \"#{status}\". Use rerun to create a fresh copy, or retry_failures / regrade to work with the existing responses.")
       end
 
-      rows = if dataset
-               CsvProcessor.process_self(self)
-             else
-               [{}]
-             end
-
-      return fail_with_summary!("Dataset has no rows") if rows.empty?
+      return fail_with_summary!("Dataset has no rows") if dataset && dataset.row_count.zero?
 
       if judge_only?
         column = output_column.presence || "actual_output"
@@ -282,17 +335,39 @@ module CompletionKit
           update!(
             status: "running",
             progress_current: 0,
-            progress_total: rows.length,
+            progress_total: 0,
             failure_summary: nil,
             error_message: nil
           )
+        end
+      rescue ActiveRecord::RecordInvalid => e
+        reload
+        return fail_with_summary!(e.record.errors.full_messages.to_sentence)
+      end
+
+      StartRunJob.perform_later(id, Response.all.where_values_hash.symbolize_keys)
+      safely_broadcast { broadcast_ui }
+      true
+    end
+
+    def fail_to_start!(message)
+      fail_with_summary!(message)
+    end
+
+    def execute_start!(scope_defaults = {})
+      rows = dataset ? CsvProcessor.process_self(self) : [{}]
+
+      return fail_with_summary!("Dataset has no rows") if rows.empty?
+
+      begin
+        transaction do
+          update!(progress_current: 0, progress_total: rows.length)
 
           now = Time.current
           out_col = output_column.presence || "actual_output"
           exp_col = expected_column.presence || "expected_output"
           judge = judge_only?
           has_output = judge && dataset && dataset.headers.include?(out_col)
-          scope_defaults = Response.all.where_values_hash.symbolize_keys
 
           response_attrs = rows.each_with_index.map do |row, index|
             {
@@ -374,12 +449,35 @@ module CompletionKit
       true
     end
 
+    # The options every generation call for this run sends to the provider.
+    # max_tokens is omitted when unset so each client keeps its own default;
+    # setting it is how a run reproduces a production cap and stops the judge
+    # scoring truncated output.
+    def generation_options(prompt)
+      options = {model: prompt.llm_model, temperature: temperature}
+      options[:max_tokens] = max_tokens if max_tokens
+      options
+    end
+
+    # Everything JudgeService needs to score this run. Judging defaults to
+    # temperature 0 so re-judging the same output yields the same score;
+    # anything above that makes the run's numbers irreproducible.
+    def judge_config
+      ApiConfig.for_model(judge_model).merge(judge_model: judge_model, judge_temperature: judge_temperature)
+    end
+
+    def nondeterministic_judge?
+      judge_temperature.to_f > 0
+    end
+
     def rerun!
       new_run = Run.create!(
         prompt_id: prompt_id,
         dataset_id: dataset_id,
         judge_model: judge_model,
         temperature: temperature,
+        max_tokens: max_tokens,
+        judge_temperature: judge_temperature,
         output_column: output_column,
         expected_column: expected_column,
         tag_names: tag_names,
@@ -460,8 +558,10 @@ module CompletionKit
         output_column: output_column,
         expected_column: expected_column,
         created_at: created_at, updated_at: updated_at,
+        max_tokens: max_tokens, judge_temperature: judge_temperature,
         responses_count: responses.count, avg_score: avg_score,
         check_pass_rate: check_pass_rate,
+        metric_averages: metric_averages,
         progress_current: snap[:generated_done],
         progress_total: snap[:generated_total],
         progress: {
@@ -565,6 +665,17 @@ module CompletionKit
       false
     end
 
+    def all_responses_failed?
+      responses.exists? && !responses.where.not(status: "failed").exists?
+    end
+
+    def all_failed_summary
+      total = responses.count
+      reason = responses.where(status: "failed").where.not(error_message: [nil, ""]).order(:row_index).limit(1).pick(:error_message)
+      head = "Every response failed to generate (#{total} of #{total})."
+      reason.present? ? "#{head} First error: #{reason}" : "#{head} Check the model and provider configuration."
+    end
+
     def render_engine_partial(partial, locals)
       CompletionKit::Engine.warm_routes!
       CompletionKit::ApplicationController.render(