completion-kit 0.3.0 → 0.4.0

data/app/models/completion_kit/run.rb

@@ -2,7 +2,7 @@ module CompletionKit
   class Run < ApplicationRecord
     include Turbo::Broadcastable
 
-    STATUSES = %w[pending generating judging completed failed].freeze
+    STATUSES = %w[pending running completed failed].freeze
 
     belongs_to :prompt
     belongs_to :dataset, optional: true
@@ -17,6 +17,30 @@ module CompletionKit
     before_validation :set_default_status, on: :create
     before_validation :set_auto_name, on: :create
 
+    def mark_completed!
+      update!(status: "completed")
+      broadcast_ui
+    end
+
+    def outstanding_work_zero?
+      return false if responses.where.not(status: Response::TERMINAL_STATUSES).exists?
+
+      metric_ids = metrics.pluck(:id)
+      return true if metric_ids.empty?
+
+      succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
+      expected_reviews = succeeded_response_ids.size * metric_ids.size
+      return true if expected_reviews.zero?
+
+      terminal_review_count = Review.where(
+        response_id: succeeded_response_ids,
+        metric_id: metric_ids,
+        status: Review::TERMINAL_STATUSES
+      ).count
+
+      terminal_review_count >= expected_reviews
+    end
+
     def judge_configured?
       judge_model.present? && metrics.any? && ApiConfig.valid_for_model?(judge_model)
     end
@@ -45,134 +69,104 @@ module CompletionKit
       end
     end
 
-    def generate_responses!
+    def start!
       rows = if dataset
                CsvProcessor.process_self(self)
              else
                [{}]
              end
 
-      if rows.empty?
-        errors.add(:base, "Dataset has no rows")
-        return false
-      end
+      return fail_with_summary!("Dataset has no rows") if rows.empty?
 
       client = LlmClient.for_model(prompt.llm_model, ApiConfig.for_model(prompt.llm_model))
-
       unless client.configured?
-        msg = "LLM API not configured: #{client.configuration_errors.join(', ')}"
-        errors.add(:base, msg)
-        update_columns(status: "failed", error_message: msg) if persisted?
-        return false
+        return fail_with_summary!("LLM API not configured: #{client.configuration_errors.join(', ')}")
       end
 
-      update!(status: "generating", progress_current: 0, progress_total: rows.length, error_message: nil)
-      responses.destroy_all
-      broadcast_ui
-      broadcast_clear_responses
-
-      rows.each_with_index do |row, index|
-        input = row.empty? ? nil : row.to_json
-        rendered = CsvProcessor.apply_variables(prompt, row)
-        response_text = client.generate_completion(rendered, model: prompt.llm_model, temperature: temperature)
-
-        resp = responses.create!(
-          input_data: input,
-          response_text: response_text,
-          expected_output: row["expected_output"]
+      transaction do
+        responses.destroy_all
+        update!(
+          status: "running",
+          progress_current: 0,
+          progress_total: rows.length,
+          failure_summary: nil,
+          error_message: nil
         )
-
-        update_columns(progress_current: index + 1)
-        broadcast_progress
-        broadcast_response(resp)
-      end
-
-      if judge_configured?
-        judge_responses!
-      else
-        update!(status: "completed")
-        broadcast_ui
+        rows.each_with_index do |row, index|
+          input = row.empty? ? nil : row.to_json
+          response = responses.create!(
+            status: "pending",
+            row_index: index,
+            input_data: input,
+            expected_output: row["expected_output"]
+          )
+          GenerateRowJob.perform_later(id, response.id)
+        end
       end
 
-      true
-    rescue Faraday::Error => e
-      update_columns(status: "failed", error_message: e.message)
-      errors.add(:base, e.message)
       broadcast_ui
-      false
-    rescue StandardError => e
-      update_columns(status: "failed", error_message: e.message) if persisted?
-      errors.add(:base, e.message)
-      broadcast_ui if persisted?
-      false
+      broadcast_clear_responses
+      true
     end
 
-    def judge_responses!
-      total_evaluations = responses.count * metrics.count
-      update!(status: "judging", progress_current: 0, progress_total: total_evaluations, error_message: nil)
-      broadcast_ui
-
-      judge = JudgeService.new(ApiConfig.for_model(judge_model).merge(judge_model: judge_model))
-      evaluation_count = 0
-
-      responses.find_each do |response|
-        metrics.each do |metric|
-          evaluation = judge.evaluate(
-            response.response_text,
-            response.expected_output,
-            prompt.template,
-            criteria: metric.instruction.to_s,
-            rubric_text: metric.display_rubric_text,
-            input_data: response.input_data
-          )
+    def generate_responses!
+      start!
+    end
 
-          response.reviews.find_or_initialize_by(metric_id: metric.id).tap do |review|
-            review.assign_attributes(
-              metric_name: metric.name,
-              instruction: metric.instruction.to_s,
-              status: "evaluated",
-              ai_score: evaluation[:score],
-              ai_feedback: evaluation[:feedback]
-            )
-            review.save!
-          end
-
-          evaluation_count += 1
-          update_columns(progress_current: evaluation_count)
-          broadcast_progress
-        end
+    def progress_snapshot
+      generated_done = responses.where(status: "succeeded").count
+      generated_failed = responses.where(status: "failed").count
+      generated_total = progress_total
 
-        broadcast_response_update(response)
-      end
+      metric_count = metrics.count
+      succeeded_count = generated_done
+      judged_total = succeeded_count * metric_count
+      judged_done = Review.joins(:response)
+        .where(completion_kit_responses: { run_id: id }, status: "succeeded").count
+      judged_failed = Review.joins(:response)
+        .where(completion_kit_responses: { run_id: id }, status: "failed").count
 
-      update!(status: "completed")
-      broadcast_ui
-      true
-    rescue Faraday::Error => e
-      update_columns(status: "failed", error_message: e.message)
-      errors.add(:base, e.message)
-      broadcast_ui
-      false
-    rescue StandardError => e
-      update_columns(status: "failed", error_message: e.message) if persisted?
-      errors.add(:base, e.message)
-      broadcast_ui if persisted?
-      false
+      {
+        generated_done: generated_done,
+        generated_total: generated_total,
+        generated_failed: generated_failed,
+        judged_done: judged_done,
+        judged_total: judged_total,
+        judged_failed: judged_failed
+      }
     end
 
     def as_json(options = {})
+      snap = progress_snapshot
       {
         id: id, name: name, status: status, prompt_id: prompt_id,
         dataset_id: dataset_id, judge_model: judge_model, temperature: temperature,
         created_at: created_at, updated_at: updated_at,
         responses_count: responses.count, avg_score: avg_score,
-        progress_current: progress_current, progress_total: progress_total,
-        error_message: error_message, metric_ids: metric_ids
+        progress_current: snap[:generated_done],
+        progress_total: snap[:generated_total],
+        progress: {
+          generated: { done: snap[:generated_done], total: snap[:generated_total], failed: snap[:generated_failed] },
+          judged:    { done: snap[:judged_done],    total: snap[:judged_total],    failed: snap[:judged_failed] }
+        },
+        failed_response_ids: responses.where(status: "failed").pluck(:id),
+        failure_summary: failure_summary,
+        error_message: error_message,
+        metric_ids: metric_ids
       }
     end
 
     private
 
+    def fail_with_summary!(message)
+      errors.add(:base, message)
+      if persisted?
+        update_columns(status: "failed", failure_summary: message, error_message: message)
+        broadcast_ui
+      end
+      false
+    end
+
     def broadcast_ui
       broadcast_progress
       broadcast_status_header

data/app/services/completion_kit/anthropic_client.rb

@@ -27,12 +27,25 @@ module CompletionKit
         }.to_json
       end
       
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "anthropic",
+          status: 429,
+          retry_after: nil
+        )
+      end
+
       if response.success?
         data = JSON.parse(response.body)
         data["content"][0]["text"].strip
       else
         "Error: #{response.status} - #{response.body}"
       end
+    rescue CompletionKit::RateLimitError
+      raise
+    rescue Faraday::Error
+      raise
     rescue => e
       "Error: #{e.message}"
     end

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -49,11 +49,6 @@ module CompletionKit
           description: "Generate responses for a run using its prompt and dataset",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :generate
-        },
-        "runs_judge" => {
-          description: "Judge responses for a run using configured metrics",
-          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
-          handler: :judge
         }
       }.freeze
 
@@ -92,14 +87,11 @@ module CompletionKit
 
       def self.generate(args)
         run = Run.find(args["id"])
-        GenerateJob.perform_later(run.id)
-        text_result(run.reload.as_json)
-      end
-
-      def self.judge(args)
-        run = Run.find(args["id"])
-        JudgeJob.perform_later(run.id)
-        text_result(run.reload.as_json)
+        if run.start!
+          text_result(run.reload.as_json)
+        else
+          text_result(run.failure_summary || run.errors.full_messages.to_sentence)
+        end
       end
     end
   end

data/app/services/completion_kit/ollama_client.rb

@@ -19,12 +19,25 @@ module CompletionKit
         }.to_json
       end
 
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "ollama",
+          status: 429,
+          retry_after: nil
+        )
+      end
+
       if response.success?
         data = JSON.parse(response.body)
         data["choices"][0]["text"].strip
       else
         "Error: #{response.status} - #{response.body}"
       end
+    rescue CompletionKit::RateLimitError
+      raise
+    rescue Faraday::Error
+      raise
     rescue => e
       "Error: #{e.message}"
     end

data/app/services/completion_kit/open_ai_client.rb

@@ -27,12 +27,23 @@ module CompletionKit
         }.to_json
       end
 
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "openai",
+          status: 429,
+          retry_after: response.headers && response.headers["Retry-After"]&.to_i
+        )
+      end
+
       if response.success?
         data = JSON.parse(response.body)
         data["output"][0]["content"][0]["text"].strip
       else
         "Error: #{response.status} - #{response.body}"
       end
+    rescue CompletionKit::RateLimitError
+      raise
     rescue Faraday::Error => e
       raise
     rescue => e

data/app/services/completion_kit/open_router_client.rb

@@ -25,12 +25,25 @@ module CompletionKit
         }.to_json
       end
 
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "openrouter",
+          status: 429,
+          retry_after: response.headers && response.headers["Retry-After"]&.to_i
+        )
+      end
+
       if response.success?
         data = JSON.parse(response.body)
         data.dig("choices", 0, "message", "content").to_s.strip
       else
         "Error: #{response.status} - #{response.body}"
       end
+    rescue CompletionKit::RateLimitError
+      raise
+    rescue Faraday::Error
+      raise
     rescue => e
       "Error: #{e.message}"
     end

data/app/services/completion_kit/worker_health.rb

@@ -0,0 +1,10 @@
+module CompletionKit
+  class WorkerHealth
+    HEARTBEAT_THRESHOLD = 30.seconds
+
+    def self.healthy?
+      return true unless defined?(::SolidQueue::Process)
+      ::SolidQueue::Process.where("last_heartbeat_at > ?", HEARTBEAT_THRESHOLD.ago).exists?
+    end
+  end
+end

data/app/views/completion_kit/api_reference/index.html.erb

@@ -174,11 +174,6 @@ end %>
         <p class="ck-meta-copy">Start generating responses. Returns 202 Accepted. Poll the run to check progress.</p>
         <%= render "example", base_url: @base_url, token: token_display, real_token: @token, cmd: "curl -X POST #{@base_url}/api/v1/runs/1/generate \\\n  -H \"Authorization: Bearer #{token_display}\"" %>
       </div>
-      <div class="ck-api-endpoint">
-        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:id/judge</p>
-        <p class="ck-meta-copy">Start judging responses with the configured judge model and metrics. Returns 202 Accepted.</p>
-        <%= render "example", base_url: @base_url, token: token_display, real_token: @token, cmd: "curl -X POST #{@base_url}/api/v1/runs/1/judge \\\n  -H \"Authorization: Bearer #{token_display}\"" %>
-      </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">PATCH</span> /api/v1/runs/:id</p>
         <p class="ck-meta-copy">Update a run. Accepts same params as create.</p>

data/app/views/completion_kit/prompts/_form.html.erb

@@ -32,21 +32,24 @@
       <% available = CompletionKit::ApiConfig.available_models(scope: :generation) %>
       <% if available.any? %>
         <div class="ck-select-with-action">
-          <%= form.select :llm_model, ck_grouped_models(available, prompt.llm_model), {}, { class: "ck-input", id: "prompt_llm_model" } %>
+          <%= form.select :llm_model, ck_grouped_models(available, prompt.llm_model), { include_blank: "— Select a model —" }, { class: "ck-input", id: "prompt_llm_model" } %>
           <button type="button" class="ck-icon-btn" title="Refresh models" onclick="ckRefreshModels()"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="16" height="16"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg></button>
         </div>
       <% else %>
         <p class="ck-meta-copy">No models available. <%= link_to "Add a provider", provider_credentials_path, class: "ck-link" %> or click refresh after configuring a provider.</p>
       <% end %>
-      <% CompletionKit::ProviderCredential.find_each do |pc| %>
-        <%= turbo_stream_from "completion_kit_provider_#{pc.id}" %>
-      <% end %>
+      <div hidden data-refresh-progress-carriers>
+        <% CompletionKit::ProviderCredential.find_each do |pc| %>
+          <%= turbo_stream_from "completion_kit_provider_#{pc.id}" %>
+          <%= render "completion_kit/provider_credentials/discovery_status", provider_credential: pc, show_completed: false %>
+        <% end %>
+      </div>
       <p class="ck-field-hint" id="refresh-status" style="min-height: 1.2em; margin-top: -0.25rem; font-size: 0.75rem;">&nbsp;</p>
     </div>
 
     <div class="ck-actions">
       <%= link_to "Cancel", prompts_path, class: ck_button_classes(:light, variant: :outline) %>
-      <%= form.submit(prompt.persisted? ? "Save prompt" : "Create prompt", class: ck_button_classes(:dark)) %>
+      <%= form.submit(prompt.persisted? ? "Save prompt" : "Create prompt", class: ck_button_classes(:dark), disabled: available.empty?) %>
     </div>
   </div>
 <% end %>

data/app/views/completion_kit/runs/_actions.html.erb

@@ -1,5 +1,5 @@
 <div class="ck-actions" id="run_actions">
-  <% running = run.status == "generating" || run.status == "judging" %>
+  <% running = run.status == "running" %>
   <%= button_to run_path(run), method: :delete, form_class: "inline-block", class: "ck-icon-btn", title: "Delete run", "aria-label": "Delete run", disabled: running, data: { turbo_confirm: "Delete this run and all its responses?" } do %><%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
   <% if running %>
     <%= link_to "Edit", edit_run_path(run), class: ck_button_classes(:light, variant: :outline) + " disabled", "aria-disabled": "true", tabindex: "-1" %>

data/app/views/completion_kit/runs/_form.html.erb

@@ -54,9 +54,12 @@
           <button type="button" class="ck-icon-btn" title="Refresh models" onclick="ckRefreshModels()"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="16" height="16"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg></button>
         </div>
         <p class="ck-field-hint" id="judge-hint"></p>
-        <% CompletionKit::ProviderCredential.find_each do |pc| %>
-          <%= turbo_stream_from "completion_kit_provider_#{pc.id}" %>
-        <% end %>
+        <div hidden data-refresh-progress-carriers>
+          <% CompletionKit::ProviderCredential.find_each do |pc| %>
+            <%= turbo_stream_from "completion_kit_provider_#{pc.id}" %>
+            <%= render "completion_kit/provider_credentials/discovery_status", provider_credential: pc, show_completed: false %>
+          <% end %>
+        </div>
         <p class="ck-field-hint" id="refresh-status" style="min-height: 1.2em; margin-top: -0.25rem; font-size: 0.75rem;">&nbsp;</p>
       <% else %>
         <p class="ck-field-hint" style="color: var(--ck-warning);">No judge models available.&ensp;<%= link_to "Add a provider", provider_credentials_path, class: "ck-link" %></p>

data/app/views/completion_kit/runs/_progress.html.erb

@@ -1,5 +1,5 @@
 <div id="run_progress">
-  <% if run.status == "generating" || run.status == "judging" %>
+  <% if run.status == "running" %>
     <div class="ck-discovery-bar">
       <div class="ck-discovery-bar__label">
         <%= ck_run_status_label(run) %>

data/app/views/completion_kit/runs/_response_row.html.erb

@@ -1,13 +1,31 @@
-<%= link_to run_response_path(run, response, sort: params[:sort]), class: "ck-response-row", id: "response_#{response.id}" do %>
+<%= link_to run_response_path(run, response, sort: params[:sort]), class: "ck-response-row ck-response-row--#{response.status}", id: "response_#{response.id}" do %>
   <span class="ck-response-row__index">#<%= index %></span>
-  <span class="ck-response-row__text"><%= truncate(response.response_text.to_s, length: 160) %></span>
+  <span class="ck-response-row__text">
+    <% if response.succeeded? %>
+      <%= truncate(response.response_text.to_s, length: 160) %>
+    <% elsif response.status == "failed" %>
+      <% err = response.error_payload %>
+      <span class="ck-response-row__error">
+        <%= err && err[:provider]&.titleize %><%= " #{err[:status]}" if err && err[:status] %> — <%= truncate(err && err[:message].to_s, length: 120) %>
+      </span>
+    <% end %>
+  </span>
   <span class="ck-response-row__score">
-    <% if response.reviewed? %>
-      <span class="ck-score"><span class="ck-score__star">★</span> <%= response.score %></span>
-    <% elsif run.status == "failed" %>
-      <span class="ck-chip">Failed</span>
-    <% elsif run.status == "judging" %>
-      <span class="ck-chip">Judging</span>
+    <% case response.status
+       when "succeeded" %>
+      <% if response.reviewed? %>
+        <span class="ck-score"><span class="ck-score__star">★</span> <%= response.score %></span>
+      <% elsif run.status == "running" %>
+        <span class="ck-chip">Judging</span>
+      <% end %>
+    <% when "pending" %>
+      <span class="ck-chip">Queued</span>
+    <% when "retrying" %>
+      <span class="ck-chip ck-chip--warning">Retrying <%= response.attempts %>/5</span>
+    <% when "failed" %>
+      <%= link_to "Retry", retry_failures_run_path(run, only: response.id),
+            data: { turbo_method: :post },
+            class: "ck-chip ck-chip--danger ck-chip--retry" %>
     <% end %>
   </span>
 <% end %>

data/app/views/completion_kit/runs/_status_header.html.erb

@@ -1,9 +1,17 @@
+<% snap = run.progress_snapshot %>
 <div id="run_status_header">
   <% if run.status == "failed" %>
     <div class="ck-flash ck-flash--alert">
-      <%= run.error_message.presence || "Run failed." %>
+      <%= run.failure_summary.presence || run.error_message.presence || "Run failed." %>
     </div>
   <% end %>
+
+  <% if run.status == "running" && !CompletionKit::WorkerHealth.healthy? %>
+    <div class="ck-flash ck-flash--alert">
+      No worker process is running. Generate and judge jobs are queued but nothing is processing them. Start <code>bin/jobs</code> (or your worker service) to resume.
+    </div>
+  <% end %>
+
   <section class="ck-page-header">
     <div>
       <p class="ck-kicker"><span class="<%= ck_run_dot(run) %>"></span> <%= ck_run_status_label(run) %></p>
@@ -12,4 +20,31 @@
     </div>
     <%= render "completion_kit/runs/actions", run: run %>
   </section>
+
+  <% if run.status.in?(%w[running completed]) && snap[:generated_total] > 0 %>
+    <div class="ck-progress-block">
+      <div class="ck-progress-line">
+        Generated <%= snap[:generated_done] %>/<%= snap[:generated_total] %>
+        <% if snap[:generated_failed] > 0 %>
+          <span class="ck-progress-failed">(<%= snap[:generated_failed] %> failed)</span>
+        <% end %>
+      </div>
+      <% if snap[:judged_total] > 0 %>
+        <div class="ck-progress-line">
+          Judged <%= snap[:judged_done] %>/<%= snap[:judged_total] %>
+          <% if snap[:judged_failed] > 0 %>
+            <span class="ck-progress-failed">(<%= snap[:judged_failed] %> failed)</span>
+          <% end %>
+        </div>
+      <% end %>
+      <% failed_count = snap[:generated_failed] + snap[:judged_failed] %>
+      <% if failed_count > 0 %>
+        <%= button_to "Retry #{failed_count} failed #{"row".pluralize(failed_count)}",
+              retry_failures_run_path(run),
+              method: :post,
+              class: ck_button_classes(:light, variant: :outline),
+              form_class: "inline-block" %>
+      <% end %>
+    </div>
+  <% end %>
 </div>

data/app/views/completion_kit/runs/show.html.erb

@@ -52,7 +52,7 @@
   </div>
   <p class="ck-prompt-preview__text" id="prompt_text"><%= @run.prompt.template %></p>
   <% if @run.prompt.template.length > 200 %>
-    <button type="button" class="ck-link" id="prompt_toggle" aria-expanded="false" aria-controls="prompt_text" onclick="var t=document.getElementById('prompt_text');var l=this;var expanded=t.classList.toggle('ck-prompt-preview__text--expanded');l.textContent=expanded?'Show less':'Show more';l.setAttribute('aria-expanded',expanded?'true':'false')">Show more</button>
+    <button type="button" class="ck-disclosure-toggle" id="prompt_toggle" aria-expanded="false" aria-controls="prompt_text" onclick="var t=document.getElementById('prompt_text');var l=this;var expanded=t.classList.toggle('ck-prompt-preview__text--expanded');l.firstChild.textContent=expanded?'Show less':'Show more';l.setAttribute('aria-expanded',expanded?'true':'false')"><span>Show more</span></button>
   <% end %>
 </div>
 

data/app/views/layouts/completion_kit/application.html.erb

@@ -56,8 +56,7 @@ function ckRefreshModels() {
   ckRefreshing = true;
   var btn = document.querySelector('.ck-icon-btn[title="Refresh models"]');
   if (btn) btn.classList.add('ck-icon-btn--spinning');
-  var status = document.getElementById('refresh-status');
-  if (status) status.textContent = 'Refreshing models\u2026';
+  ckUpdateRefreshProgress();
   var csrfToken = document.querySelector('meta[name="csrf-token"]').getAttribute("content");
   fetch("/completion_kit/refresh_models", {
     method: "POST",
@@ -65,8 +64,35 @@ function ckRefreshModels() {
   });
 }
 
+function ckUpdateRefreshProgress() {
+  var status = document.getElementById('refresh-status');
+  if (!status) return;
+  var carriers = document.querySelectorAll('[data-refresh-progress-carriers] [id^="discovery_status_"]');
+  var totalCurrent = 0, totalTotal = 0, anyDiscovering = false;
+  carriers.forEach(function(node) {
+    if (!node.querySelector('.ck-discovery-bar')) return;
+    if (node.querySelector('.ck-discovery-bar--failed') || node.querySelector('.ck-discovery-bar--completed')) return;
+    anyDiscovering = true;
+    var match = node.textContent.match(/(\d+)\s*\/\s*(\d+)/);
+    if (match) {
+      totalCurrent += parseInt(match[1], 10);
+      totalTotal += parseInt(match[2], 10);
+    }
+  });
+  if (anyDiscovering || ckRefreshing) {
+    if (totalTotal > 0) {
+      status.textContent = 'Refreshing models\u2026 ' + totalCurrent + '/' + totalTotal;
+    } else {
+      status.textContent = 'Refreshing models\u2026';
+    }
+  }
+}
+
 document.addEventListener("turbo:before-stream-render", function(event) {
   var target = event.target.getAttribute("target");
+  if (target && target.indexOf("discovery_status_") === 0) {
+    requestAnimationFrame(ckUpdateRefreshProgress);
+  }
   if (target === "prompt_llm_model" || target === "run_judge_model") {
     ckRefreshing = false;
     var btn = document.querySelector('.ck-icon-btn[title="Refresh models"]');

data/config/routes.rb

@@ -14,10 +14,10 @@ CompletionKit::Engine.routes.draw do
   resources :runs do
     member do
       post :generate
-      post :judge
       get :suggestion
       post :suggest
       post :apply_suggestion
+      post :retry_failures
     end
     resources :responses, only: [:show]
   end
@@ -39,7 +39,7 @@ CompletionKit::Engine.routes.draw do
       resources :runs do
         member do
           post :generate
-          post :judge
+          post :retry_failures
         end
         resources :responses, only: [:index, :show]
       end