completion-kit 0.2.1 → 0.4.0

data/app/jobs/completion_kit/run_completion_check_job.rb

@@ -0,0 +1,18 @@
+module CompletionKit
+  class RunCompletionCheckJob < ApplicationJob
+    queue_as :default
+
+    limits_concurrency to: 1,
+                       key: ->(run_id) { "run:#{run_id}:completion" },
+                       duration: 5.minutes
+
+    def perform(run_id)
+      run = Run.find_by(id: run_id)
+      return unless run
+      return unless run.status == "running"
+      return unless run.outstanding_work_zero?
+
+      run.mark_completed!
+    end
+  end
+end

data/app/models/completion_kit/prompt.rb

@@ -42,6 +42,10 @@ module CompletionKit
       "#{name} — #{version_label}"
     end
 
+    def llm_model_provider
+      ApiConfig.provider_for_model(llm_model)
+    end
+
     def family_versions
       self.class.where(family_key: family_key).order(version_number: :desc, created_at: :desc)
     end

data/app/models/completion_kit/response.rb

@@ -1,18 +1,34 @@
 module CompletionKit
   class Response < ApplicationRecord
+    STATUSES = %w[pending retrying succeeded failed].freeze
+    TERMINAL_STATUSES = %w[succeeded failed].freeze
+
     belongs_to :run
     has_many :reviews, dependent: :destroy
 
     delegate :prompt, to: :run
 
-    validates :response_text, presence: true
+    validates :response_text, presence: true, if: :succeeded?
+    validates :status, inclusion: { in: STATUSES }
+
+    before_validation :set_default_status, on: :create
+
+    def terminal?
+      TERMINAL_STATUSES.include?(status)
+    end
+
+    def succeeded?
+      status == "succeeded"
+    end
 
     def as_json(options = {})
       {
         id: id, run_id: run_id, input_data: input_data,
         response_text: response_text, expected_output: expected_output,
         created_at: created_at, score: score, reviewed: reviewed?,
-        reviews: reviews.map(&:as_json)
+        reviews: reviews.map(&:as_json),
+        status: status, attempts: attempts, row_index: row_index,
+        error: error_payload
       }
     end
 
@@ -26,5 +42,16 @@ module CompletionKit
     def reviewed?
       reviews.any? { |r| r.ai_score.present? }
     end
+
+    def error_payload
+      return nil if error_class.blank?
+      { provider: error_provider, class: error_class, status: error_status, message: error_message }
+    end
+
+    private
+
+    def set_default_status
+      self.status ||= "pending"
+    end
   end
 end

data/app/models/completion_kit/review.rb

@@ -1,6 +1,7 @@
 module CompletionKit
   class Review < ApplicationRecord
-    STATUSES = %w[pending evaluated failed].freeze
+    STATUSES = %w[pending retrying succeeded failed].freeze
+    TERMINAL_STATUSES = %w[succeeded failed].freeze
 
     belongs_to :response
     belongs_to :metric, optional: true
@@ -11,11 +12,25 @@ module CompletionKit
 
     before_validation :set_default_status
 
+    def terminal?
+      TERMINAL_STATUSES.include?(status)
+    end
+
+    def succeeded?
+      status == "succeeded"
+    end
+
+    def error_payload
+      return nil if error_class.blank?
+      { provider: error_provider, class: error_class, status: error_status, message: error_message }
+    end
+
     def as_json(options = {})
       {
         id: id, response_id: response_id, metric_id: metric_id,
         metric_name: metric_name, ai_score: ai_score,
-        ai_feedback: ai_feedback, status: status
+        ai_feedback: ai_feedback, status: status, attempts: attempts,
+        error: error_payload
       }
     end
 

data/app/models/completion_kit/run.rb

@@ -2,7 +2,7 @@ module CompletionKit
   class Run < ApplicationRecord
     include Turbo::Broadcastable
 
-    STATUSES = %w[pending generating judging completed failed].freeze
+    STATUSES = %w[pending running completed failed].freeze
 
     belongs_to :prompt
     belongs_to :dataset, optional: true
@@ -17,6 +17,30 @@ module CompletionKit
     before_validation :set_default_status, on: :create
     before_validation :set_auto_name, on: :create
 
+    def mark_completed!
+      update!(status: "completed")
+      broadcast_ui
+    end
+
+    def outstanding_work_zero?
+      return false if responses.where.not(status: Response::TERMINAL_STATUSES).exists?
+
+      metric_ids = metrics.pluck(:id)
+      return true if metric_ids.empty?
+
+      succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
+      expected_reviews = succeeded_response_ids.size * metric_ids.size
+      return true if expected_reviews.zero?
+
+      terminal_review_count = Review.where(
+        response_id: succeeded_response_ids,
+        metric_id: metric_ids,
+        status: Review::TERMINAL_STATUSES
+      ).count
+
+      terminal_review_count >= expected_reviews
+    end
+
     def judge_configured?
       judge_model.present? && metrics.any? && ApiConfig.valid_for_model?(judge_model)
     end
@@ -45,134 +69,104 @@ module CompletionKit
       end
     end
 
-    def generate_responses!
+    def start!
       rows = if dataset
                CsvProcessor.process_self(self)
              else
                [{}]
              end
 
-      if rows.empty?
-        errors.add(:base, "Dataset has no rows")
-        return false
-      end
+      return fail_with_summary!("Dataset has no rows") if rows.empty?
 
       client = LlmClient.for_model(prompt.llm_model, ApiConfig.for_model(prompt.llm_model))
-
       unless client.configured?
-        msg = "LLM API not configured: #{client.configuration_errors.join(', ')}"
-        errors.add(:base, msg)
-        update_columns(status: "failed", error_message: msg) if persisted?
-        return false
+        return fail_with_summary!("LLM API not configured: #{client.configuration_errors.join(', ')}")
       end
 
-      update!(status: "generating", progress_current: 0, progress_total: rows.length, error_message: nil)
-      responses.destroy_all
-      broadcast_ui
-      broadcast_clear_responses
-
-      rows.each_with_index do |row, index|
-        input = row.empty? ? nil : row.to_json
-        rendered = CsvProcessor.apply_variables(prompt, row)
-        response_text = client.generate_completion(rendered, model: prompt.llm_model, temperature: temperature)
-
-        resp = responses.create!(
-          input_data: input,
-          response_text: response_text,
-          expected_output: row["expected_output"]
+      transaction do
+        responses.destroy_all
+        update!(
+          status: "running",
+          progress_current: 0,
+          progress_total: rows.length,
+          failure_summary: nil,
+          error_message: nil
         )
-
-        update_columns(progress_current: index + 1)
-        broadcast_progress
-        broadcast_response(resp)
-      end
-
-      if judge_configured?
-        judge_responses!
-      else
-        update!(status: "completed")
-        broadcast_ui
+        rows.each_with_index do |row, index|
+          input = row.empty? ? nil : row.to_json
+          response = responses.create!(
+            status: "pending",
+            row_index: index,
+            input_data: input,
+            expected_output: row["expected_output"]
+          )
+          GenerateRowJob.perform_later(id, response.id)
+        end
       end
 
-      true
-    rescue Faraday::Error => e
-      update_columns(status: "failed", error_message: e.message)
-      errors.add(:base, e.message)
       broadcast_ui
-      false
-    rescue StandardError => e
-      update_columns(status: "failed", error_message: e.message) if persisted?
-      errors.add(:base, e.message)
-      broadcast_ui if persisted?
-      false
+      broadcast_clear_responses
+      true
     end
 
-    def judge_responses!
-      total_evaluations = responses.count * metrics.count
-      update!(status: "judging", progress_current: 0, progress_total: total_evaluations, error_message: nil)
-      broadcast_ui
-
-      judge = JudgeService.new(ApiConfig.for_model(judge_model).merge(judge_model: judge_model))
-      evaluation_count = 0
-
-      responses.find_each do |response|
-        metrics.each do |metric|
-          evaluation = judge.evaluate(
-            response.response_text,
-            response.expected_output,
-            prompt.template,
-            criteria: metric.instruction.to_s,
-            rubric_text: metric.display_rubric_text,
-            input_data: response.input_data
-          )
+    def generate_responses!
+      start!
+    end
 
-          response.reviews.find_or_initialize_by(metric_id: metric.id).tap do |review|
-            review.assign_attributes(
-              metric_name: metric.name,
-              instruction: metric.instruction.to_s,
-              status: "evaluated",
-              ai_score: evaluation[:score],
-              ai_feedback: evaluation[:feedback]
-            )
-            review.save!
-          end
-
-          evaluation_count += 1
-          update_columns(progress_current: evaluation_count)
-          broadcast_progress
-        end
+    def progress_snapshot
+      generated_done = responses.where(status: "succeeded").count
+      generated_failed = responses.where(status: "failed").count
+      generated_total = progress_total
 
-        broadcast_response_update(response)
-      end
+      metric_count = metrics.count
+      succeeded_count = generated_done
+      judged_total = succeeded_count * metric_count
+      judged_done = Review.joins(:response)
+        .where(completion_kit_responses: { run_id: id }, status: "succeeded").count
+      judged_failed = Review.joins(:response)
+        .where(completion_kit_responses: { run_id: id }, status: "failed").count
 
-      update!(status: "completed")
-      broadcast_ui
-      true
-    rescue Faraday::Error => e
-      update_columns(status: "failed", error_message: e.message)
-      errors.add(:base, e.message)
-      broadcast_ui
-      false
-    rescue StandardError => e
-      update_columns(status: "failed", error_message: e.message) if persisted?
-      errors.add(:base, e.message)
-      broadcast_ui if persisted?
-      false
+      {
+        generated_done: generated_done,
+        generated_total: generated_total,
+        generated_failed: generated_failed,
+        judged_done: judged_done,
+        judged_total: judged_total,
+        judged_failed: judged_failed
+      }
     end
 
     def as_json(options = {})
+      snap = progress_snapshot
       {
         id: id, name: name, status: status, prompt_id: prompt_id,
         dataset_id: dataset_id, judge_model: judge_model, temperature: temperature,
         created_at: created_at, updated_at: updated_at,
         responses_count: responses.count, avg_score: avg_score,
-        progress_current: progress_current, progress_total: progress_total,
-        error_message: error_message, metric_ids: metric_ids
+        progress_current: snap[:generated_done],
+        progress_total: snap[:generated_total],
+        progress: {
+          generated: { done: snap[:generated_done], total: snap[:generated_total], failed: snap[:generated_failed] },
+          judged:    { done: snap[:judged_done],    total: snap[:judged_total],    failed: snap[:judged_failed] }
+        },
+        failed_response_ids: responses.where(status: "failed").pluck(:id),
+        failure_summary: failure_summary,
+        error_message: error_message,
+        metric_ids: metric_ids
       }
     end
 
     private
 
+    def fail_with_summary!(message)
+      errors.add(:base, message)
+      if persisted?
+        update_columns(status: "failed", failure_summary: message, error_message: message)
+        broadcast_ui
+      end
+      false
+    end
+
     def broadcast_ui
       broadcast_progress
       broadcast_status_header

data/app/services/completion_kit/anthropic_client.rb

@@ -27,12 +27,25 @@ module CompletionKit
         }.to_json
       end
       
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "anthropic",
+          status: 429,
+          retry_after: nil
+        )
+      end
+
       if response.success?
         data = JSON.parse(response.body)
         data["content"][0]["text"].strip
       else
         "Error: #{response.status} - #{response.body}"
       end
+    rescue CompletionKit::RateLimitError
+      raise
+    rescue Faraday::Error
+      raise
     rescue => e
       "Error: #{e.message}"
     end

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -49,11 +49,6 @@ module CompletionKit
           description: "Generate responses for a run using its prompt and dataset",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :generate
-        },
-        "runs_judge" => {
-          description: "Judge responses for a run using configured metrics",
-          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
-          handler: :judge
         }
       }.freeze
 
@@ -92,14 +87,11 @@ module CompletionKit
 
       def self.generate(args)
         run = Run.find(args["id"])
-        GenerateJob.perform_later(run.id)
-        text_result(run.reload.as_json)
-      end
-
-      def self.judge(args)
-        run = Run.find(args["id"])
-        JudgeJob.perform_later(run.id)
-        text_result(run.reload.as_json)
+        if run.start!
+          text_result(run.reload.as_json)
+        else
+          text_result(run.failure_summary || run.errors.full_messages.to_sentence)
+        end
       end
     end
   end

data/app/services/completion_kit/ollama_client.rb

@@ -19,12 +19,25 @@ module CompletionKit
         }.to_json
       end
 
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "ollama",
+          status: 429,
+          retry_after: nil
+        )
+      end
+
       if response.success?
         data = JSON.parse(response.body)
         data["choices"][0]["text"].strip
       else
         "Error: #{response.status} - #{response.body}"
       end
+    rescue CompletionKit::RateLimitError
+      raise
+    rescue Faraday::Error
+      raise
     rescue => e
       "Error: #{e.message}"
     end

data/app/services/completion_kit/open_ai_client.rb

@@ -27,12 +27,23 @@ module CompletionKit
         }.to_json
       end
 
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "openai",
+          status: 429,
+          retry_after: response.headers && response.headers["Retry-After"]&.to_i
+        )
+      end
+
       if response.success?
         data = JSON.parse(response.body)
         data["output"][0]["content"][0]["text"].strip
       else
         "Error: #{response.status} - #{response.body}"
       end
+    rescue CompletionKit::RateLimitError
+      raise
     rescue Faraday::Error => e
       raise
     rescue => e

data/app/services/completion_kit/open_router_client.rb

@@ -25,12 +25,25 @@ module CompletionKit
         }.to_json
       end
 
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "openrouter",
+          status: 429,
+          retry_after: response.headers && response.headers["Retry-After"]&.to_i
+        )
+      end
+
       if response.success?
         data = JSON.parse(response.body)
         data.dig("choices", 0, "message", "content").to_s.strip
       else
         "Error: #{response.status} - #{response.body}"
       end
+    rescue CompletionKit::RateLimitError
+      raise
+    rescue Faraday::Error
+      raise
     rescue => e
       "Error: #{e.message}"
     end

data/app/services/completion_kit/worker_health.rb

@@ -0,0 +1,10 @@
+module CompletionKit
+  class WorkerHealth
+    HEARTBEAT_THRESHOLD = 30.seconds
+
+    def self.healthy?
+      return true unless defined?(::SolidQueue::Process)
+      ::SolidQueue::Process.where("last_heartbeat_at > ?", HEARTBEAT_THRESHOLD.ago).exists?
+    end
+  end
+end

data/app/views/completion_kit/api_reference/index.html.erb

@@ -174,11 +174,6 @@ end %>
         <p class="ck-meta-copy">Start generating responses. Returns 202 Accepted. Poll the run to check progress.</p>
         <%= render "example", base_url: @base_url, token: token_display, real_token: @token, cmd: "curl -X POST #{@base_url}/api/v1/runs/1/generate \\\n  -H \"Authorization: Bearer #{token_display}\"" %>
       </div>
-      <div class="ck-api-endpoint">
-        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:id/judge</p>
-        <p class="ck-meta-copy">Start judging responses with the configured judge model and metrics. Returns 202 Accepted.</p>
-        <%= render "example", base_url: @base_url, token: token_display, real_token: @token, cmd: "curl -X POST #{@base_url}/api/v1/runs/1/judge \\\n  -H \"Authorization: Bearer #{token_display}\"" %>
-      </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">PATCH</span> /api/v1/runs/:id</p>
         <p class="ck-meta-copy">Update a run. Accepts same params as create.</p>

data/app/views/completion_kit/prompts/_form.html.erb

@@ -32,21 +32,24 @@
       <% available = CompletionKit::ApiConfig.available_models(scope: :generation) %>
       <% if available.any? %>
         <div class="ck-select-with-action">
-          <%= form.select :llm_model, ck_grouped_models(available, prompt.llm_model), {}, { class: "ck-input", id: "prompt_llm_model" } %>
+          <%= form.select :llm_model, ck_grouped_models(available, prompt.llm_model), { include_blank: "— Select a model —" }, { class: "ck-input", id: "prompt_llm_model" } %>
           <button type="button" class="ck-icon-btn" title="Refresh models" onclick="ckRefreshModels()"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="16" height="16"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg></button>
         </div>
       <% else %>
         <p class="ck-meta-copy">No models available. <%= link_to "Add a provider", provider_credentials_path, class: "ck-link" %> or click refresh after configuring a provider.</p>
       <% end %>
-      <% CompletionKit::ProviderCredential.find_each do |pc| %>
-        <%= turbo_stream_from "completion_kit_provider_#{pc.id}" %>
-      <% end %>
+      <div hidden data-refresh-progress-carriers>
+        <% CompletionKit::ProviderCredential.find_each do |pc| %>
+          <%= turbo_stream_from "completion_kit_provider_#{pc.id}" %>
+          <%= render "completion_kit/provider_credentials/discovery_status", provider_credential: pc, show_completed: false %>
+        <% end %>
+      </div>
       <p class="ck-field-hint" id="refresh-status" style="min-height: 1.2em; margin-top: -0.25rem; font-size: 0.75rem;">&nbsp;</p>
     </div>
 
     <div class="ck-actions">
       <%= link_to "Cancel", prompts_path, class: ck_button_classes(:light, variant: :outline) %>
-      <%= form.submit(prompt.persisted? ? "Save prompt" : "Create prompt", class: ck_button_classes(:dark)) %>
+      <%= form.submit(prompt.persisted? ? "Save prompt" : "Create prompt", class: ck_button_classes(:dark), disabled: available.empty?) %>
     </div>
   </div>
 <% end %>

data/app/views/completion_kit/runs/_actions.html.erb

@@ -1,5 +1,5 @@
 <div class="ck-actions" id="run_actions">
-  <% running = run.status == "generating" || run.status == "judging" %>
+  <% running = run.status == "running" %>
   <%= button_to run_path(run), method: :delete, form_class: "inline-block", class: "ck-icon-btn", title: "Delete run", "aria-label": "Delete run", disabled: running, data: { turbo_confirm: "Delete this run and all its responses?" } do %><%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %><% end %>
   <% if running %>
     <%= link_to "Edit", edit_run_path(run), class: ck_button_classes(:light, variant: :outline) + " disabled", "aria-disabled": "true", tabindex: "-1" %>

data/app/views/completion_kit/runs/_form.html.erb

@@ -54,9 +54,12 @@
           <button type="button" class="ck-icon-btn" title="Refresh models" onclick="ckRefreshModels()"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="16" height="16"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg></button>
         </div>
         <p class="ck-field-hint" id="judge-hint"></p>
-        <% CompletionKit::ProviderCredential.find_each do |pc| %>
-          <%= turbo_stream_from "completion_kit_provider_#{pc.id}" %>
-        <% end %>
+        <div hidden data-refresh-progress-carriers>
+          <% CompletionKit::ProviderCredential.find_each do |pc| %>
+            <%= turbo_stream_from "completion_kit_provider_#{pc.id}" %>
+            <%= render "completion_kit/provider_credentials/discovery_status", provider_credential: pc, show_completed: false %>
+          <% end %>
+        </div>
         <p class="ck-field-hint" id="refresh-status" style="min-height: 1.2em; margin-top: -0.25rem; font-size: 0.75rem;">&nbsp;</p>
       <% else %>
         <p class="ck-field-hint" style="color: var(--ck-warning);">No judge models available.&ensp;<%= link_to "Add a provider", provider_credentials_path, class: "ck-link" %></p>

data/app/views/completion_kit/runs/_progress.html.erb

@@ -1,5 +1,5 @@
 <div id="run_progress">
-  <% if run.status == "generating" || run.status == "judging" %>
+  <% if run.status == "running" %>
     <div class="ck-discovery-bar">
       <div class="ck-discovery-bar__label">
         <%= ck_run_status_label(run) %>

data/app/views/completion_kit/runs/_response_row.html.erb

@@ -1,13 +1,31 @@
-<%= link_to run_response_path(run, response, sort: params[:sort]), class: "ck-response-row", id: "response_#{response.id}" do %>
+<%= link_to run_response_path(run, response, sort: params[:sort]), class: "ck-response-row ck-response-row--#{response.status}", id: "response_#{response.id}" do %>
   <span class="ck-response-row__index">#<%= index %></span>
-  <span class="ck-response-row__text"><%= truncate(response.response_text.to_s, length: 160) %></span>
+  <span class="ck-response-row__text">
+    <% if response.succeeded? %>
+      <%= truncate(response.response_text.to_s, length: 160) %>
+    <% elsif response.status == "failed" %>
+      <% err = response.error_payload %>
+      <span class="ck-response-row__error">
+        <%= err && err[:provider]&.titleize %><%= " #{err[:status]}" if err && err[:status] %> — <%= truncate(err && err[:message].to_s, length: 120) %>
+      </span>
+    <% end %>
+  </span>
   <span class="ck-response-row__score">
-    <% if response.reviewed? %>
-      <span class="ck-score"><span class="ck-score__star">★</span> <%= response.score %></span>
-    <% elsif run.status == "failed" %>
-      <span class="ck-chip">Failed</span>
-    <% elsif run.status == "judging" %>
-      <span class="ck-chip">Judging</span>
+    <% case response.status
+       when "succeeded" %>
+      <% if response.reviewed? %>
+        <span class="ck-score"><span class="ck-score__star">★</span> <%= response.score %></span>
+      <% elsif run.status == "running" %>
+        <span class="ck-chip">Judging</span>
+      <% end %>
+    <% when "pending" %>
+      <span class="ck-chip">Queued</span>
+    <% when "retrying" %>
+      <span class="ck-chip ck-chip--warning">Retrying <%= response.attempts %>/5</span>
+    <% when "failed" %>
+      <%= link_to "Retry", retry_failures_run_path(run, only: response.id),
+            data: { turbo_method: :post },
+            class: "ck-chip ck-chip--danger ck-chip--retry" %>
     <% end %>
   </span>
 <% end %>