completion-kit 0.2.1 → 0.4.0

data/app/views/completion_kit/runs/_status_header.html.erb

@@ -1,9 +1,17 @@
+<% snap = run.progress_snapshot %>
 <div id="run_status_header">
   <% if run.status == "failed" %>
     <div class="ck-flash ck-flash--alert">
-      <%= run.error_message.presence || "Run failed." %>
+      <%= run.failure_summary.presence || run.error_message.presence || "Run failed." %>
     </div>
   <% end %>
+
+  <% if run.status == "running" && !CompletionKit::WorkerHealth.healthy? %>
+    <div class="ck-flash ck-flash--alert">
+      No worker process is running. Generate and judge jobs are queued but nothing is processing them. Start <code>bin/jobs</code> (or your worker service) to resume.
+    </div>
+  <% end %>
+
   <section class="ck-page-header">
     <div>
       <p class="ck-kicker"><span class="<%= ck_run_dot(run) %>"></span> <%= ck_run_status_label(run) %></p>
@@ -12,4 +20,31 @@
     </div>
     <%= render "completion_kit/runs/actions", run: run %>
   </section>
+
+  <% if run.status.in?(%w[running completed]) && snap[:generated_total] > 0 %>
+    <div class="ck-progress-block">
+      <div class="ck-progress-line">
+        Generated <%= snap[:generated_done] %>/<%= snap[:generated_total] %>
+        <% if snap[:generated_failed] > 0 %>
+          <span class="ck-progress-failed">(<%= snap[:generated_failed] %> failed)</span>
+        <% end %>
+      </div>
+      <% if snap[:judged_total] > 0 %>
+        <div class="ck-progress-line">
+          Judged <%= snap[:judged_done] %>/<%= snap[:judged_total] %>
+          <% if snap[:judged_failed] > 0 %>
+            <span class="ck-progress-failed">(<%= snap[:judged_failed] %> failed)</span>
+          <% end %>
+        </div>
+      <% end %>
+      <% failed_count = snap[:generated_failed] + snap[:judged_failed] %>
+      <% if failed_count > 0 %>
+        <%= button_to "Retry #{failed_count} failed #{"row".pluralize(failed_count)}",
+              retry_failures_run_path(run),
+              method: :post,
+              class: ck_button_classes(:light, variant: :outline),
+              form_class: "inline-block" %>
+      <% end %>
+    </div>
+  <% end %>
 </div>

data/app/views/completion_kit/runs/show.html.erb

@@ -52,7 +52,7 @@
   </div>
   <p class="ck-prompt-preview__text" id="prompt_text"><%= @run.prompt.template %></p>
   <% if @run.prompt.template.length > 200 %>
-    <button type="button" class="ck-link" id="prompt_toggle" aria-expanded="false" aria-controls="prompt_text" onclick="var t=document.getElementById('prompt_text');var l=this;var expanded=t.classList.toggle('ck-prompt-preview__text--expanded');l.textContent=expanded?'Show less':'Show more';l.setAttribute('aria-expanded',expanded?'true':'false')">Show more</button>
+    <button type="button" class="ck-disclosure-toggle" id="prompt_toggle" aria-expanded="false" aria-controls="prompt_text" onclick="var t=document.getElementById('prompt_text');var l=this;var expanded=t.classList.toggle('ck-prompt-preview__text--expanded');l.firstChild.textContent=expanded?'Show less':'Show more';l.setAttribute('aria-expanded',expanded?'true':'false')"><span>Show more</span></button>
   <% end %>
 </div>
 

data/app/views/layouts/completion_kit/application.html.erb

@@ -56,8 +56,7 @@ function ckRefreshModels() {
   ckRefreshing = true;
   var btn = document.querySelector('.ck-icon-btn[title="Refresh models"]');
   if (btn) btn.classList.add('ck-icon-btn--spinning');
-  var status = document.getElementById('refresh-status');
-  if (status) status.textContent = 'Refreshing models\u2026';
+  ckUpdateRefreshProgress();
   var csrfToken = document.querySelector('meta[name="csrf-token"]').getAttribute("content");
   fetch("/completion_kit/refresh_models", {
     method: "POST",
@@ -65,8 +64,35 @@ function ckRefreshModels() {
   });
 }
 
+function ckUpdateRefreshProgress() {
+  var status = document.getElementById('refresh-status');
+  if (!status) return;
+  var carriers = document.querySelectorAll('[data-refresh-progress-carriers] [id^="discovery_status_"]');
+  var totalCurrent = 0, totalTotal = 0, anyDiscovering = false;
+  carriers.forEach(function(node) {
+    if (!node.querySelector('.ck-discovery-bar')) return;
+    if (node.querySelector('.ck-discovery-bar--failed') || node.querySelector('.ck-discovery-bar--completed')) return;
+    anyDiscovering = true;
+    var match = node.textContent.match(/(\d+)\s*\/\s*(\d+)/);
+    if (match) {
+      totalCurrent += parseInt(match[1], 10);
+      totalTotal += parseInt(match[2], 10);
+    }
+  });
+  if (anyDiscovering || ckRefreshing) {
+    if (totalTotal > 0) {
+      status.textContent = 'Refreshing models\u2026 ' + totalCurrent + '/' + totalTotal;
+    } else {
+      status.textContent = 'Refreshing models\u2026';
+    }
+  }
+}
+
 document.addEventListener("turbo:before-stream-render", function(event) {
   var target = event.target.getAttribute("target");
+  if (target && target.indexOf("discovery_status_") === 0) {
+    requestAnimationFrame(ckUpdateRefreshProgress);
+  }
   if (target === "prompt_llm_model" || target === "run_judge_model") {
     ckRefreshing = false;
     var btn = document.querySelector('.ck-icon-btn[title="Refresh models"]');

data/config/routes.rb

@@ -14,10 +14,10 @@ CompletionKit::Engine.routes.draw do
   resources :runs do
     member do
       post :generate
-      post :judge
       get :suggestion
       post :suggest
       post :apply_suggestion
+      post :retry_failures
     end
     resources :responses, only: [:show]
   end
@@ -39,7 +39,7 @@ CompletionKit::Engine.routes.draw do
       resources :runs do
         member do
           post :generate
-          post :judge
+          post :retry_failures
         end
         resources :responses, only: [:index, :show]
       end

data/db/migrate/20260501000001_add_status_and_error_to_responses.rb

@@ -0,0 +1,21 @@
+class AddStatusAndErrorToResponses < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_responses, :status, :string, default: "pending", null: false
+    add_column :completion_kit_responses, :error_provider, :string
+    add_column :completion_kit_responses, :error_class, :string
+    add_column :completion_kit_responses, :error_status, :integer
+    add_column :completion_kit_responses, :error_message, :text
+    add_column :completion_kit_responses, :attempts, :integer, default: 0, null: false
+    add_column :completion_kit_responses, :row_index, :integer
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+          UPDATE completion_kit_responses
+          SET status = 'succeeded'
+          WHERE response_text IS NOT NULL AND length(response_text) > 0
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20260501000002_index_responses_on_run_id_and_status.rb

@@ -0,0 +1,9 @@
+class IndexResponsesOnRunIdAndStatus < ActiveRecord::Migration[7.1]
+  disable_ddl_transaction!
+
+  def change
+    options = { if_not_exists: true }
+    options[:algorithm] = :concurrently unless connection.adapter_name == "SQLite"
+    add_index :completion_kit_responses, [:run_id, :status], **options
+  end
+end

data/db/migrate/20260501000003_add_status_and_error_to_reviews.rb

@@ -0,0 +1,25 @@
+class AddStatusAndErrorToReviews < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_reviews, :error_provider, :string
+    add_column :completion_kit_reviews, :error_class, :string
+    add_column :completion_kit_reviews, :error_status, :integer
+    add_column :completion_kit_reviews, :error_message, :text
+    add_column :completion_kit_reviews, :attempts, :integer, default: 0, null: false
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+          UPDATE completion_kit_reviews
+          SET status = 'succeeded'
+          WHERE ai_score IS NOT NULL
+        SQL
+
+        execute <<~SQL
+          UPDATE completion_kit_reviews
+          SET status = 'succeeded'
+          WHERE status = 'evaluated'
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20260501000004_index_reviews_on_response_id_and_status.rb

@@ -0,0 +1,9 @@
+class IndexReviewsOnResponseIdAndStatus < ActiveRecord::Migration[7.1]
+  disable_ddl_transaction!
+
+  def change
+    options = { if_not_exists: true }
+    options[:algorithm] = :concurrently unless connection.adapter_name == "SQLite"
+    add_index :completion_kit_reviews, [:response_id, :status], **options
+  end
+end

data/db/migrate/20260501000005_collapse_run_status_and_add_failure_summary.rb

@@ -0,0 +1,15 @@
+class CollapseRunStatusAndAddFailureSummary < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_runs, :failure_summary, :text
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+          UPDATE completion_kit_runs
+          SET status = 'running'
+          WHERE status IN ('generating', 'judging')
+        SQL
+      end
+    end
+  end
+end

data/lib/completion_kit/concurrency_check.rb

@@ -0,0 +1,16 @@
+module CompletionKit
+  module ConcurrencyCheck
+    def self.warn_if_misconfigured(logger)
+      threads = ENV.fetch("SOLID_QUEUE_THREADS", 10).to_i
+      llm_cap = ENV.fetch("COMPLETION_KIT_LLM_CONCURRENCY", 10).to_i
+
+      if threads < llm_cap
+        logger.warn(
+          "[CompletionKit] SOLID_QUEUE_THREADS=#{threads} is less than " \
+          "COMPLETION_KIT_LLM_CONCURRENCY=#{llm_cap}; threads will be the " \
+          "actual bottleneck and the per-provider cap will never be reached."
+        )
+      end
+    end
+  end
+end

data/lib/completion_kit/errors.rb

@@ -0,0 +1,16 @@
+module CompletionKit
+  class Error < StandardError; end
+
+  class ConfigurationError < Error; end
+
+  class RateLimitError < Error
+    attr_reader :provider, :status, :retry_after
+
+    def initialize(message = nil, provider: nil, status: nil, retry_after: nil)
+      super(message)
+      @provider = provider
+      @status = status
+      @retry_after = retry_after
+    end
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.2.1"
+  VERSION = "0.4.0"
 end

data/lib/completion_kit.rb

@@ -1,9 +1,9 @@
+require "completion_kit/errors"
 require "completion_kit/version"
 require "completion_kit/engine"
+require "completion_kit/concurrency_check"
 
 module CompletionKit
-  class ConfigurationError < StandardError; end
-
   class Configuration
     attr_accessor :openai_api_key, :anthropic_api_key, :ollama_api_key, :ollama_api_endpoint
     attr_accessor :judge_model, :high_quality_threshold, :medium_quality_threshold

data/lib/tasks/completion_kit_runs.rake

@@ -0,0 +1,13 @@
+namespace :completion_kit do
+  desc "Mark in-flight runs as failed (for use after the queue adapter cutover)"
+  task mark_interrupted_runs_failed: :environment do
+    scope = CompletionKit::Run.where(status: "running")
+    count = scope.count
+    scope.update_all(
+      status: "failed",
+      failure_summary: "Interrupted by deploy",
+      updated_at: Time.current
+    )
+    puts "Marked #{count} runs as failed."
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Damien Bastin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-04-24 00:00:00.000000000 Z
+date: 2026-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -198,18 +198,32 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: solid_queue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: CompletionKit is a prompt testing platform that runs as a Rails engine
   or a standalone app. Run prompts against real datasets, score every output with
   an LLM judge against criteria you define, track prompt versions, and get AI-generated
   improvement suggestions grounded in your actual results. Includes a web UI, REST
-  API, and a built-in MCP server with 36 tools.
+  API, and a built-in MCP server with 34 tools.
 email:
 - damien@homemade.software
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- MIT-LICENSE
+- LICENSE
 - README.md
 - Rakefile
 - app/assets/config/completion_kit_manifest.js
@@ -236,9 +250,10 @@ files:
 - app/controllers/completion_kit/runs_controller.rb
 - app/helpers/completion_kit/application_helper.rb
 - app/jobs/completion_kit/application_job.rb
-- app/jobs/completion_kit/generate_job.rb
-- app/jobs/completion_kit/judge_job.rb
+- app/jobs/completion_kit/generate_row_job.rb
+- app/jobs/completion_kit/judge_review_job.rb
 - app/jobs/completion_kit/model_discovery_job.rb
+- app/jobs/completion_kit/run_completion_check_job.rb
 - app/mailers/completion_kit/application_mailer.rb
 - app/models/completion_kit/application_record.rb
 - app/models/completion_kit/dataset.rb
@@ -272,6 +287,7 @@ files:
 - app/services/completion_kit/open_ai_client.rb
 - app/services/completion_kit/open_router_client.rb
 - app/services/completion_kit/prompt_improvement_service.rb
+- app/services/completion_kit/worker_health.rb
 - app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb
 - app/views/completion_kit/api_reference/_example.html.erb
 - app/views/completion_kit/api_reference/index.html.erb
@@ -326,16 +342,24 @@ files:
 - db/migrate/20260403000003_add_applied_at_to_completion_kit_suggestions.rb
 - db/migrate/20260416000001_remove_evaluation_steps_from_metrics.rb
 - db/migrate/20260417000001_rename_criteria_to_metric_groups.rb
+- db/migrate/20260501000001_add_status_and_error_to_responses.rb
+- db/migrate/20260501000002_index_responses_on_run_id_and_status.rb
+- db/migrate/20260501000003_add_status_and_error_to_reviews.rb
+- db/migrate/20260501000004_index_reviews_on_response_id_and_status.rb
+- db/migrate/20260501000005_collapse_run_status_and_add_failure_summary.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
+- lib/completion_kit/concurrency_check.rb
 - lib/completion_kit/engine.rb
+- lib/completion_kit/errors.rb
 - lib/completion_kit/version.rb
 - lib/generators/completion_kit/install_generator.rb
 - lib/generators/completion_kit/templates/README
 - lib/generators/completion_kit/templates/initializer.rb
+- lib/tasks/completion_kit_runs.rake
 homepage: https://github.com/homemade-software-inc/completion-kit
 licenses:
-- MIT
+- BUSL-1.1
 metadata:
   homepage_uri: https://github.com/homemade-software-inc/completion-kit
   source_code_uri: https://github.com/homemade-software-inc/completion-kit

data/MIT-LICENSE

@@ -1,20 +0,0 @@
-Copyright 
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/app/jobs/completion_kit/generate_job.rb

@@ -1,12 +0,0 @@
-module CompletionKit
-  class GenerateJob < ApplicationJob
-    queue_as :default
-
-    def perform(run_id)
-      run = Run.find_by(id: run_id)
-      return unless run
-
-      run.generate_responses!
-    end
-  end
-end

data/app/jobs/completion_kit/judge_job.rb

@@ -1,12 +0,0 @@
-module CompletionKit
-  class JudgeJob < ApplicationJob
-    queue_as :default
-
-    def perform(run_id)
-      run = Run.find_by(id: run_id)
-      return unless run
-
-      run.judge_responses!
-    end
-  end
-end