completion-kit 0.3.0 → 0.4.0

data/db/migrate/20260501000001_add_status_and_error_to_responses.rb

@@ -0,0 +1,21 @@
+class AddStatusAndErrorToResponses < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_responses, :status, :string, default: "pending", null: false
+    add_column :completion_kit_responses, :error_provider, :string
+    add_column :completion_kit_responses, :error_class, :string
+    add_column :completion_kit_responses, :error_status, :integer
+    add_column :completion_kit_responses, :error_message, :text
+    add_column :completion_kit_responses, :attempts, :integer, default: 0, null: false
+    add_column :completion_kit_responses, :row_index, :integer
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+          UPDATE completion_kit_responses
+          SET status = 'succeeded'
+          WHERE response_text IS NOT NULL AND length(response_text) > 0
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20260501000002_index_responses_on_run_id_and_status.rb

@@ -0,0 +1,9 @@
+class IndexResponsesOnRunIdAndStatus < ActiveRecord::Migration[7.1]
+  disable_ddl_transaction!
+
+  def change
+    options = { if_not_exists: true }
+    options[:algorithm] = :concurrently unless connection.adapter_name == "SQLite"
+    add_index :completion_kit_responses, [:run_id, :status], **options
+  end
+end

data/db/migrate/20260501000003_add_status_and_error_to_reviews.rb

@@ -0,0 +1,25 @@
+class AddStatusAndErrorToReviews < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_reviews, :error_provider, :string
+    add_column :completion_kit_reviews, :error_class, :string
+    add_column :completion_kit_reviews, :error_status, :integer
+    add_column :completion_kit_reviews, :error_message, :text
+    add_column :completion_kit_reviews, :attempts, :integer, default: 0, null: false
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+          UPDATE completion_kit_reviews
+          SET status = 'succeeded'
+          WHERE ai_score IS NOT NULL
+        SQL
+
+        execute <<~SQL
+          UPDATE completion_kit_reviews
+          SET status = 'succeeded'
+          WHERE status = 'evaluated'
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20260501000004_index_reviews_on_response_id_and_status.rb

@@ -0,0 +1,9 @@
+class IndexReviewsOnResponseIdAndStatus < ActiveRecord::Migration[7.1]
+  disable_ddl_transaction!
+
+  def change
+    options = { if_not_exists: true }
+    options[:algorithm] = :concurrently unless connection.adapter_name == "SQLite"
+    add_index :completion_kit_reviews, [:response_id, :status], **options
+  end
+end

data/db/migrate/20260501000005_collapse_run_status_and_add_failure_summary.rb

@@ -0,0 +1,15 @@
+class CollapseRunStatusAndAddFailureSummary < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_runs, :failure_summary, :text
+
+    reversible do |dir|
+      dir.up do
+        execute <<~SQL
+          UPDATE completion_kit_runs
+          SET status = 'running'
+          WHERE status IN ('generating', 'judging')
+        SQL
+      end
+    end
+  end
+end

data/lib/completion_kit/concurrency_check.rb

@@ -0,0 +1,16 @@
+module CompletionKit
+  module ConcurrencyCheck
+    def self.warn_if_misconfigured(logger)
+      threads = ENV.fetch("SOLID_QUEUE_THREADS", 10).to_i
+      llm_cap = ENV.fetch("COMPLETION_KIT_LLM_CONCURRENCY", 10).to_i
+
+      if threads < llm_cap
+        logger.warn(
+          "[CompletionKit] SOLID_QUEUE_THREADS=#{threads} is less than " \
+          "COMPLETION_KIT_LLM_CONCURRENCY=#{llm_cap}; threads will be the " \
+          "actual bottleneck and the per-provider cap will never be reached."
+        )
+      end
+    end
+  end
+end

data/lib/completion_kit/errors.rb

@@ -0,0 +1,16 @@
+module CompletionKit
+  class Error < StandardError; end
+
+  class ConfigurationError < Error; end
+
+  class RateLimitError < Error
+    attr_reader :provider, :status, :retry_after
+
+    def initialize(message = nil, provider: nil, status: nil, retry_after: nil)
+      super(message)
+      @provider = provider
+      @status = status
+      @retry_after = retry_after
+    end
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/completion_kit.rb

@@ -1,9 +1,9 @@
+require "completion_kit/errors"
 require "completion_kit/version"
 require "completion_kit/engine"
+require "completion_kit/concurrency_check"
 
 module CompletionKit
-  class ConfigurationError < StandardError; end
-
   class Configuration
     attr_accessor :openai_api_key, :anthropic_api_key, :ollama_api_key, :ollama_api_endpoint
     attr_accessor :judge_model, :high_quality_threshold, :medium_quality_threshold

data/lib/tasks/completion_kit_runs.rake

@@ -0,0 +1,13 @@
+namespace :completion_kit do
+  desc "Mark in-flight runs as failed (for use after the queue adapter cutover)"
+  task mark_interrupted_runs_failed: :environment do
+    scope = CompletionKit::Run.where(status: "running")
+    count = scope.count
+    scope.update_all(
+      status: "failed",
+      failure_summary: "Interrupted by deploy",
+      updated_at: Time.current
+    )
+    puts "Marked #{count} runs as failed."
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Damien Bastin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-04-26 00:00:00.000000000 Z
+date: 2026-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -198,11 +198,25 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: solid_queue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: CompletionKit is a prompt testing platform that runs as a Rails engine
   or a standalone app. Run prompts against real datasets, score every output with
   an LLM judge against criteria you define, track prompt versions, and get AI-generated
   improvement suggestions grounded in your actual results. Includes a web UI, REST
-  API, and a built-in MCP server with 36 tools.
+  API, and a built-in MCP server with 34 tools.
 email:
 - damien@homemade.software
 executables: []
@@ -236,9 +250,10 @@ files:
 - app/controllers/completion_kit/runs_controller.rb
 - app/helpers/completion_kit/application_helper.rb
 - app/jobs/completion_kit/application_job.rb
-- app/jobs/completion_kit/generate_job.rb
-- app/jobs/completion_kit/judge_job.rb
+- app/jobs/completion_kit/generate_row_job.rb
+- app/jobs/completion_kit/judge_review_job.rb
 - app/jobs/completion_kit/model_discovery_job.rb
+- app/jobs/completion_kit/run_completion_check_job.rb
 - app/mailers/completion_kit/application_mailer.rb
 - app/models/completion_kit/application_record.rb
 - app/models/completion_kit/dataset.rb
@@ -272,6 +287,7 @@ files:
 - app/services/completion_kit/open_ai_client.rb
 - app/services/completion_kit/open_router_client.rb
 - app/services/completion_kit/prompt_improvement_service.rb
+- app/services/completion_kit/worker_health.rb
 - app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb
 - app/views/completion_kit/api_reference/_example.html.erb
 - app/views/completion_kit/api_reference/index.html.erb
@@ -326,13 +342,21 @@ files:
 - db/migrate/20260403000003_add_applied_at_to_completion_kit_suggestions.rb
 - db/migrate/20260416000001_remove_evaluation_steps_from_metrics.rb
 - db/migrate/20260417000001_rename_criteria_to_metric_groups.rb
+- db/migrate/20260501000001_add_status_and_error_to_responses.rb
+- db/migrate/20260501000002_index_responses_on_run_id_and_status.rb
+- db/migrate/20260501000003_add_status_and_error_to_reviews.rb
+- db/migrate/20260501000004_index_reviews_on_response_id_and_status.rb
+- db/migrate/20260501000005_collapse_run_status_and_add_failure_summary.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
+- lib/completion_kit/concurrency_check.rb
 - lib/completion_kit/engine.rb
+- lib/completion_kit/errors.rb
 - lib/completion_kit/version.rb
 - lib/generators/completion_kit/install_generator.rb
 - lib/generators/completion_kit/templates/README
 - lib/generators/completion_kit/templates/initializer.rb
+- lib/tasks/completion_kit_runs.rake
 homepage: https://github.com/homemade-software-inc/completion-kit
 licenses:
 - BUSL-1.1

data/app/jobs/completion_kit/generate_job.rb

@@ -1,12 +0,0 @@
-module CompletionKit
-  class GenerateJob < ApplicationJob
-    queue_as :default
-
-    def perform(run_id)
-      run = Run.find_by(id: run_id)
-      return unless run
-
-      run.generate_responses!
-    end
-  end
-end

data/app/jobs/completion_kit/judge_job.rb

@@ -1,12 +0,0 @@
-module CompletionKit
-  class JudgeJob < ApplicationJob
-    queue_as :default
-
-    def perform(run_id)
-      run = Run.find_by(id: run_id)
-      return unless run
-
-      run.judge_responses!
-    end
-  end
-end