completion-kit 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 158791345bf5503ca9dbca6d6e8374c17b7802cbee673cf783d6c2f33ac144db
-  data.tar.gz: 6912a4de3c685d62adeaa5670a2ad6db6c847f941d225356db788ec1275d7652
+  metadata.gz: 2ae15d465cd8a4a617df6ff8f8cd401495b18b07813e944149e4eba940d007e9
+  data.tar.gz: bd35dcea6cc46ca13d388b448b5d5ee3028519786e40154931fa5724d09d894b
 SHA512:
-  metadata.gz: '09bf06065c3c7219456f2f5f6588d0121fcbc4c7d68baf6fff9cb18f6196dae7170d0068ad0d3d7f46cb1e4e339ec775eecd355321ee8866ac1252af5234ca4d'
-  data.tar.gz: 12e422f715cb21eed7d759ad8b2924a3f7daeff88d816c278f8c0af778e6dd9583203a25be7214d9959646a6a435420658d481f0b983b66097e765db5244eb2f
+  metadata.gz: f51ddf1a4d6a1eae534078d8e12b3f8a4e03576ee665cc575025038d0e63db72f44ab32627aa660b8863d1b49639286668818581c36efbbd2a21e99bba8f5e1b
+  data.tar.gz: 171fc9597f7b44a80c3f0d80a0ff5abd258fb766a34089632c4040940dd1bbcc9dcf0544bed77309c398d8e2187025e25271fa9a9fb2709c23f04d6cc2836cfa

data/README.md

@@ -35,9 +35,20 @@ cd completion-kit/standalone
 bundle install
 bin/rails completion_kit:install:migrations
 bin/rails db:migrate
+```
+
+Then run **both** processes — a web server and a Solid Queue worker. In two terminals:
+
+```bash
 bin/rails server
 ```
 
+```bash
+bin/jobs
+```
+
+Or with [foreman](https://github.com/ddollar/foreman) in one terminal: `foreman start -f Procfile.dev`.
+
 Visit `http://localhost:3000`. Add a provider credential (Settings), create a prompt, upload a CSV dataset, and run it.
 
 ### Or mount as an engine in your existing Rails app
@@ -51,7 +62,7 @@ bin/rails generate completion_kit:install
 bin/rails db:migrate
 ```
 
-The engine mounts at `/completion_kit` in your app.
+The engine mounts at `/completion_kit` in your app. CompletionKit's generate and judge flows enqueue Active Job jobs (`CompletionKit::GenerateRowJob`, `CompletionKit::JudgeReviewJob`, `CompletionKit::RunCompletionCheckJob`), so your host app needs an Active Job adapter that actually processes them — Solid Queue, Sidekiq, GoodJob, etc. The `:async` adapter is **not** suitable for production: it runs jobs in the web Puma's thread pool with no durability and no retry, and a long LLM call will block request handling.
 
 ## Providers
 

data/Rakefile

@@ -1,7 +1,4 @@
 require "bundler/setup"
-
-load "rails/tasks/statistics.rake"
-
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 

data/app/assets/stylesheets/completion_kit/application.css

@@ -274,6 +274,39 @@ form.button_to {
   color: var(--ck-accent);
 }
 
+.ck-disclosure-toggle {
+  appearance: none;
+  background: transparent;
+  border: 0;
+  padding: 0;
+  margin: 0.5rem 0 0;
+  font-family: var(--ck-mono);
+  font-size: 0.75rem;
+  font-weight: 500;
+  letter-spacing: 0.12em;
+  text-transform: uppercase;
+  color: var(--ck-muted);
+  cursor: pointer;
+  transition: color 0.15s;
+}
+
+.ck-disclosure-toggle:hover,
+.ck-disclosure-toggle:focus-visible {
+  color: var(--ck-accent);
+  outline: none;
+}
+
+.ck-disclosure-toggle::after {
+  content: " ↓";
+  display: inline-block;
+  margin-left: 0.25rem;
+  transition: transform 0.15s;
+}
+
+.ck-disclosure-toggle[aria-expanded="true"]::after {
+  transform: rotate(180deg);
+}
+
 .ck-list {
   display: grid;
   gap: 0.5rem;
@@ -385,6 +418,18 @@ tr:hover .ck-chip--publish {
   color: var(--ck-accent);
 }
 
+.ck-chip--warning {
+  background: var(--ck-warning-soft);
+  border-color: rgba(224, 164, 88, 0.3);
+  color: var(--ck-warning);
+}
+
+.ck-chip--danger {
+  background: var(--ck-danger-soft);
+  border-color: rgba(248, 113, 113, 0.3);
+  color: var(--ck-danger);
+}
+
 .ck-badge--high {
   background: var(--ck-success-soft);
   border: 1px solid rgba(34, 197, 94, 0.25);
@@ -679,6 +724,27 @@ tr:hover .ck-chip--publish {
   color: var(--ck-text);
 }
 
+.ck-progress-block {
+  padding: 0.5rem 1rem 0.75rem;
+  border-top: 1px solid var(--ck-line);
+  font-size: 0.72rem;
+  font-family: var(--ck-mono);
+  color: var(--ck-muted);
+  display: flex;
+  flex-direction: column;
+  gap: 0.25rem;
+}
+
+.ck-progress-line {
+  display: flex;
+  gap: 0.4rem;
+  align-items: baseline;
+}
+
+.ck-progress-failed {
+  color: var(--ck-danger);
+}
+
 .ck-model-list-details summary {
   list-style: none;
 }
@@ -802,6 +868,12 @@ tr:hover .ck-chip--publish {
   color: var(--ck-muted);
 }
 
+.ck-field--info #refresh-status,
+.ck-field--warn #refresh-status,
+.ck-field--error #refresh-status {
+  color: var(--ck-muted);
+}
+
 .ck-field--info .ck-input {
   border-color: var(--ck-accent);
 }
@@ -1815,6 +1887,21 @@ select.ck-input {
   flex-shrink: 0;
 }
 
+.ck-response-row--pending .ck-response-row__text,
+.ck-response-row--retrying .ck-response-row__text {
+  color: var(--ck-dim);
+}
+
+.ck-response-row--failed .ck-response-row__text {
+  color: var(--ck-danger);
+  opacity: 0.8;
+}
+
+.ck-response-row__error {
+  font-family: var(--ck-mono);
+  font-size: 0.82rem;
+}
+
 .ck-score {
   font-size: 0.85rem;
   color: var(--ck-muted);

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -2,7 +2,7 @@ module CompletionKit
   module Api
     module V1
       class RunsController < BaseController
-        before_action :set_run, only: [:show, :update, :destroy, :generate, :judge]
+        before_action :set_run, only: [:show, :update, :destroy, :generate, :retry_failures]
 
         def index
           render json: Run.order(created_at: :desc)
@@ -37,12 +37,33 @@ module CompletionKit
         end
 
         def generate
-          GenerateJob.perform_later(@run.id)
-          render json: @run.reload, status: :accepted
+          if @run.start!
+            render json: @run.reload, status: :accepted
+          else
+            render json: { errors: [@run.failure_summary || @run.errors.full_messages.to_sentence] }, status: :unprocessable_entity
+          end
         end
 
-        def judge
-          JudgeJob.perform_later(@run.id)
+        def retry_failures
+          scope = @run.responses.where(status: "failed")
+          scope = scope.where(id: params[:only]) if params[:only].present?
+
+          ActiveRecord::Base.transaction do
+            failed_response_ids = scope.pluck(:id)
+            CompletionKit::Review.where(response_id: failed_response_ids, status: "failed").update_all(
+              status: "pending", attempts: 0,
+              error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+              ai_score: nil, ai_feedback: nil
+            )
+            scope.update_all(
+              status: "pending", attempts: 0,
+              error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+              response_text: nil
+            )
+            @run.update!(status: "running")
+            failed_response_ids.each { |rid| CompletionKit::GenerateRowJob.perform_later(@run.id, rid) }
+          end
+
           render json: @run.reload, status: :accepted
         end
 

data/app/controllers/completion_kit/runs_controller.rb

@@ -1,6 +1,6 @@
 module CompletionKit
   class RunsController < ApplicationController
-    before_action :set_run, only: [:show, :edit, :update, :destroy, :generate, :judge, :suggest, :suggestion, :apply_suggestion]
+    before_action :set_run, only: [:show, :edit, :update, :destroy, :generate, :suggest, :suggestion, :apply_suggestion, :retry_failures]
     before_action :load_form_collections, only: [:new, :edit, :create, :update]
 
     def index
@@ -63,17 +63,11 @@ module CompletionKit
     end
 
     def generate
-      @run.update!(status: "generating", progress_current: 0, progress_total: 0, error_message: nil)
-      GenerateJob.perform_later(@run.id)
-      redirect_to run_path(@run)
-    end
-
-    def judge
-      if params[:run]
-        @run.update(judge_model: params[:run][:judge_model])
+      if @run.start!
+        redirect_to run_path(@run)
+      else
+        redirect_to run_path(@run), alert: @run.failure_summary || @run.errors.full_messages.to_sentence
       end
-      JudgeJob.perform_later(@run.id)
-      redirect_to run_path(@run)
     end
 
     def suggest
@@ -93,6 +87,32 @@ module CompletionKit
       return redirect_to run_path(@run), alert: "No suggestion available. Generate one first." unless @suggestion
     end
 
+    def retry_failures
+      scope = @run.responses.where(status: "failed")
+      scope = scope.where(id: params[:only]) if params[:only].present?
+
+      ActiveRecord::Base.transaction do
+        failed_response_ids = scope.pluck(:id)
+        Review.where(response_id: failed_response_ids, status: "failed").update_all(
+          status: "pending",
+          attempts: 0,
+          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+          ai_score: nil, ai_feedback: nil
+        )
+        scope.update_all(
+          status: "pending",
+          attempts: 0,
+          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+          response_text: nil
+        )
+        @run.update!(status: "running")
+        failed_response_ids.each { |rid| GenerateRowJob.perform_later(@run.id, rid) }
+      end
+
+      @run.send(:broadcast_ui)
+      redirect_to run_path(@run)
+    end
+
     def apply_suggestion
       suggestion = @run.suggestions.order(created_at: :desc).first
       return redirect_to run_path(@run), alert: "No suggestion to apply." unless suggestion

data/app/helpers/completion_kit/application_helper.rb

@@ -35,8 +35,6 @@ module CompletionKit
         "ck-badge ck-badge--pending"
       when "running"
         "ck-badge ck-badge--running"
-      when "generating", "judging"
-        "ck-badge ck-badge--running"
       when "completed"
         "ck-badge ck-badge--high"
       when "failed"
@@ -48,7 +46,7 @@ module CompletionKit
 
     def ck_run_dot(run)
       case run.status
-      when "generating", "judging" then "ck-dot ck-dot--running"
+      when "running" then "ck-dot ck-dot--running"
       when "failed" then "ck-dot ck-dot--failed"
       when "completed" then "ck-dot ck-dot--completed"
       else "ck-dot ck-dot--pending"
@@ -58,17 +56,11 @@ module CompletionKit
     def ck_run_status_label(run)
       case run.status
       when "pending" then "Ready to run"
-      when "generating"
-        if run.progress_total.to_i > 0
-          "Generating responses (#{run.progress_current}/#{run.progress_total})"
-        else
-          "Generating responses…"
-        end
-      when "judging"
+      when "running"
         if run.progress_total.to_i > 0
-          "Judging (#{run.progress_current}/#{run.progress_total} evaluations)"
+          "Running (#{run.progress_current}/#{run.progress_total})"
         else
-          "Judging…"
+          "Running…"
         end
       when "completed" then "Completed"
       when "failed" then "Failed"

data/app/jobs/completion_kit/generate_row_job.rb

@@ -0,0 +1,102 @@
+require "faraday"
+
+module CompletionKit
+  class GenerateRowJob < ApplicationJob
+    queue_as :llm
+
+    limits_concurrency to: ENV.fetch("COMPLETION_KIT_PER_RUN_CONCURRENCY", 5).to_i,
+                       key: ->(run_id, _) { "run:#{run_id}" },
+                       duration: 10.minutes
+
+    def self.rate_limit_wait(executions)
+      30 * executions
+    end
+
+    retry_on Faraday::TimeoutError,
+             Faraday::ConnectionFailed,
+             wait: :polynomially_longer, attempts: 5
+
+    retry_on CompletionKit::RateLimitError,
+             wait: method(:rate_limit_wait), attempts: 5
+
+    discard_on ActiveJob::DeserializationError
+    discard_on CompletionKit::ConfigurationError
+
+    rescue_from(StandardError) do |error|
+      record_terminal_failure!(error)
+      enqueue_completion_check
+    end
+
+    before_perform do |job|
+      response = Response.find_by(id: job.arguments.last)
+      next unless response
+      response.update_columns(status: "retrying", attempts: response.attempts + 1)
+      response.run.send(:broadcast_response_update, response) if response.run
+    end
+
+    def perform(run_id, response_id)
+      @run_id = run_id
+      @response_id = response_id
+
+      response = Response.find(response_id)
+      run = response.run
+      prompt = run.prompt
+
+      row = parsed_input(response)
+      rendered = CsvProcessor.apply_variables(prompt, row)
+      client = LlmClient.for_model(prompt.llm_model, ApiConfig.for_model(prompt.llm_model))
+
+      raise ConfigurationError, client.configuration_errors.join(", ") unless client.configured?
+
+      text = client.generate_completion(rendered, model: prompt.llm_model, temperature: run.temperature)
+
+      response.update!(
+        status: "succeeded",
+        response_text: text,
+        error_provider: nil, error_class: nil, error_status: nil, error_message: nil
+      )
+      run.send(:broadcast_response_update, response)
+
+      if run.judge_configured?
+        run.metrics.each do |metric|
+          JudgeReviewJob.perform_later(response.id, metric.id)
+        end
+      end
+
+      enqueue_completion_check
+    end
+
+    private
+
+    def parsed_input(response)
+      return {} if response.input_data.blank?
+      JSON.parse(response.input_data)
+    rescue JSON::ParserError
+      {}
+    end
+
+    def record_terminal_failure!(error)
+      response_id = @response_id || arguments.last
+      response = Response.find_by(id: response_id)
+      return unless response
+
+      response.update_columns(
+        status: "failed",
+        error_provider: provider_for(response),
+        error_class: error.class.name,
+        error_status: error.respond_to?(:status) ? error.status : nil,
+        error_message: error.message.to_s.truncate(2000)
+      )
+      response.run&.send(:broadcast_response_update, response)
+    end
+
+    def provider_for(response)
+      response.run&.prompt&.llm_model_provider
+    end
+
+    def enqueue_completion_check
+      run_id = @run_id || arguments.first
+      RunCompletionCheckJob.perform_later(run_id)
+    end
+  end
+end

data/app/jobs/completion_kit/judge_review_job.rb

@@ -0,0 +1,110 @@
+require "faraday"
+
+module CompletionKit
+  class JudgeReviewJob < ApplicationJob
+    queue_as :llm
+
+    limits_concurrency to: ENV.fetch("COMPLETION_KIT_PER_RUN_CONCURRENCY", 5).to_i,
+                       key: ->(response_id, _) { "run:#{Response.find_by(id: response_id)&.run_id}" },
+                       duration: 10.minutes
+
+    def self.rate_limit_wait(executions)
+      30 * executions
+    end
+
+    retry_on Faraday::TimeoutError,
+             Faraday::ConnectionFailed,
+             wait: :polynomially_longer, attempts: 5
+
+    retry_on CompletionKit::RateLimitError,
+             wait: method(:rate_limit_wait), attempts: 5
+
+    discard_on ActiveJob::DeserializationError
+    discard_on CompletionKit::ConfigurationError
+
+    rescue_from(StandardError) do |error|
+      record_terminal_failure!(error)
+      enqueue_completion_check
+    end
+
+    before_perform do |job|
+      response_id, metric_id = job.arguments
+      response = Response.find_by(id: response_id)
+      next unless response
+      review = response.reviews.find_or_initialize_by(metric_id: metric_id)
+      review.metric_name ||= Metric.find_by(id: metric_id)&.name || "(deleted metric)"
+      review.attempts = (review.attempts || 0) + 1
+      review.status = "retrying"
+      review.save!(validate: false)
+      response.run.send(:broadcast_response_update, response) if response.run
+    end
+
+    def perform(response_id, metric_id)
+      @response_id = response_id
+      @metric_id = metric_id
+
+      response = Response.find(response_id)
+      metric = Metric.find(metric_id)
+      run = response.run
+
+      config = ApiConfig.for_model(run.judge_model).merge(judge_model: run.judge_model)
+      judge = JudgeService.new(config)
+
+      evaluation = judge.evaluate(
+        response.response_text,
+        response.expected_output,
+        run.prompt.template,
+        criteria: metric.instruction.to_s,
+        rubric_text: metric.display_rubric_text,
+        input_data: response.input_data
+      )
+
+      review = response.reviews.find_or_initialize_by(metric_id: metric.id)
+      review.assign_attributes(
+        metric_name: metric.name,
+        instruction: metric.instruction.to_s,
+        status: "succeeded",
+        ai_score: evaluation[:score],
+        ai_feedback: evaluation[:feedback],
+        error_provider: nil, error_class: nil, error_status: nil, error_message: nil
+      )
+      review.save!
+
+      run.send(:broadcast_response_update, response)
+      enqueue_completion_check
+    end
+
+    private
+
+    def record_terminal_failure!(error)
+      response_id = @response_id || arguments.first
+      metric_id = @metric_id || arguments.last
+      response = Response.find_by(id: response_id)
+      return unless response
+
+      review = response.reviews.find_or_initialize_by(metric_id: metric_id)
+      review.assign_attributes(
+        metric_name: review.metric_name || Metric.find_by(id: metric_id)&.name || "(deleted metric)",
+        status: "failed",
+        error_provider: provider_for(response),
+        error_class: error.class.name,
+        error_status: error.respond_to?(:status) ? error.status : nil,
+        error_message: error.message.to_s.truncate(2000)
+      )
+      review.save!(validate: false)
+      response.run&.send(:broadcast_response_update, response)
+    end
+
+    def provider_for(response)
+      run = response.run
+      return nil unless run&.judge_model
+      ApiConfig.provider_for_model(run.judge_model)
+    end
+
+    def enqueue_completion_check
+      response_id = @response_id || arguments.first
+      response = Response.find_by(id: response_id)
+      RunCompletionCheckJob.perform_later(response.run_id) if response
+    end
+  end
+end

data/app/jobs/completion_kit/model_discovery_job.rb

@@ -1,7 +1,29 @@
+require "faraday"
+
 module CompletionKit
   class ModelDiscoveryJob < ApplicationJob
     queue_as :default
 
+    def self.rate_limit_wait(executions)
+      30 * executions
+    end
+
+    retry_on Faraday::TimeoutError,
+             Faraday::ConnectionFailed,
+             wait: :polynomially_longer, attempts: 5
+
+    retry_on CompletionKit::RateLimitError,
+             wait: method(:rate_limit_wait), attempts: 5
+
+    discard_on ActiveJob::DeserializationError
+
+    rescue_from(StandardError) do |_error|
+      credential = ProviderCredential.find(arguments.first)
+      credential.update_columns(discovery_status: "failed")
+      credential.reload
+      credential.broadcast_discovery_progress
+    end
+
     def perform(provider_credential_id)
       credential = ProviderCredential.find_by(id: provider_credential_id)
       return unless credential
@@ -20,10 +42,6 @@ module CompletionKit
       credential.update_columns(discovery_status: "completed", updated_at: Time.current)
       credential.reload
       credential.broadcast_discovery_complete
-    rescue StandardError
-      credential.update_columns(discovery_status: "failed")
-      credential.reload
-      credential.broadcast_discovery_progress
     end
   end
 end

data/app/jobs/completion_kit/run_completion_check_job.rb

@@ -0,0 +1,18 @@
+module CompletionKit
+  class RunCompletionCheckJob < ApplicationJob
+    queue_as :default
+
+    limits_concurrency to: 1,
+                       key: ->(run_id) { "run:#{run_id}:completion" },
+                       duration: 5.minutes
+
+    def perform(run_id)
+      run = Run.find_by(id: run_id)
+      return unless run
+      return unless run.status == "running"
+      return unless run.outstanding_work_zero?
+
+      run.mark_completed!
+    end
+  end
+end

data/app/models/completion_kit/prompt.rb

@@ -42,6 +42,10 @@ module CompletionKit
       "#{name} — #{version_label}"
     end
 
+    def llm_model_provider
+      ApiConfig.provider_for_model(llm_model)
+    end
+
     def family_versions
       self.class.where(family_key: family_key).order(version_number: :desc, created_at: :desc)
     end

data/app/models/completion_kit/response.rb

@@ -1,18 +1,34 @@
 module CompletionKit
   class Response < ApplicationRecord
+    STATUSES = %w[pending retrying succeeded failed].freeze
+    TERMINAL_STATUSES = %w[succeeded failed].freeze
+
     belongs_to :run
     has_many :reviews, dependent: :destroy
 
     delegate :prompt, to: :run
 
-    validates :response_text, presence: true
+    validates :response_text, presence: true, if: :succeeded?
+    validates :status, inclusion: { in: STATUSES }
+
+    before_validation :set_default_status, on: :create
+
+    def terminal?
+      TERMINAL_STATUSES.include?(status)
+    end
+
+    def succeeded?
+      status == "succeeded"
+    end
 
     def as_json(options = {})
       {
         id: id, run_id: run_id, input_data: input_data,
         response_text: response_text, expected_output: expected_output,
         created_at: created_at, score: score, reviewed: reviewed?,
-        reviews: reviews.map(&:as_json)
+        reviews: reviews.map(&:as_json),
+        status: status, attempts: attempts, row_index: row_index,
+        error: error_payload
       }
     end
 
@@ -26,5 +42,16 @@ module CompletionKit
     def reviewed?
       reviews.any? { |r| r.ai_score.present? }
     end
+
+    def error_payload
+      return nil if error_class.blank?
+      { provider: error_provider, class: error_class, status: error_status, message: error_message }
+    end
+
+    private
+
+    def set_default_status
+      self.status ||= "pending"
+    end
   end
 end

data/app/models/completion_kit/review.rb

@@ -1,6 +1,7 @@
 module CompletionKit
   class Review < ApplicationRecord
-    STATUSES = %w[pending evaluated failed].freeze
+    STATUSES = %w[pending retrying succeeded failed].freeze
+    TERMINAL_STATUSES = %w[succeeded failed].freeze
 
     belongs_to :response
     belongs_to :metric, optional: true
@@ -11,11 +12,25 @@ module CompletionKit
 
     before_validation :set_default_status
 
+    def terminal?
+      TERMINAL_STATUSES.include?(status)
+    end
+
+    def succeeded?
+      status == "succeeded"
+    end
+
+    def error_payload
+      return nil if error_class.blank?
+      { provider: error_provider, class: error_class, status: error_status, message: error_message }
+    end
+
     def as_json(options = {})
       {
         id: id, response_id: response_id, metric_id: metric_id,
         metric_name: metric_name, ai_score: ai_score,
-        ai_feedback: ai_feedback, status: status
+        ai_feedback: ai_feedback, status: status, attempts: attempts,
+        error: error_payload
       }
     end