completion-kit 0.4.2 → 0.4.8

data/app/controllers/completion_kit/runs_controller.rb

@@ -1,6 +1,6 @@
 module CompletionKit
   class RunsController < ApplicationController
-    before_action :set_run, only: [:show, :edit, :update, :destroy, :generate, :suggest, :suggestion, :apply_suggestion, :retry_failures]
+    before_action :set_run, only: [:show, :edit, :update, :destroy, :generate, :suggest, :retry_failures, :rerun, :refresh_status]
     before_action :load_form_collections, only: [:new, :edit, :create, :update]
 
     def index
@@ -72,21 +72,44 @@ module CompletionKit
       end
     end
 
+    def rerun
+      new_run = Run.create!(
+        prompt_id: @run.prompt_id,
+        dataset_id: @run.dataset_id,
+        judge_model: @run.judge_model,
+        temperature: @run.temperature,
+        status: "pending"
+      )
+      new_run.replace_metrics!(@run.metric_ids)
+      if new_run.start!
+        redirect_to run_path(new_run), notice: "Re-running with the same configuration."
+      else
+        redirect_to run_path(new_run), alert: new_run.failure_summary || "Could not start the new run."
+      end
+    end
+
+    def refresh_status
+      respond_to do |format|
+        format.turbo_stream do
+          render turbo_stream: turbo_stream.replace(
+            "run_status_header",
+            partial: "completion_kit/runs/status_header",
+            locals: { run: @run }
+          )
+        end
+      end
+    end
+
     def suggest
       service = PromptImprovementService.new(@run)
       result = service.suggest
-      @run.suggestions.create!(
+      suggestion = @run.suggestions.create!(
         prompt: @run.prompt,
         reasoning: result["reasoning"],
         suggested_template: result["suggested_template"],
         original_template: result["original_template"]
       )
-      redirect_to suggestion_run_path(@run)
-    end
-
-    def suggestion
-      @suggestion = @run.suggestions.order(created_at: :desc).first
-      return redirect_to run_path(@run), alert: "No suggestion available. Generate one first." unless @suggestion
+      redirect_to suggestion_path(suggestion, from: "run")
     end
 
     def retry_failures
@@ -115,16 +138,6 @@ module CompletionKit
       redirect_to run_path(@run)
     end
 
-    def apply_suggestion
-      suggestion = @run.suggestions.order(created_at: :desc).first
-      return redirect_to run_path(@run), alert: "No suggestion to apply." unless suggestion
-
-      new_prompt = @run.prompt.clone_as_new_version(template: suggestion.suggested_template)
-      new_prompt.publish!
-      suggestion.update!(applied_at: Time.current)
-      redirect_to prompt_path(new_prompt), notice: "Suggestion applied."
-    end
-
     private
 
     def set_run

data/app/controllers/completion_kit/suggestions_controller.rb

@@ -0,0 +1,24 @@
+module CompletionKit
+  class SuggestionsController < ApplicationController
+    before_action :set_suggestion
+
+    def show
+      @run = @suggestion.run
+      @from = params[:from] == "run" ? "run" : "prompt"
+    end
+
+    def apply
+      run = @suggestion.run
+      new_prompt = run.prompt.clone_as_new_version(template: @suggestion.suggested_template)
+      new_prompt.publish!
+      @suggestion.update!(applied_at: Time.current)
+      redirect_to prompt_path(new_prompt), notice: "Suggestion applied."
+    end
+
+    private
+
+    def set_suggestion
+      @suggestion = Suggestion.find(params[:id])
+    end
+  end
+end

data/app/jobs/completion_kit/generate_row_job.rb

@@ -49,6 +49,11 @@ module CompletionKit
       raise ConfigurationError, client.configuration_errors.join(", ") unless client.configured?
 
       text = client.generate_completion(rendered, model: prompt.llm_model, temperature: run.temperature)
+      raise StandardError, text.to_s.sub(/\AError:\s*/, "") if text.to_s.start_with?("Error:")
+
+      if client.respond_to?(:temperature_dropped?) && client.temperature_dropped? && !run.temperature_ignored?
+        run.update_columns(temperature_ignored: true)
+      end
 
       response.update!(
         status: "succeeded",
@@ -56,6 +61,7 @@ module CompletionKit
         error_provider: nil, error_class: nil, error_status: nil, error_message: nil
       )
       run.send(:broadcast_response_update, response)
+      run.send(:broadcast_progress)
 
       if run.judge_configured?
         run.metrics.each do |metric|
@@ -88,6 +94,7 @@ module CompletionKit
         error_message: error.message.to_s.truncate(2000)
       )
       response.run&.send(:broadcast_response_update, response)
+      response.run&.send(:broadcast_progress)
     end
 
     def provider_for(response)

data/app/jobs/completion_kit/judge_review_job.rb

@@ -71,6 +71,7 @@ module CompletionKit
       review.save!
 
       run.send(:broadcast_response_update, response)
+      run.send(:broadcast_progress)
       enqueue_completion_check
     end
 
@@ -93,6 +94,7 @@ module CompletionKit
       )
       review.save!(validate: false)
       response.run&.send(:broadcast_response_update, response)
+      response.run&.send(:broadcast_progress)
     end
 
     def provider_for(response)

data/app/jobs/completion_kit/model_discovery_job.rb

@@ -17,9 +17,9 @@ module CompletionKit
 
     discard_on ActiveJob::DeserializationError
 
-    rescue_from(StandardError) do |_error|
+    rescue_from(StandardError) do |error|
       credential = ProviderCredential.find(arguments.first)
-      credential.update_columns(discovery_status: "failed")
+      credential.update_columns(discovery_status: "failed", discovery_error: error.message.to_s.truncate(500))
       credential.reload
       credential.broadcast_discovery_progress
     end
@@ -28,7 +28,12 @@ module CompletionKit
       credential = ProviderCredential.find_by(id: provider_credential_id)
       return unless credential
 
-      credential.update_columns(discovery_status: "discovering", discovery_current: 0, discovery_total: 0)
+      credential.update_columns(
+        discovery_status: "discovering",
+        discovery_current: 0,
+        discovery_total: 0,
+        discovery_error: nil
+      )
       credential.reload
       credential.broadcast_discovery_progress
 
@@ -39,7 +44,7 @@ module CompletionKit
         credential.broadcast_discovery_progress
       end
 
-      credential.update_columns(discovery_status: "completed", updated_at: Time.current)
+      credential.update_columns(discovery_status: "completed", discovery_error: nil, updated_at: Time.current)
       credential.reload
       credential.broadcast_discovery_complete
     end

data/app/models/completion_kit/dataset.rb

@@ -20,5 +20,14 @@ module CompletionKit
     rescue ::CSV::MalformedCSVError
       0
     end
+
+    def headers
+      return [] if csv_data.blank?
+
+      require "csv"
+      ::CSV.parse(csv_data.lines.first.to_s).first.to_a.map(&:to_s).map(&:strip)
+    rescue ::CSV::MalformedCSVError
+      []
+    end
   end
 end

data/app/models/completion_kit/provider_credential.rb

@@ -56,7 +56,7 @@ module CompletionKit
     def judge_count
       model_ids = Model.where(provider: provider).pluck(:model_id)
       return 0 if model_ids.empty?
-      Run.where(judge_model: model_ids).count
+      Run.where(judge_model: model_ids).distinct.count(:judge_model)
     end
 
     def last_used_at

data/app/models/completion_kit/response.rb

@@ -43,6 +43,13 @@ module CompletionKit
       reviews.any? { |r| r.ai_score.present? }
     end
 
+    def fully_reviewed?
+      metric_ids = run.metric_ids
+      return true if metric_ids.empty?
+      reviewed_metric_ids = reviews.where(status: Review::TERMINAL_STATUSES).pluck(:metric_id).uniq
+      (metric_ids - reviewed_metric_ids).empty?
+    end
+
     def error_payload
       return nil if error_class.blank?
       { provider: error_provider, class: error_class, status: error_status, message: error_message }

data/app/models/completion_kit/run.rb

@@ -13,10 +13,20 @@ module CompletionKit
 
     validates :name, presence: true
     validates :status, inclusion: { in: STATUSES }
+    validate :dataset_supplies_prompt_variables
 
     before_validation :set_default_status, on: :create
     before_validation :set_auto_name, on: :create
 
+    def missing_dataset_variables
+      return [] unless prompt
+      vars = prompt.variables
+      return [] if vars.empty?
+      return vars if dataset.nil?
+
+      vars - dataset.headers
+    end
+
     def mark_completed!
       update!(status: "completed")
       broadcast_ui
@@ -236,7 +246,7 @@ module CompletionKit
       broadcast_replace_to(
         "completion_kit_run_#{id}",
         target: "run_responses",
-        html: '<div id="run_responses"></div>'
+        html: '<tbody id="run_responses"></tbody>'
       )
     end
 
@@ -267,5 +277,16 @@ module CompletionKit
       count = Run.where(prompt_id: prompt_id).count + 1
       self.name = "#{prompt.name} — v#{prompt.version_number} ##{count}"
     end
+
+    def dataset_supplies_prompt_variables
+      missing = missing_dataset_variables
+      return if missing.empty?
+
+      if dataset.nil?
+        errors.add(:dataset_id, "is required: prompt uses #{missing.join(', ')}")
+      else
+        errors.add(:dataset_id, "is missing columns required by the prompt: #{missing.join(', ')}")
+      end
+    end
   end
 end

data/app/services/completion_kit/anthropic_client.rb

@@ -5,28 +5,25 @@ module CompletionKit
       { id: "claude-3-5-haiku-latest", name: "Claude 3.5 Haiku" }
     ].freeze
 
+    def temperature_dropped?
+      @temperature_dropped == true
+    end
+
     def generate_completion(prompt, options = {})
+      @temperature_dropped = false
       return "Error: API key not configured" unless configured?
 
       model = options[:model] || "claude-3-7-sonnet-latest"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      response = build_connection("https://api.anthropic.com").post do |req|
-        req.url "/v1/messages"
-        req.headers["Content-Type"] = "application/json"
-        req.headers["x-api-key"] = api_key
-        req.headers["anthropic-version"] = "2023-06-01"
-        req.body = {
-          model: model,
-          messages: [
-            { role: "user", content: prompt }
-          ],
-          max_tokens: max_tokens,
-          temperature: temperature
-        }.to_json
+      response = post_messages(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
+
+      if response.status == 400 && temperature_unsupported?(response.body)
+        @temperature_dropped = true
+        response = post_messages(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
-      
+
       if response.status == 429
         raise CompletionKit::RateLimitError.new(
           response.body.to_s.truncate(500),
@@ -82,5 +79,27 @@ module CompletionKit
     def api_key
       @config[:api_key] || ENV["ANTHROPIC_API_KEY"]
     end
+
+    def post_messages(model:, prompt:, max_tokens:, temperature:)
+      body = {
+        model: model,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: max_tokens
+      }
+      body[:temperature] = temperature unless temperature.nil?
+
+      build_connection("https://api.anthropic.com").post do |req|
+        req.url "/v1/messages"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["x-api-key"] = api_key
+        req.headers["anthropic-version"] = "2023-06-01"
+        req.body = body.to_json
+      end
+    end
+
+    def temperature_unsupported?(body)
+      s = body.to_s
+      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported"))
+    end
   end
 end

data/app/services/completion_kit/model_discovery_service.rb

@@ -4,6 +4,8 @@ require "json"
 
 module CompletionKit
   class ModelDiscoveryService
+    class DiscoveryError < StandardError; end
+
     def initialize(config:)
       @provider = config[:provider]
       @api_key = config[:api_key]
@@ -36,11 +38,31 @@ module CompletionKit
       end
     end
 
+    def raise_fetch_error!(response)
+      label = case response.status
+              when 401, 403 then "Invalid API key for #{@provider}"
+              when 429 then "Rate limited by #{@provider}"
+              when 500..599 then "#{@provider} returned #{response.status}"
+              else "#{@provider} model list request failed (#{response.status})"
+              end
+      detail = extract_provider_error_message(response.body)
+      raise DiscoveryError, detail.present? ? "#{label}: #{detail}" : label
+    end
+
+    def extract_provider_error_message(body)
+      return nil if body.blank?
+      data = JSON.parse(body)
+      err = data["error"]
+      (err.is_a?(Hash) && err["message"]) || data["message"] || (err.is_a?(String) && err) || nil
+    rescue JSON::ParserError
+      body.to_s.truncate(200)
+    end
+
     def fetch_openai_models
       response = fetch_connection("https://api.openai.com").get("/v1/models") do |req|
         req.headers["Authorization"] = "Bearer #{@api_key}"
       end
-      return [] unless response.success?
+      raise_fetch_error!(response) unless response.success?
       JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: nil } }
     end
 
@@ -49,7 +71,7 @@ module CompletionKit
         req.headers["x-api-key"] = @api_key
         req.headers["anthropic-version"] = "2023-06-01"
       end
-      return [] unless response.success?
+      raise_fetch_error!(response) unless response.success?
       JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: e["display_name"] } }
     end
 
@@ -59,7 +81,7 @@ module CompletionKit
         req.headers["HTTP-Referer"] = "https://completionkit.com"
         req.headers["X-Title"] = "CompletionKit"
       end
-      return [] unless response.success?
+      raise_fetch_error!(response) unless response.success?
       JSON.parse(response.body).fetch("data", []).filter_map do |entry|
         next nil if entry["deprecated"] == true
         context_length = entry["context_length"].to_i
@@ -69,12 +91,12 @@ module CompletionKit
     end
 
     def fetch_ollama_models
-      return [] if @api_endpoint.nil?
+      raise DiscoveryError, "Ollama endpoint URL is required" if @api_endpoint.blank?
       base_url = @api_endpoint.to_s.delete_suffix("/")
       response = fetch_connection(base_url).get("/models") do |req|
         req.headers["Authorization"] = "Bearer #{@api_key}" if @api_key.present?
       end
-      return [] unless response.success?
+      raise_fetch_error!(response) unless response.success?
       JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: e["id"] } }
     end
 
@@ -147,14 +169,18 @@ module CompletionKit
     end
 
     def probe_generation(model)
-      response = send_probe(model.model_id, "Say hello", 65536)
+      probe_input = "Reply with exactly this token and nothing else: PING-OK"
+      response = send_probe(model.model_id, probe_input, 65536)
       if response.success?
-        text = extract_text(response)
-        if text.present?
+        text = extract_text(response).to_s
+        if text.blank?
+          model.supports_generation = false
+          model.generation_error = "Empty response"
+        elsif text.include?("PING-OK")
           model.supports_generation = true
         else
           model.supports_generation = false
-          model.generation_error = "Empty response"
+          model.generation_error = "Did not follow text completion instruction (likely non-text-output model): #{text.truncate(200)}"
         end
       else
         model.supports_generation = false

data/app/services/completion_kit/ollama_client.rb

@@ -1,22 +1,22 @@
 module CompletionKit
   class OllamaClient < LlmClient
+    def temperature_dropped?
+      @temperature_dropped == true
+    end
+
     def generate_completion(prompt, options = {})
+      @temperature_dropped = false
       return "Error: API endpoint not configured" unless configured?
 
       model = options[:model]
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      response = build_connection(api_endpoint).post do |req|
-        req.url "/v1/completions"
-        req.headers["Content-Type"] = "application/json"
-        req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
-        req.body = {
-          model: model,
-          prompt: prompt,
-          max_tokens: max_tokens,
-          temperature: temperature
-        }.to_json
+      response = post_completion(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
+
+      if response.status == 400 && temperature_unsupported?(response.body)
+        @temperature_dropped = true
+        response = post_completion(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
 
       if response.status == 429
@@ -76,5 +76,26 @@ module CompletionKit
     def api_endpoint
       (@config[:api_endpoint] || ENV["OLLAMA_API_ENDPOINT"] || "http://localhost:11434/v1").to_s.delete_suffix("/")
     end
+
+    def post_completion(model:, prompt:, max_tokens:, temperature:)
+      body = {
+        model: model,
+        prompt: prompt,
+        max_tokens: max_tokens
+      }
+      body[:temperature] = temperature unless temperature.nil?
+
+      build_connection(api_endpoint).post do |req|
+        req.url "/v1/completions"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
+        req.body = body.to_json
+      end
+    end
+
+    def temperature_unsupported?(body)
+      s = body.to_s
+      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported") || s.include?("Unsupported parameter"))
+    end
   end
 end

data/app/services/completion_kit/open_ai_client.rb

@@ -6,25 +6,23 @@ module CompletionKit
       { id: "gpt-4o-mini", name: "GPT-4o Mini" }
     ].freeze
 
+    def temperature_dropped?
+      @temperature_dropped == true
+    end
+
     def generate_completion(prompt, options = {})
+      @temperature_dropped = false
       return "Error: API key not configured" unless configured?
 
       model = options[:model] || "gpt-4.1-mini"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      response = build_connection("https://api.openai.com").post do |req|
-        req.url "/v1/responses"
-        req.headers["Content-Type"] = "application/json"
-        req.headers["Authorization"] = "Bearer #{api_key}"
-        req.body = {
-          model: model,
-          input: prompt,
-          instructions: "You are a helpful assistant.",
-          max_output_tokens: max_tokens,
-          temperature: temperature,
-          store: false
-        }.to_json
+      response = post_responses(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
+
+      if response.status == 400 && temperature_unsupported?(response.body)
+        @temperature_dropped = true
+        response = post_responses(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
 
       if response.status == 429
@@ -38,7 +36,8 @@ module CompletionKit
 
       if response.success?
         data = JSON.parse(response.body)
-        data["output"][0]["content"][0]["text"].strip
+        message = Array(data["output"]).find { |o| o["type"] == "message" }
+        message&.dig("content", 0, "text").to_s.strip
       else
         "Error: #{response.status} - #{response.body}"
       end
@@ -69,5 +68,28 @@ module CompletionKit
     def api_key
       @config[:api_key] || ENV["OPENAI_API_KEY"]
     end
+
+    def post_responses(model:, prompt:, max_tokens:, temperature:)
+      body = {
+        model: model,
+        input: prompt,
+        instructions: "You are a helpful assistant.",
+        max_output_tokens: max_tokens,
+        store: false
+      }
+      body[:temperature] = temperature unless temperature.nil?
+
+      build_connection("https://api.openai.com").post do |req|
+        req.url "/v1/responses"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["Authorization"] = "Bearer #{api_key}"
+        req.body = body.to_json
+      end
+    end
+
+    def temperature_unsupported?(body)
+      s = body.to_s
+      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported") || s.include?("Unsupported parameter"))
+    end
   end
 end

data/app/services/completion_kit/open_router_client.rb

@@ -1,28 +1,26 @@
 module CompletionKit
   class OpenRouterClient < LlmClient
-    BASE_URL = "https://openrouter.ai/api/v1".freeze
+    BASE_URL = "https://openrouter.ai".freeze
     REFERER = "https://completionkit.com".freeze
     APP_TITLE = "CompletionKit".freeze
 
+    def temperature_dropped?
+      @temperature_dropped == true
+    end
+
     def generate_completion(prompt, options = {})
+      @temperature_dropped = false
       return "Error: API key not configured" unless configured?
 
       model = options[:model] || "openai/gpt-4o-mini"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      response = build_connection(BASE_URL, timeout: 30, open_timeout: 5).post do |req|
-        req.url "/chat/completions"
-        req.headers["Content-Type"] = "application/json"
-        req.headers["Authorization"] = "Bearer #{api_key}"
-        req.headers["HTTP-Referer"] = REFERER
-        req.headers["X-Title"] = APP_TITLE
-        req.body = {
-          model: model,
-          messages: [{ role: "user", content: prompt }],
-          max_tokens: max_tokens,
-          temperature: temperature
-        }.to_json
+      response = post_chat(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
+
+      if response.status == 400 && temperature_unsupported?(response.body)
+        @temperature_dropped = true
+        response = post_chat(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
 
       if response.status == 429
@@ -67,5 +65,28 @@ module CompletionKit
     def api_key
       @config[:api_key] || ENV["OPENROUTER_API_KEY"]
     end
+
+    def post_chat(model:, prompt:, max_tokens:, temperature:)
+      body = {
+        model: model,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: max_tokens
+      }
+      body[:temperature] = temperature unless temperature.nil?
+
+      build_connection(BASE_URL, timeout: 30, open_timeout: 5).post do |req|
+        req.url "/api/v1/chat/completions"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["Authorization"] = "Bearer #{api_key}"
+        req.headers["HTTP-Referer"] = REFERER
+        req.headers["X-Title"] = APP_TITLE
+        req.body = body.to_json
+      end
+    end
+
+    def temperature_unsupported?(body)
+      s = body.to_s
+      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported") || s.include?("Unsupported parameter"))
+    end
   end
 end

data/app/services/completion_kit/worker_health.rb

@@ -4,7 +4,10 @@ module CompletionKit
 
     def self.healthy?
       return true unless defined?(::SolidQueue::Process)
-      ::SolidQueue::Process.where("last_heartbeat_at > ?", HEARTBEAT_THRESHOLD.ago).exists?
+      ::SolidQueue::Process
+        .where("kind LIKE 'Worker%'")
+        .where("last_heartbeat_at > ?", HEARTBEAT_THRESHOLD.ago)
+        .exists?
     end
   end
 end

data/app/views/completion_kit/datasets/index.html.erb

@@ -25,7 +25,7 @@
           <td><strong><%= dataset.name %></strong></td>
           <td><%= dataset.row_count %></td>
           <td><%= dataset.runs.count %></td>
-          <td><%= dataset.created_at.strftime("%Y-%m-%d") %></td>
+          <td class="ck-meta-copy"><time datetime="<%= dataset.created_at.iso8601 %>"><%= dataset.created_at.strftime("%b %-d, %Y") %></time></td>
           <td class="ck-results-table__arrow">&rarr;</td>
         </tr>
       <% end %>