completion-kit 0.4.1 → 0.4.7

data/app/controllers/completion_kit/runs_controller.rb

@@ -1,6 +1,6 @@
 module CompletionKit
   class RunsController < ApplicationController
-    before_action :set_run, only: [:show, :edit, :update, :destroy, :generate, :suggest, :suggestion, :apply_suggestion, :retry_failures]
+    before_action :set_run, only: [:show, :edit, :update, :destroy, :generate, :suggest, :retry_failures, :rerun, :refresh_status]
     before_action :load_form_collections, only: [:new, :edit, :create, :update]
 
     def index
@@ -45,7 +45,9 @@ module CompletionKit
 
     def update
       if @run.responses.any?
-        new_run = Run.create!(run_params.except(:metric_ids).to_h.merge(status: "pending"))
+        attrs = run_params.except(:metric_ids).to_h
+        attrs.delete("name") if attrs["name"].to_s == @run.name.to_s
+        new_run = Run.create!(attrs.merge(status: "pending"))
         new_run.replace_metrics!(params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
         redirect_to run_path(new_run), notice: "Saved as a new run. The previous run and its results are preserved."
       elsif @run.update(run_params.except(:metric_ids))
@@ -70,21 +72,44 @@ module CompletionKit
       end
     end
 
+    def rerun
+      new_run = Run.create!(
+        prompt_id: @run.prompt_id,
+        dataset_id: @run.dataset_id,
+        judge_model: @run.judge_model,
+        temperature: @run.temperature,
+        status: "pending"
+      )
+      new_run.replace_metrics!(@run.metric_ids)
+      if new_run.start!
+        redirect_to run_path(new_run), notice: "Re-running with the same configuration."
+      else
+        redirect_to run_path(new_run), alert: new_run.failure_summary || "Could not start the new run."
+      end
+    end
+
+    def refresh_status
+      respond_to do |format|
+        format.turbo_stream do
+          render turbo_stream: turbo_stream.replace(
+            "run_status_header",
+            partial: "completion_kit/runs/status_header",
+            locals: { run: @run }
+          )
+        end
+      end
+    end
+
     def suggest
       service = PromptImprovementService.new(@run)
       result = service.suggest
-      @run.suggestions.create!(
+      suggestion = @run.suggestions.create!(
         prompt: @run.prompt,
         reasoning: result["reasoning"],
         suggested_template: result["suggested_template"],
         original_template: result["original_template"]
       )
-      redirect_to suggestion_run_path(@run)
-    end
-
-    def suggestion
-      @suggestion = @run.suggestions.order(created_at: :desc).first
-      return redirect_to run_path(@run), alert: "No suggestion available. Generate one first." unless @suggestion
+      redirect_to suggestion_path(suggestion, from: "run")
     end
 
     def retry_failures
@@ -113,16 +138,6 @@ module CompletionKit
       redirect_to run_path(@run)
     end
 
-    def apply_suggestion
-      suggestion = @run.suggestions.order(created_at: :desc).first
-      return redirect_to run_path(@run), alert: "No suggestion to apply." unless suggestion
-
-      new_prompt = @run.prompt.clone_as_new_version(template: suggestion.suggested_template)
-      new_prompt.publish!
-      suggestion.update!(applied_at: Time.current)
-      redirect_to prompt_path(new_prompt), notice: "Suggestion applied."
-    end
-
     private
 
     def set_run

data/app/controllers/completion_kit/suggestions_controller.rb

@@ -0,0 +1,24 @@
+module CompletionKit
+  class SuggestionsController < ApplicationController
+    before_action :set_suggestion
+
+    def show
+      @run = @suggestion.run
+      @from = params[:from] == "run" ? "run" : "prompt"
+    end
+
+    def apply
+      run = @suggestion.run
+      new_prompt = run.prompt.clone_as_new_version(template: @suggestion.suggested_template)
+      new_prompt.publish!
+      @suggestion.update!(applied_at: Time.current)
+      redirect_to prompt_path(new_prompt), notice: "Suggestion applied."
+    end
+
+    private
+
+    def set_suggestion
+      @suggestion = Suggestion.find(params[:id])
+    end
+  end
+end

data/app/jobs/completion_kit/generate_row_job.rb

@@ -49,6 +49,11 @@ module CompletionKit
       raise ConfigurationError, client.configuration_errors.join(", ") unless client.configured?
 
       text = client.generate_completion(rendered, model: prompt.llm_model, temperature: run.temperature)
+      raise StandardError, text.to_s.sub(/\AError:\s*/, "") if text.to_s.start_with?("Error:")
+
+      if client.respond_to?(:temperature_dropped?) && client.temperature_dropped? && !run.temperature_ignored?
+        run.update_columns(temperature_ignored: true)
+      end
 
       response.update!(
         status: "succeeded",
@@ -56,6 +61,7 @@ module CompletionKit
         error_provider: nil, error_class: nil, error_status: nil, error_message: nil
       )
       run.send(:broadcast_response_update, response)
+      run.send(:broadcast_progress)
 
       if run.judge_configured?
         run.metrics.each do |metric|
@@ -88,6 +94,7 @@ module CompletionKit
         error_message: error.message.to_s.truncate(2000)
       )
       response.run&.send(:broadcast_response_update, response)
+      response.run&.send(:broadcast_progress)
     end
 
     def provider_for(response)

data/app/jobs/completion_kit/judge_review_job.rb

@@ -71,6 +71,7 @@ module CompletionKit
       review.save!
 
       run.send(:broadcast_response_update, response)
+      run.send(:broadcast_progress)
       enqueue_completion_check
     end
 
@@ -93,6 +94,7 @@ module CompletionKit
       )
       review.save!(validate: false)
       response.run&.send(:broadcast_response_update, response)
+      response.run&.send(:broadcast_progress)
     end
 
     def provider_for(response)

data/app/jobs/completion_kit/model_discovery_job.rb

@@ -17,9 +17,9 @@ module CompletionKit
 
     discard_on ActiveJob::DeserializationError
 
-    rescue_from(StandardError) do |_error|
+    rescue_from(StandardError) do |error|
       credential = ProviderCredential.find(arguments.first)
-      credential.update_columns(discovery_status: "failed")
+      credential.update_columns(discovery_status: "failed", discovery_error: error.message.to_s.truncate(500))
       credential.reload
       credential.broadcast_discovery_progress
     end
@@ -28,7 +28,12 @@ module CompletionKit
       credential = ProviderCredential.find_by(id: provider_credential_id)
       return unless credential
 
-      credential.update_columns(discovery_status: "discovering", discovery_current: 0, discovery_total: 0)
+      credential.update_columns(
+        discovery_status: "discovering",
+        discovery_current: 0,
+        discovery_total: 0,
+        discovery_error: nil
+      )
       credential.reload
       credential.broadcast_discovery_progress
 
@@ -39,7 +44,7 @@ module CompletionKit
         credential.broadcast_discovery_progress
       end
 
-      credential.update_columns(discovery_status: "completed", updated_at: Time.current)
+      credential.update_columns(discovery_status: "completed", discovery_error: nil, updated_at: Time.current)
       credential.reload
       credential.broadcast_discovery_complete
     end

data/app/models/completion_kit/dataset.rb

@@ -20,5 +20,14 @@ module CompletionKit
     rescue ::CSV::MalformedCSVError
       0
     end
+
+    def headers
+      return [] if csv_data.blank?
+
+      require "csv"
+      ::CSV.parse(csv_data.lines.first.to_s).first.to_a.map(&:to_s).map(&:strip)
+    rescue ::CSV::MalformedCSVError
+      []
+    end
   end
 end

data/app/models/completion_kit/provider_credential.rb

@@ -56,7 +56,7 @@ module CompletionKit
     def judge_count
       model_ids = Model.where(provider: provider).pluck(:model_id)
       return 0 if model_ids.empty?
-      Run.where(judge_model: model_ids).count
+      Run.where(judge_model: model_ids).distinct.count(:judge_model)
     end
 
     def last_used_at
@@ -75,6 +75,7 @@ module CompletionKit
         target: "discovery_status_#{id}",
         html: render_partial("completion_kit/provider_credentials/discovery_status", provider_credential: self)
       )
+      broadcast_provider_models
     end
 
     def broadcast_discovery_complete
@@ -82,6 +83,16 @@ module CompletionKit
       broadcast_model_dropdowns
     end
 
+    def broadcast_provider_models
+      Turbo::StreamsChannel.broadcast_action_to(
+        "completion_kit_provider_#{id}",
+        action: "replace",
+        target: "provider_models_#{id}",
+        method: "morph",
+        html: render_partial("completion_kit/provider_credentials/models_card", provider_credential: self)
+      )
+    end
+
     private
 
     def enqueue_discovery

data/app/models/completion_kit/response.rb

@@ -43,6 +43,13 @@ module CompletionKit
       reviews.any? { |r| r.ai_score.present? }
     end
 
+    def fully_reviewed?
+      metric_ids = run.metric_ids
+      return true if metric_ids.empty?
+      reviewed_metric_ids = reviews.where(status: Review::TERMINAL_STATUSES).pluck(:metric_id).uniq
+      (metric_ids - reviewed_metric_ids).empty?
+    end
+
     def error_payload
       return nil if error_class.blank?
       { provider: error_provider, class: error_class, status: error_status, message: error_message }

data/app/models/completion_kit/run.rb

@@ -13,10 +13,20 @@ module CompletionKit
 
     validates :name, presence: true
     validates :status, inclusion: { in: STATUSES }
+    validate :dataset_supplies_prompt_variables
 
     before_validation :set_default_status, on: :create
     before_validation :set_auto_name, on: :create
 
+    def missing_dataset_variables
+      return [] unless prompt
+      vars = prompt.variables
+      return [] if vars.empty?
+      return vars if dataset.nil?
+
+      vars - dataset.headers
+    end
+
     def mark_completed!
       update!(status: "completed")
       broadcast_ui
@@ -119,12 +129,28 @@ module CompletionKit
       generated_total = progress_total
 
       metric_count = metrics.count
-      succeeded_count = generated_done
-      judged_total = succeeded_count * metric_count
-      judged_done = Review.joins(:response)
-        .where(completion_kit_responses: { run_id: id }, status: "succeeded").count
-      judged_failed = Review.joins(:response)
-        .where(completion_kit_responses: { run_id: id }, status: "failed").count
+      judged_total = metric_count > 0 ? generated_done : 0
+      judged_done = 0
+      judged_failed = 0
+
+      if metric_count > 0 && judged_total > 0
+        succeeded_response_ids = responses.where(status: "succeeded").pluck(:id)
+        metric_ids = metrics.pluck(:id)
+        review_counts = Review
+          .where(response_id: succeeded_response_ids, metric_id: metric_ids)
+          .group(:response_id, :status)
+          .count
+        succeeded_response_ids.each do |rid|
+          ok = review_counts[[rid, "succeeded"]] || 0
+          bad = review_counts[[rid, "failed"]] || 0
+          next unless ok + bad == metric_count
+          if bad > 0
+            judged_failed += 1
+          else
+            judged_done += 1
+          end
+        end
+      end
 
       {
         generated_done: generated_done,
@@ -175,6 +201,7 @@ module CompletionKit
     end
 
     def render_engine_partial(partial, locals)
+      CompletionKit::Engine.routes.url_helpers
       CompletionKit::ApplicationController.render(
         partial: partial,
         locals: locals
@@ -185,8 +212,8 @@ module CompletionKit
       reload
       broadcast_replace_to(
         "completion_kit_run_#{id}",
-        target: "run_progress",
-        html: render_engine_partial("completion_kit/runs/progress", run: self)
+        target: "run_status_panel",
+        html: render_engine_partial("completion_kit/runs/status_panel", run: self)
       )
       broadcast_status_header
     end
@@ -219,7 +246,7 @@ module CompletionKit
       broadcast_replace_to(
         "completion_kit_run_#{id}",
         target: "run_responses",
-        html: '<div id="run_responses"></div>'
+        html: '<tbody id="run_responses"></tbody>'
       )
     end
 
@@ -250,5 +277,16 @@ module CompletionKit
       count = Run.where(prompt_id: prompt_id).count + 1
       self.name = "#{prompt.name} — v#{prompt.version_number} ##{count}"
     end
+
+    def dataset_supplies_prompt_variables
+      missing = missing_dataset_variables
+      return if missing.empty?
+
+      if dataset.nil?
+        errors.add(:dataset_id, "is required: prompt uses #{missing.join(', ')}")
+      else
+        errors.add(:dataset_id, "is missing columns required by the prompt: #{missing.join(', ')}")
+      end
+    end
   end
 end

data/app/services/completion_kit/anthropic_client.rb

@@ -5,28 +5,25 @@ module CompletionKit
       { id: "claude-3-5-haiku-latest", name: "Claude 3.5 Haiku" }
     ].freeze
 
+    def temperature_dropped?
+      @temperature_dropped == true
+    end
+
     def generate_completion(prompt, options = {})
+      @temperature_dropped = false
       return "Error: API key not configured" unless configured?
 
       model = options[:model] || "claude-3-7-sonnet-latest"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      response = build_connection("https://api.anthropic.com").post do |req|
-        req.url "/v1/messages"
-        req.headers["Content-Type"] = "application/json"
-        req.headers["x-api-key"] = api_key
-        req.headers["anthropic-version"] = "2023-06-01"
-        req.body = {
-          model: model,
-          messages: [
-            { role: "user", content: prompt }
-          ],
-          max_tokens: max_tokens,
-          temperature: temperature
-        }.to_json
+      response = post_messages(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
+
+      if response.status == 400 && temperature_unsupported?(response.body)
+        @temperature_dropped = true
+        response = post_messages(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
       end
-      
+
       if response.status == 429
         raise CompletionKit::RateLimitError.new(
           response.body.to_s.truncate(500),
@@ -82,5 +79,27 @@ module CompletionKit
     def api_key
       @config[:api_key] || ENV["ANTHROPIC_API_KEY"]
     end
+
+    def post_messages(model:, prompt:, max_tokens:, temperature:)
+      body = {
+        model: model,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: max_tokens
+      }
+      body[:temperature] = temperature unless temperature.nil?
+
+      build_connection("https://api.anthropic.com").post do |req|
+        req.url "/v1/messages"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["x-api-key"] = api_key
+        req.headers["anthropic-version"] = "2023-06-01"
+        req.body = body.to_json
+      end
+    end
+
+    def temperature_unsupported?(body)
+      s = body.to_s
+      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported"))
+    end
   end
 end