completion-kit 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8454bbb11d5064ca0c6d4355c780425a28198280dffe7dd424d266fbeef6a09
-  data.tar.gz: 24c1da76e1e9118d5e2a732e8e45b684f588f553ad4d8bac89e239bc22c953c3
+  metadata.gz: 285fba79d665c4fe077b42f0e8ce888ce84a314b95698a561eea9fb48c75b045
+  data.tar.gz: a984971e294bda341824e3696cab11662f82fcee2c50974798356754ba55126c
 SHA512:
-  metadata.gz: 6fbc5b8047a20240897e19c389bb3f6104d3e2a219794d190183b5433e14d524bb692eb0a27b36ab6471e596c2b9b8af2d70a4f56ae81aa327726fe92f092eb9
-  data.tar.gz: a3399003a48836fd457a8c8b488305fad6d006596c6f940a82b232e2a731dfbc3df5ded4ba8bc16b94690a88d56baaaff6edc6801f47f2bbf422ca8fb74270df
+  metadata.gz: 1a12beaf77a9d8071949bede78336910eb8da43598609ac6307e09493d1c088a82cc3056ccaf63b3ae4eeb4ea022d19c182a05c0f037bc8a31ba21246cc5bd56
+  data.tar.gz: e007e6eeb9f7e89f3aa5ba8397338ce19778b5040f184b8cb6c012faf3f6ea464f6c3d5423fd7f7b36882494fbb90d70e068da7fe15bb76e67e9992585ba80c6

data/app/assets/stylesheets/completion_kit/application.css

@@ -3117,6 +3117,7 @@ select.ck-input {
 #ck-tab-datasets:checked ~ .ck-api-tabs__nav label[for="ck-tab-datasets"],
 #ck-tab-metrics:checked ~ .ck-api-tabs__nav label[for="ck-tab-metrics"],
 #ck-tab-metric-groups:checked ~ .ck-api-tabs__nav label[for="ck-tab-metric-groups"],
+#ck-tab-calibrations:checked ~ .ck-api-tabs__nav label[for="ck-tab-calibrations"],
 #ck-tab-tags:checked ~ .ck-api-tabs__nav label[for="ck-tab-tags"],
 #ck-tab-providers:checked ~ .ck-api-tabs__nav label[for="ck-tab-providers"] {
   color: var(--ck-accent);
@@ -3131,8 +3132,9 @@ select.ck-input {
 #ck-tab-datasets:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(5),
 #ck-tab-metrics:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(6),
 #ck-tab-metric-groups:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(7),
-#ck-tab-tags:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(8),
-#ck-tab-providers:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(9) {
+#ck-tab-calibrations:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(8),
+#ck-tab-tags:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(9),
+#ck-tab-providers:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(10) {
   display: block;
 }
 
@@ -3172,6 +3174,7 @@ select.ck-input {
   #ck-tab-datasets:checked ~ .ck-api-tabs__nav label[for="ck-tab-datasets"],
   #ck-tab-metrics:checked ~ .ck-api-tabs__nav label[for="ck-tab-metrics"],
   #ck-tab-metric-groups:checked ~ .ck-api-tabs__nav label[for="ck-tab-metric-groups"],
+  #ck-tab-calibrations:checked ~ .ck-api-tabs__nav label[for="ck-tab-calibrations"],
   #ck-tab-tags:checked ~ .ck-api-tabs__nav label[for="ck-tab-tags"],
   #ck-tab-providers:checked ~ .ck-api-tabs__nav label[for="ck-tab-providers"] {
     border-left-color: transparent;
@@ -3603,6 +3606,11 @@ select.ck-input {
   border-color: var(--ck-line);
   color: var(--ck-dim);
 }
+.ck-source-chip--revert {
+  border-color: rgba(245, 158, 11, 0.35);
+  background: rgba(245, 158, 11, 0.08);
+  color: rgb(217, 119, 6);
+}
 .ck-source-chip--current {
   border-color: var(--ck-line-strong);
   color: var(--ck-text);
@@ -6021,8 +6029,14 @@ a.tag-mark {
 }
 .ck-starter-grid {
   display: grid;
-  grid-template-columns: repeat(auto-fill, minmax(240px, 1fr));
-  gap: 10px;
+  grid-template-columns: repeat(4, 1fr);
+  gap: 12px;
+}
+@media (max-width: 1000px) {
+  .ck-starter-grid { grid-template-columns: repeat(2, 1fr); }
+}
+@media (max-width: 600px) {
+  .ck-starter-grid { grid-template-columns: 1fr; }
 }
 .ck-starter-card {
   display: flex;

data/app/controllers/completion_kit/api/v1/base_controller.rb

@@ -25,6 +25,28 @@ module CompletionKit
           render json: {error: "Record not found"}, status: :not_found
         end
 
+        PAGINATION_DEFAULT_LIMIT = 50
+        PAGINATION_MAX_LIMIT = 500
+
+        def paginate(scope)
+          total = scope.count
+          limit = (params[:limit].presence || PAGINATION_DEFAULT_LIMIT).to_i
+          limit = PAGINATION_DEFAULT_LIMIT if limit <= 0
+          limit = PAGINATION_MAX_LIMIT if limit > PAGINATION_MAX_LIMIT
+          offset = params[:offset].to_i
+          offset = 0 if offset < 0
+          response.set_header("X-Total-Count", total.to_s)
+          response.set_header("X-Limit", limit.to_s)
+          response.set_header("X-Offset", offset.to_s)
+          scope.limit(limit).offset(offset)
+        end
+
+        def filter_by_tags(scope)
+          names = Array(params[:tag]).map(&:to_s).reject(&:blank?)
+          return scope if names.empty?
+          scope.joins(:tags).where(completion_kit_tags: { name: names }).distinct
+        end
+
       end
     end
   end

data/app/controllers/completion_kit/api/v1/calibrations_controller.rb

@@ -3,10 +3,18 @@ module CompletionKit
     module V1
       class CalibrationsController < BaseController
         before_action :ensure_calibration_enabled
-        before_action :set_scope
+        before_action :set_nested_scope, only: [:create]
+        before_action :load_calibration, only: [:destroy]
 
         def index
-          render json: scope_calibrations
+          scope = Calibration.all
+          scope = scope.where(run_id: params[:run_id]) if params[:run_id].present?
+          scope = scope.where(response_id: params[:response_id]) if params[:response_id].present?
+          scope = scope.where(metric_id: params[:metric_id]) if params[:metric_id].present?
+          scope = scope.where(metric_version_id: params[:metric_version_id]) if params[:metric_version_id].present?
+          scope = scope.where(created_by: params[:created_by]) if params[:created_by].present?
+          scope = scope.where(verdict: params[:verdict]) if params[:verdict].present?
+          render json: paginate(scope.order(:created_at))
         end
 
         def create
@@ -26,13 +34,18 @@ module CompletionKit
           end
         end
 
+        def destroy
+          @calibration.destroy!
+          head :no_content
+        end
+
         private
 
         def ensure_calibration_enabled
           render(json: { error: "Calibration disabled" }, status: :not_found) unless CompletionKit.config.judge_calibration_enabled
         end
 
-        def set_scope
+        def set_nested_scope
           @run = Run.find(params[:run_id])
           @response = @run.responses.find(params[:response_id])
           @metric = Metric.find(params[:metric_id])
@@ -40,6 +53,12 @@ module CompletionKit
           not_found
         end
 
+        def load_calibration
+          @calibration = Calibration.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
         def scope_calibrations
           Calibration.where(run_id: @run.id, response_id: @response.id, metric_id: @metric.id)
         end

data/app/controllers/completion_kit/api/v1/datasets_controller.rb

@@ -5,7 +5,9 @@ module CompletionKit
         before_action :set_dataset, only: [:show, :update, :destroy]
 
         def index
-          render json: Dataset.includes(:tags).order(created_at: :desc)
+          scope = Dataset.includes(:tags)
+          scope = filter_by_tags(scope)
+          render json: paginate(scope.order(created_at: :desc))
         end
 
         def show

data/app/controllers/completion_kit/api/v1/metric_groups_controller.rb

@@ -5,7 +5,9 @@ module CompletionKit
         before_action :set_metric_group, only: [:show, :update, :destroy]
 
         def index
-          render json: MetricGroup.includes(:tags).order(created_at: :desc)
+          scope = MetricGroup.includes(:tags)
+          scope = filter_by_tags(scope)
+          render json: paginate(scope.order(created_at: :desc))
         end
 
         def show

data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb

@@ -0,0 +1,51 @@
+module CompletionKit
+  module Api
+    module V1
+      class MetricVersionsController < BaseController
+        before_action :set_metric
+        before_action :set_version, only: [:show, :publish, :destroy]
+
+        def index
+          render json: paginate(@metric.metric_versions.order(version_number: :desc))
+        end
+
+        def show
+          render json: @version
+        end
+
+        def publish
+          if @version.published? && !@version.current?
+            audit = @version.revert!
+            render json: audit
+          else
+            @version.publish!
+            render json: @version.reload
+          end
+        end
+
+        def destroy
+          if @version.published?
+            render json: { error: "Cannot dismiss a published version. Publish a different version as current instead." }, status: :conflict
+            return
+          end
+          @version.destroy!
+          head :no_content
+        end
+
+        private
+
+        def set_metric
+          @metric = Metric.find(params[:metric_id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def set_version
+          @version = @metric.metric_versions.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api/v1/metrics_controller.rb

@@ -2,10 +2,12 @@ module CompletionKit
   module Api
     module V1
       class MetricsController < BaseController
-        before_action :set_metric, only: [:show, :update, :destroy]
+        before_action :set_metric, only: [:show, :update, :destroy, :suggest_variants, :add_few_shot, :remove_few_shot]
 
         def index
-          render json: Metric.includes(:tags).order(created_at: :desc)
+          scope = Metric.includes(:tags)
+          scope = filter_by_tags(scope)
+          render json: paginate(scope.order(created_at: :desc))
         end
 
         def show
@@ -34,6 +36,51 @@ module CompletionKit
           head :no_content
         end
 
+        def suggest_variants
+          disagreement_count = Calibration.where(metric_id: @metric.id, verdict: "disagree").count
+          if disagreement_count.zero?
+            render json: { error: "Mark at least one case as Disagree before asking the model to suggest a change." }, status: :unprocessable_entity
+            return
+          end
+
+          MetricVersion.drafts.where(metric_id: @metric.id, source: "suggestion").destroy_all
+          generator = MetricVariantGenerator.new(@metric, count: params[:count].to_i, model: params[:model])
+          variants = generator.call
+          if variants.empty?
+            render json: { error: "The model returned no usable variants. Try again with a different model." }, status: :unprocessable_entity
+            return
+          end
+          versions = generator.persist!(variants)
+          render json: versions, status: :created
+        end
+
+        def add_few_shot
+          calibration = Calibration.where(metric_id: @metric.id, verdict: "disagree").find(params[:calibration_id])
+          review = calibration.response.reviews.find_by(metric_id: @metric.id)
+          examples = Array(@metric.few_shot_examples)
+          examples << {
+            "input" => calibration.response.input_data.to_s.truncate(2000),
+            "response" => calibration.response.response_text.to_s.truncate(2000),
+            "judge_score" => review&.ai_score&.to_f,
+            "judge_feedback" => review&.ai_feedback.to_s.truncate(1000),
+            "human_score" => calibration.corrected_score&.to_f,
+            "human_note" => calibration.note.to_s.truncate(1000),
+            "calibration_id" => calibration.id,
+            "added_at" => Time.current.utc.iso8601
+          }
+          @metric.update!(few_shot_examples: examples)
+          render json: @metric.reload
+        rescue ActiveRecord::RecordNotFound
+          render json: { error: "Calibration not found or not a disagree on this metric." }, status: :not_found
+        end
+
+        def remove_few_shot
+          cal_id = params[:calibration_id].to_i
+          remaining = Array(@metric.few_shot_examples).reject { |fs| fs["calibration_id"].to_i == cal_id }
+          @metric.update!(few_shot_examples: remaining)
+          render json: @metric.reload
+        end
+
         private
 
         def set_metric

data/app/controllers/completion_kit/api/v1/prompts_controller.rb

@@ -5,7 +5,9 @@ module CompletionKit
         before_action :set_prompt, only: [:show, :update, :destroy, :publish]
 
         def index
-          render json: Prompt.includes(:tags).order(created_at: :desc)
+          scope = Prompt.includes(:tags)
+          scope = filter_by_tags(scope)
+          render json: paginate(scope.order(created_at: :desc))
         end
 
         def show

data/app/controllers/completion_kit/api/v1/provider_credentials_controller.rb

@@ -5,7 +5,7 @@ module CompletionKit
         before_action :set_credential, only: [:show, :update, :destroy]
 
         def index
-          render json: ProviderCredential.order(created_at: :desc)
+          render json: paginate(ProviderCredential.order(created_at: :desc))
         end
 
         def show

data/app/controllers/completion_kit/api/v1/responses_controller.rb

@@ -6,7 +6,9 @@ module CompletionKit
         before_action :set_response, only: [:show]
 
         def index
-          render json: @run.responses.includes(:reviews)
+          scope = @run.responses.includes(:reviews)
+          scope = scope.where(status: params[:status]) if params[:status].present?
+          render json: paginate(scope.order(:id))
         end
 
         def show

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -2,10 +2,15 @@ module CompletionKit
   module Api
     module V1
       class RunsController < BaseController
-        before_action :set_run, only: [:show, :update, :destroy, :generate, :retry_failures]
+        before_action :set_run, only: [:show, :update, :destroy, :generate, :retry_failures, :rerun, :regrade, :compare]
 
         def index
-          render json: Run.includes(:tags).order(created_at: :desc)
+          scope = Run.includes(:tags)
+          scope = scope.where(status: params[:status]) if params[:status].present?
+          scope = scope.where(prompt_id: params[:prompt_id]) if params[:prompt_id].present?
+          scope = scope.where(dataset_id: params[:dataset_id]) if params[:dataset_id].present?
+          scope = filter_by_tags(scope)
+          render json: paginate(scope.order(created_at: :desc))
         end
 
         def show
@@ -71,8 +76,76 @@ module CompletionKit
           render json: @run.reload, status: :accepted
         end
 
+        def rerun
+          new_run = Run.create!(
+            prompt_id: @run.prompt_id,
+            dataset_id: @run.dataset_id,
+            judge_model: @run.judge_model,
+            temperature: @run.temperature,
+            output_column: @run.output_column,
+            tag_names: @run.tag_names,
+            status: "pending"
+          )
+          new_run.replace_metrics!(@run.metric_ids)
+          if new_run.start!
+            render json: new_run.reload, status: :accepted
+          else
+            render json: { errors: [new_run.failure_summary || "Could not start the new run."] }, status: :unprocessable_entity
+          end
+        end
+
+        def regrade
+          if @run.regrade!
+            render json: @run.reload, status: :accepted
+          else
+            render json: { error: "Nothing to re-grade. The run has no succeeded responses or no metrics attached." }, status: :unprocessable_entity
+          end
+        end
+
+        def compare
+          other = Run.find(params[:with])
+          comparison = build_run_comparison(@run, other)
+          render json: { left_run_id: @run.id, right_run_id: other.id, metric_ids: comparison[:metric_ids], rows: comparison[:rows] }
+        rescue ActiveRecord::RecordNotFound
+          render json: { error: "Other run not found. Pass ?with=<run_id>." }, status: :not_found
+        end
+
         private
 
+        def build_run_comparison(left, right)
+          left_responses = left.responses.includes(:reviews).order(:row_index, :id)
+          right_responses = right.responses.includes(:reviews).order(:row_index, :id)
+          right_by_input = right_responses.each_with_object({}) { |r, h| h[r.input_data.to_s] ||= r }
+          all_reviews = left_responses.flat_map(&:reviews) + right_responses.flat_map(&:reviews)
+          metric_ids = all_reviews.map(&:metric_id).compact.uniq
+          metric_versions = MetricVersion.where(id: all_reviews.map(&:metric_version_id).compact.uniq).index_by(&:id)
+
+          rows = left_responses.map do |lr|
+            rr = right_by_input[lr.input_data.to_s]
+            {
+              left_response_id: lr.id,
+              right_response_id: rr&.id,
+              row_index: lr.row_index,
+              per_metric: metric_ids.map do |mid|
+                l_review = lr.reviews.find { |r| r.metric_id == mid }
+                r_review = rr && rr.reviews.find { |r| r.metric_id == mid }
+                next nil if l_review.nil? && r_review.nil?
+                anchor = l_review || r_review
+                {
+                  metric_id: mid,
+                  metric_name: anchor.metric_name,
+                  left_score: l_review ? l_review.ai_score : nil,
+                  right_score: r_review ? r_review.ai_score : nil,
+                  left_metric_version_id: l_review&.metric_version_id,
+                  right_metric_version_id: r_review&.metric_version_id,
+                  delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil
+                }
+              end.compact
+            }
+          end
+          { rows: rows, metric_ids: metric_ids }
+        end
+
         def set_run
           @run = Run.find(params[:id])
         rescue ActiveRecord::RecordNotFound

data/app/controllers/completion_kit/api/v1/tags_controller.rb

@@ -5,7 +5,7 @@ module CompletionKit
         before_action :set_tag, only: [:show, :update, :destroy]
 
         def index
-          render json: Tag.order(:name)
+          render json: paginate(Tag.order(:name))
         end
 
         def show

data/app/controllers/completion_kit/metrics_controller.rb

@@ -160,13 +160,13 @@ module CompletionKit
       reverting = was_published_already && !version.current?
       previously_current = MetricVersion.current.find_by(metric_id: @metric.id)
 
-      version.publish!
-
       if reverting
+        audit = version.revert!
         prior_label = previously_current.version_label
         redirect_to metric_path(@metric),
-                    notice: "Reverted to #{@metric.name} #{version.version_label}. Pinned cases still flow to the judge, and calibration verdicts collected against #{prior_label} stay tied to it."
+                    notice: "Reverted to #{@metric.name} #{version.version_label} (now logged as #{audit.version_label}). Pinned cases still flow to the judge, and calibration verdicts collected against #{prior_label} stay tied to it."
       else
+        version.publish!
         redirect_to metric_path(@metric),
                     notice: "#{@metric.name} #{version.version_label} is now the published version."
       end

data/app/controllers/completion_kit/runs_controller.rb

@@ -176,7 +176,7 @@ module CompletionKit
         failed_response_ids.each { |rid| GenerateRowJob.perform_later(@run.id, rid) }
       end
 
-      @run.send(:broadcast_ui)
+      @run.broadcast_ui
       redirect_to run_path(@run)
     end
 

data/app/helpers/completion_kit/application_helper.rb

@@ -53,20 +53,6 @@ module CompletionKit
       end
     end
 
-    def ck_run_status_label(run)
-      case run.status
-      when "pending" then "Ready to run"
-      when "running"
-        if run.progress_total.to_i > 0
-          "Running (#{run.progress_current}/#{run.progress_total})"
-        else
-          "Running…"
-        end
-      when "completed" then "Completed"
-      when "failed" then "Failed"
-      else run.status.capitalize
-      end
-    end
 
     def ck_provider_label(provider)
       CompletionKit::ProviderCredential::PROVIDER_LABELS[provider.to_s] || provider.to_s.titleize

data/app/jobs/completion_kit/generate_row_job.rb

@@ -31,8 +31,7 @@ module CompletionKit
     before_perform do |job|
       response = Response.find_by(id: job.arguments.last)
       next unless response
-      response.update_columns(status: "retrying", attempts: response.attempts + 1)
-      response.run.send(:broadcast_response_update, response) if response.run
+      response.update!(status: "retrying", attempts: response.attempts + 1)
     end
 
     def perform(run_id, response_id)
@@ -61,12 +60,10 @@ module CompletionKit
         response_text: text,
         error_provider: nil, error_class: nil, error_status: nil, error_message: nil
       )
-      run.send(:broadcast_response_update, response)
-      run.send(:broadcast_progress)
 
       if run.judge_configured?
         run.metrics.each do |metric|
-          JudgeReviewJob.perform_later(response.id, metric.id)
+          JudgeReviewJob.perform_later(response.id, metric.id, run.id)
         end
       end
 
@@ -87,15 +84,13 @@ module CompletionKit
       response = Response.find_by(id: response_id)
       return unless response
 
-      response.update_columns(
+      response.update!(
         status: "failed",
         error_provider: provider_for(response),
         error_class: error.class.name,
         error_status: error.respond_to?(:status) ? error.status : nil,
         error_message: error.message.to_s.truncate(2000)
       )
-      response.run&.send(:broadcast_response_update, response)
-      response.run&.send(:broadcast_progress)
     end
 
     def provider_for(response)

data/app/jobs/completion_kit/judge_review_job.rb

@@ -5,7 +5,9 @@ module CompletionKit
     queue_as :llm
 
     limits_concurrency to: ENV.fetch("COMPLETION_KIT_PER_RUN_CONCURRENCY", 5).to_i,
-                       key: ->(response_id, _) { "run:#{Response.find_by(id: response_id)&.run_id}" },
+                       key: ->(response_id, _metric_id, run_id = nil) {
+                         "run:#{run_id || Response.where(id: response_id).pick(:run_id)}"
+                       },
                        duration: 10.minutes
 
     def self.rate_limit_wait(executions)
@@ -29,7 +31,7 @@ module CompletionKit
     end
 
     before_perform do |job|
-      response_id, metric_id = job.arguments
+      response_id, metric_id, _run_id = job.arguments
       response = Response.find_by(id: response_id)
       next unless response
       review = response.reviews.find_or_initialize_by(metric_id: metric_id)
@@ -37,10 +39,9 @@ module CompletionKit
       review.attempts = (review.attempts || 0) + 1
       review.status = "retrying"
       review.save!(validate: false)
-      response.run.send(:broadcast_response_update, response) if response.run
     end
 
-    def perform(response_id, metric_id)
+    def perform(response_id, metric_id, _run_id = nil)
       @response_id = response_id
       @metric_id = metric_id
 
@@ -75,8 +76,6 @@ module CompletionKit
       review.save!
 
       confirm_judging_capability(run.judge_model)
-      run.send(:broadcast_response_update, response)
-      run.send(:broadcast_progress)
       enqueue_completion_check
     end
 
@@ -107,13 +106,11 @@ module CompletionKit
         error_message: error.message.to_s.truncate(2000)
       )
       review.save!(validate: false)
-      response.run&.send(:broadcast_response_update, response)
-      response.run&.send(:broadcast_progress)
     end
 
     def provider_for(response)
       run = response.run
-      return nil unless run&.judge_model
+      return nil unless run.judge_model
       ApiConfig.provider_for_model(run.judge_model)
     end
 

data/app/models/completion_kit/metric.rb

@@ -12,6 +12,7 @@ module CompletionKit
 
     has_many :metric_group_memberships, dependent: :destroy
     has_many :metric_groups, through: :metric_group_memberships, source: :metric_group
+    has_many :metric_versions, dependent: :destroy
     has_many :reviews, dependent: :nullify
     has_many :dashboard_dismissals, as: :dismissable, dependent: :destroy
 

data/app/models/completion_kit/metric_version.rb

@@ -53,6 +53,22 @@ module CompletionKit
       self
     end
 
+    def revert!
+      raise ArgumentError, "only a published version can be reverted to" unless published?
+      audit = nil
+      MetricVersion.transaction do
+        audit = self.class.create!(
+          metric: metric,
+          instruction: instruction,
+          rubric_bands: rubric_bands,
+          state: "draft",
+          source: "revert"
+        )
+        audit.publish!
+      end
+      audit
+    end
+
     def as_json(options = {})
       {
         id: id,

data/app/models/completion_kit/response.rb

@@ -1,7 +1,6 @@
 module CompletionKit
   class Response < ApplicationRecord
-    STATUSES = %w[pending retrying succeeded failed].freeze
-    TERMINAL_STATUSES = %w[succeeded failed].freeze
+    include HasJobStatus
 
     belongs_to :run
     has_many :reviews, dependent: :destroy
@@ -10,17 +9,11 @@ module CompletionKit
     delegate :prompt, to: :run
 
     validates :response_text, presence: true, if: :succeeded?
-    validates :status, inclusion: { in: STATUSES }
 
     before_validation :set_default_status, on: :create
 
-    def terminal?
-      TERMINAL_STATUSES.include?(status)
-    end
-
-    def succeeded?
-      status == "succeeded"
-    end
+    after_save_commit :broadcast_row_update, unless: :destroyed?
+    after_save_commit :broadcast_run_progress, if: :should_broadcast_progress?
 
     def as_json(options = {})
       {
@@ -47,19 +40,22 @@ module CompletionKit
     def fully_reviewed?
       metric_ids = run.metric_ids
       return true if metric_ids.empty?
-      reviewed_metric_ids = reviews.where(status: Review::TERMINAL_STATUSES).pluck(:metric_id).uniq
+      reviewed_metric_ids = reviews.where(status: HasJobStatus::TERMINAL_STATUSES).pluck(:metric_id).uniq
       (metric_ids - reviewed_metric_ids).empty?
     end
 
-    def error_payload
-      return nil if error_class.blank?
-      { provider: error_provider, class: error_class, status: error_status, message: error_message }
+    private
+
+    def broadcast_row_update
+      run.broadcast_response_update(self)
     end
 
-    private
+    def broadcast_run_progress
+      run.broadcast_progress
+    end
 
-    def set_default_status
-      self.status ||= "pending"
+    def should_broadcast_progress?
+      saved_change_to_status? && terminal?
     end
   end
 end