completion-kit 0.6.0 → 0.8.0

data/app/jobs/completion_kit/generate_row_job.rb

@@ -31,8 +31,7 @@ module CompletionKit
     before_perform do |job|
       response = Response.find_by(id: job.arguments.last)
       next unless response
-      response.update_columns(status: "retrying", attempts: response.attempts + 1)
-      response.run.send(:broadcast_response_update, response) if response.run
+      response.update!(status: "retrying", attempts: response.attempts + 1)
     end
 
     def perform(run_id, response_id)
@@ -61,12 +60,10 @@ module CompletionKit
         response_text: text,
         error_provider: nil, error_class: nil, error_status: nil, error_message: nil
       )
-      run.send(:broadcast_response_update, response)
-      run.send(:broadcast_progress)
 
       if run.judge_configured?
         run.metrics.each do |metric|
-          JudgeReviewJob.perform_later(response.id, metric.id)
+          JudgeReviewJob.perform_later(response.id, metric.id, run.id)
         end
       end
 
@@ -83,19 +80,16 @@ module CompletionKit
     end
 
     def record_terminal_failure!(error)
-      response_id = @response_id || arguments.last
-      response = Response.find_by(id: response_id)
+      response = Response.find_by(id: @response_id)
       return unless response
 
-      response.update_columns(
+      response.update!(
         status: "failed",
         error_provider: provider_for(response),
         error_class: error.class.name,
         error_status: error.respond_to?(:status) ? error.status : nil,
         error_message: error.message.to_s.truncate(2000)
       )
-      response.run&.send(:broadcast_response_update, response)
-      response.run&.send(:broadcast_progress)
     end
 
     def provider_for(response)
@@ -103,8 +97,7 @@ module CompletionKit
     end
 
     def enqueue_completion_check
-      run_id = @run_id || arguments.first
-      RunCompletionCheckJob.perform_later(run_id)
+      RunCompletionCheckJob.perform_later(@run_id)
     end
   end
 end

data/app/jobs/completion_kit/judge_review_job.rb

@@ -5,7 +5,9 @@ module CompletionKit
     queue_as :llm
 
     limits_concurrency to: ENV.fetch("COMPLETION_KIT_PER_RUN_CONCURRENCY", 5).to_i,
-                       key: ->(response_id, _) { "run:#{Response.find_by(id: response_id)&.run_id}" },
+                       key: ->(response_id, _metric_id, run_id = nil) {
+                         "run:#{run_id || Response.where(id: response_id).pick(:run_id)}"
+                       },
                        duration: 10.minutes
 
     def self.rate_limit_wait(executions)
@@ -29,7 +31,7 @@ module CompletionKit
     end
 
     before_perform do |job|
-      response_id, metric_id = job.arguments
+      response_id, metric_id, _run_id = job.arguments
       response = Response.find_by(id: response_id)
       next unless response
       review = response.reviews.find_or_initialize_by(metric_id: metric_id)
@@ -37,10 +39,9 @@ module CompletionKit
       review.attempts = (review.attempts || 0) + 1
       review.status = "retrying"
       review.save!(validate: false)
-      response.run.send(:broadcast_response_update, response) if response.run
     end
 
-    def perform(response_id, metric_id)
+    def perform(response_id, metric_id, _run_id = nil)
       @response_id = response_id
       @metric_id = metric_id
 
@@ -75,8 +76,6 @@ module CompletionKit
       review.save!
 
       confirm_judging_capability(run.judge_model)
-      run.send(:broadcast_response_update, response)
-      run.send(:broadcast_progress)
       enqueue_completion_check
     end
 
@@ -92,14 +91,12 @@ module CompletionKit
     end
 
     def record_terminal_failure!(error)
-      response_id = @response_id || arguments.first
-      metric_id = @metric_id || arguments.last
-      response = Response.find_by(id: response_id)
+      response = Response.find_by(id: @response_id)
       return unless response
 
-      review = response.reviews.find_or_initialize_by(metric_id: metric_id)
+      review = response.reviews.find_or_initialize_by(metric_id: @metric_id)
       review.assign_attributes(
-        metric_name: review.metric_name || Metric.find_by(id: metric_id)&.name || "(deleted metric)",
+        metric_name: review.metric_name || Metric.find_by(id: @metric_id)&.name || "(deleted metric)",
         status: "failed",
         error_provider: provider_for(response),
         error_class: error.class.name,
@@ -107,19 +104,16 @@ module CompletionKit
         error_message: error.message.to_s.truncate(2000)
       )
       review.save!(validate: false)
-      response.run&.send(:broadcast_response_update, response)
-      response.run&.send(:broadcast_progress)
     end
 
     def provider_for(response)
       run = response.run
-      return nil unless run&.judge_model
+      return nil unless run.judge_model
       ApiConfig.provider_for_model(run.judge_model)
     end
 
     def enqueue_completion_check
-      response_id = @response_id || arguments.first
-      response = Response.find_by(id: response_id)
+      response = Response.find_by(id: @response_id)
       RunCompletionCheckJob.perform_later(response.run_id) if response
     end
 

data/app/models/completion_kit/metric.rb

@@ -12,6 +12,7 @@ module CompletionKit
 
     has_many :metric_group_memberships, dependent: :destroy
     has_many :metric_groups, through: :metric_group_memberships, source: :metric_group
+    has_many :metric_versions, dependent: :destroy
     has_many :reviews, dependent: :nullify
     has_many :dashboard_dismissals, as: :dismissable, dependent: :destroy
 

data/app/models/completion_kit/metric_version.rb

@@ -53,6 +53,22 @@ module CompletionKit
       self
     end
 
+    def revert!
+      raise ArgumentError, "only a published version can be reverted to" unless published?
+      audit = nil
+      MetricVersion.transaction do
+        audit = self.class.create!(
+          metric: metric,
+          instruction: instruction,
+          rubric_bands: rubric_bands,
+          state: "draft",
+          source: "revert"
+        )
+        audit.publish!
+      end
+      audit
+    end
+
     def as_json(options = {})
       {
         id: id,

data/app/models/completion_kit/response.rb

@@ -1,7 +1,6 @@
 module CompletionKit
   class Response < ApplicationRecord
-    STATUSES = %w[pending retrying succeeded failed].freeze
-    TERMINAL_STATUSES = %w[succeeded failed].freeze
+    include HasJobStatus
 
     belongs_to :run
     has_many :reviews, dependent: :destroy
@@ -10,17 +9,11 @@ module CompletionKit
     delegate :prompt, to: :run
 
     validates :response_text, presence: true, if: :succeeded?
-    validates :status, inclusion: { in: STATUSES }
 
     before_validation :set_default_status, on: :create
 
-    def terminal?
-      TERMINAL_STATUSES.include?(status)
-    end
-
-    def succeeded?
-      status == "succeeded"
-    end
+    after_save_commit :broadcast_row_update, unless: :destroyed?
+    after_save_commit :broadcast_run_progress, if: :should_broadcast_progress?
 
     def as_json(options = {})
       {
@@ -47,19 +40,22 @@ module CompletionKit
     def fully_reviewed?
       metric_ids = run.metric_ids
       return true if metric_ids.empty?
-      reviewed_metric_ids = reviews.where(status: Review::TERMINAL_STATUSES).pluck(:metric_id).uniq
+      reviewed_metric_ids = reviews.where(status: HasJobStatus::TERMINAL_STATUSES).pluck(:metric_id).uniq
       (metric_ids - reviewed_metric_ids).empty?
     end
 
-    def error_payload
-      return nil if error_class.blank?
-      { provider: error_provider, class: error_class, status: error_status, message: error_message }
+    private
+
+    def broadcast_row_update
+      run.broadcast_response_update(self)
     end
 
-    private
+    def broadcast_run_progress
+      run.broadcast_progress
+    end
 
-    def set_default_status
-      self.status ||= "pending"
+    def should_broadcast_progress?
+      saved_change_to_status? && terminal?
     end
   end
 end

data/app/models/completion_kit/review.rb

@@ -1,37 +1,25 @@
 module CompletionKit
   class Review < ApplicationRecord
-    STATUSES = %w[pending retrying succeeded failed].freeze
-    TERMINAL_STATUSES = %w[succeeded failed].freeze
+    include HasJobStatus
 
     belongs_to :response
     belongs_to :metric, optional: true
     belongs_to :metric_version, optional: true
     has_many :dashboard_dismissals, as: :dismissable, dependent: :destroy
 
-    def stale_against_current_judge?
-      return false unless metric_id && metric_version_id
-      current_id = MetricVersion.current.where(metric_id: metric_id).limit(1).pick(:id)
-      return false if current_id.nil?
-      metric_version_id != current_id
-    end
-
     validates :metric_name, presence: true
-    validates :status, inclusion: { in: STATUSES }
     validates :ai_score, numericality: { greater_than_or_equal_to: 1, less_than_or_equal_to: 5 }, allow_nil: true
 
     before_validation :set_default_status
 
-    def terminal?
-      TERMINAL_STATUSES.include?(status)
-    end
-
-    def succeeded?
-      status == "succeeded"
-    end
+    after_save_commit :broadcast_parent_row_update, unless: :destroyed?
+    after_save_commit :broadcast_run_progress, if: :should_broadcast_progress?
 
-    def error_payload
-      return nil if error_class.blank?
-      { provider: error_provider, class: error_class, status: error_status, message: error_message }
+    def stale_against_current_judge?
+      return false unless metric_id && metric_version_id
+      current_id = MetricVersion.current.where(metric_id: metric_id).limit(1).pick(:id)
+      return false if current_id.nil?
+      metric_version_id != current_id
     end
 
     def as_json(options = {})
@@ -46,8 +34,16 @@ module CompletionKit
 
     private
 
-    def set_default_status
-      self.status ||= "pending"
+    def broadcast_parent_row_update
+      response.run.broadcast_response_update(response)
+    end
+
+    def broadcast_run_progress
+      response.run.broadcast_progress
+    end
+
+    def should_broadcast_progress?
+      saved_change_to_status? && terminal?
     end
   end
 end

data/app/models/completion_kit/run.rb

@@ -43,7 +43,7 @@ module CompletionKit
     end
 
     def outstanding_work_zero?
-      return false if responses.where.not(status: Response::TERMINAL_STATUSES).exists?
+      return false if responses.where.not(status: HasJobStatus::TERMINAL_STATUSES).exists?
 
       metric_ids = metrics.pluck(:id)
       return true if metric_ids.empty?
@@ -55,7 +55,7 @@ module CompletionKit
       terminal_review_count = Review.where(
         response_id: succeeded_response_ids,
         metric_id: metric_ids,
-        status: Review::TERMINAL_STATUSES
+        status: HasJobStatus::TERMINAL_STATUSES
       ).count
 
       terminal_review_count >= expected_reviews
@@ -118,6 +118,10 @@ module CompletionKit
     end
 
     def start!
+      unless %w[pending failed].include?(status)
+        return fail_with_summary!("Cannot start a run in state \"#{status}\". Use rerun to create a fresh copy, or retry_failures / regrade to work with the existing responses.")
+      end
+
       rows = if dataset
                CsvProcessor.process_self(self)
              else
@@ -161,7 +165,7 @@ module CompletionKit
           response = responses.create!(attrs)
 
           if judge_only?
-            metrics.each { |m| JudgeReviewJob.perform_later(response.id, m.id) } if judge_configured?
+            metrics.each { |m| JudgeReviewJob.perform_later(response.id, m.id, id) } if judge_configured?
           else
             GenerateRowJob.perform_later(id, response.id)
           end
@@ -202,7 +206,7 @@ module CompletionKit
         update!(status: "running", failure_summary: nil, error_message: nil)
 
         response_ids.each do |rid|
-          grading_metrics.each { |m| JudgeReviewJob.perform_later(rid, m.id) }
+          grading_metrics.each { |m| JudgeReviewJob.perform_later(rid, m.id, id) }
         end
         RunCompletionCheckJob.perform_later(id)
       end
@@ -272,17 +276,6 @@ module CompletionKit
       }
     end
 
-    private
-
-    def fail_with_summary!(message)
-      errors.add(:base, message)
-      if persisted?
-        update_columns(status: "failed", failure_summary: message, error_message: message)
-        broadcast_ui
-      end
-      false
-    end
-
     def broadcast_ui
       broadcast_progress
       broadcast_status_header
@@ -290,14 +283,6 @@ module CompletionKit
       broadcast_sort_toolbar
     end
 
-    def render_engine_partial(partial, locals)
-      CompletionKit::Engine.warm_routes!
-      CompletionKit::ApplicationController.render(
-        partial: partial,
-        locals: locals
-      )
-    end
-
     def broadcast_progress
       reload
       broadcast_replace_to(
@@ -305,7 +290,6 @@ module CompletionKit
         target: "run_status_panel",
         html: render_engine_partial("completion_kit/runs/status_panel", run: self)
       )
-      broadcast_status_header
     end
 
     def broadcast_status_header
@@ -356,6 +340,25 @@ module CompletionKit
       )
     end
 
+    private
+
+    def fail_with_summary!(message)
+      errors.add(:base, message)
+      if persisted?
+        update_columns(status: "failed", failure_summary: message, error_message: message)
+        broadcast_ui
+      end
+      false
+    end
+
+    def render_engine_partial(partial, locals)
+      CompletionKit::Engine.warm_routes!
+      CompletionKit::ApplicationController.render(
+        partial: partial,
+        locals: locals
+      )
+    end
+
     def set_default_status
       self.status ||= "pending"
     end

data/app/models/concerns/completion_kit/has_job_status.rb

@@ -0,0 +1,31 @@
+module CompletionKit
+  module HasJobStatus
+    extend ActiveSupport::Concern
+
+    STATUSES = %w[pending retrying succeeded failed].freeze
+    TERMINAL_STATUSES = %w[succeeded failed].freeze
+
+    included do
+      validates :status, inclusion: { in: STATUSES }
+    end
+
+    def terminal?
+      TERMINAL_STATUSES.include?(status)
+    end
+
+    def succeeded?
+      status == "succeeded"
+    end
+
+    def error_payload
+      return nil if error_class.blank?
+      { provider: error_provider, class: error_class, status: error_status, message: error_message }
+    end
+
+    private
+
+    def set_default_status
+      self.status ||= "pending"
+    end
+  end
+end

data/app/services/completion_kit/mcp_dispatcher.rb

@@ -32,6 +32,7 @@ module CompletionKit
         McpTools::Datasets.definitions +
         McpTools::Metrics.definitions +
         McpTools::MetricGroups.definitions +
+        McpTools::MetricVersions.definitions +
         McpTools::ProviderCredentials.definitions +
         McpTools::Tags.definitions +
         McpTools::Calibrations.definitions +
@@ -44,8 +45,9 @@ module CompletionKit
       when /\Aruns_/                 then McpTools::Runs.call(name, arguments)
       when /\Aresponses_/            then McpTools::Responses.call(name, arguments)
       when /\Adatasets_/             then McpTools::Datasets.call(name, arguments)
-      when /\Ametrics_/              then McpTools::Metrics.call(name, arguments)
+      when /\Ametric_versions_/      then McpTools::MetricVersions.call(name, arguments)
       when /\Ametric_groups_/        then McpTools::MetricGroups.call(name, arguments)
+      when /\Ametrics_/              then McpTools::Metrics.call(name, arguments)
       when /\Aprovider_credentials_/ then McpTools::ProviderCredentials.call(name, arguments)
       when /\Atags_/                 then McpTools::Tags.call(name, arguments)
       when /\Acalibrations_/         then McpTools::Calibrations.call(name, arguments)

data/app/services/completion_kit/mcp_tools/metric_versions.rb

@@ -0,0 +1,67 @@
+module CompletionKit
+  module McpTools
+    module MetricVersions
+      extend Base
+
+      TOOLS = {
+        "metric_versions_list" => {
+          description: "List every MetricVersion (drafts + published) for a metric, newest first. Each row carries version_number, state, source, current flag, and timestamps.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              metric_id: { type: "integer" }
+            },
+            required: ["metric_id"]
+          },
+          handler: :list
+        },
+        "metric_versions_publish" => {
+          description: "Publish a MetricVersion as the live judge for its metric. Works for both 'draft → published' and 'revert to an older published version → current'. Transactionally flips current, demotes peers, and writes the version's instruction + rubric_bands back onto the metric so the judge actually uses them.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              metric_version_id: { type: "integer" }
+            },
+            required: ["metric_version_id"]
+          },
+          handler: :publish
+        },
+        "metric_versions_dismiss" => {
+          description: "Destroy a draft MetricVersion (use for either source: 'edit' or source: 'suggestion'). Published versions are refused — to demote a published version, publish a different one as current instead.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              metric_version_id: { type: "integer" }
+            },
+            required: ["metric_version_id"]
+          },
+          handler: :dismiss
+        }
+      }.freeze
+
+      def self.list(args)
+        metric = CompletionKit::Metric.find(args["metric_id"])
+        versions = CompletionKit::MetricVersion.where(metric_id: metric.id).order(version_number: :desc)
+        text_result(versions.map(&:as_json))
+      end
+
+      def self.publish(args)
+        version = CompletionKit::MetricVersion.find(args["metric_version_id"])
+        if version.published? && !version.current?
+          audit = version.revert!
+          text_result(audit.as_json)
+        else
+          version.publish!
+          text_result(version.reload.as_json)
+        end
+      end
+
+      def self.dismiss(args)
+        version = CompletionKit::MetricVersion.find(args["metric_version_id"])
+        return error_result("Cannot dismiss a published version. Publish a different version as current instead.") if version.published?
+        version.destroy!
+        text_result({id: version.id, destroyed: true})
+      end
+    end
+  end
+end

data/app/services/completion_kit/starter_metrics.rb

@@ -21,8 +21,8 @@ module CompletionKit
         key: "instruction_following",
         name: "Instruction following",
         description: "Did the model do everything that was asked?",
-        catches: "The response is factually right but ignores \"answer in two sentences\", \"use bullet points\", \"do not include X\". Distinct from Correctness — a response can be right and still fail this.",
-        instruction: "Did the model do every concrete thing the prompt asked for? Score against the explicit requirements in the prompt (format constraints, count limits, exclusions, audience cues). Factual accuracy is a different dimension — score that elsewhere.",
+        catches: "The response is factually right but ignores \"answer in two sentences\", \"use bullet points\", \"do not include X\". Distinct from Correctness; a response can be right and still fail this.",
+        instruction: "Did the model do every concrete thing the prompt asked for? Score against the explicit requirements in the prompt (format constraints, count limits, exclusions, audience cues). Factual accuracy is a different dimension; score that elsewhere.",
         rubric_bands: [
           { "stars" => 5, "description" => "Followed every requirement in the prompt exactly." },
           { "stars" => 4, "description" => "Followed every requirement with a small slip." },
@@ -36,7 +36,7 @@ module CompletionKit
         name: "Format compliance",
         description: "Does the output follow the required structure?",
         catches: "Invalid JSON, missing schema fields, extra prose around a structured response, wrong casing on keys. Critical for any LLM wired into an API.",
-        instruction: "Does the output match the format the prompt asked for — JSON shape, schema, keys, casing, no stray prose? Score on whether a downstream parser would accept it without massaging.",
+        instruction: "Does the output match the format the prompt asked for: JSON shape, schema, keys, casing, no stray prose? Score on whether a downstream parser would accept it without massaging.",
         rubric_bands: [
           { "stars" => 5, "description" => "Exact spec, ready to consume programmatically." },
           { "stars" => 4, "description" => "Spec-compliant with one cosmetic issue." },
@@ -62,9 +62,9 @@ module CompletionKit
       Starter.new(
         key: "conciseness",
         name: "Conciseness",
-        description: "Is it the right length — no padding, no missing detail?",
+        description: "Is it the right length, no padding, no missing detail?",
         catches: "Rambling responses, repetitive caveats, over-hedging. LLMs default to verbose. Conciseness is the dimension where users most often see scores move after tuning.",
-        instruction: "Is the output the right length for the task — no padding, no missing detail, no hedging filler? Penalise rambling, repetition, over-caveating, and unnecessary preamble. Penalise too-short outputs that drop information.",
+        instruction: "Is the output the right length for the task: no padding, no missing detail, no hedging filler? Penalise rambling, repetition, over-caveating, and unnecessary preamble. Penalise too-short outputs that drop information.",
         rubric_bands: [
           { "stars" => 5, "description" => "Exactly as long as the task needs, no more, no less." },
           { "stars" => 4, "description" => "Right length with a small redundancy." },