completion-kit 0.28.19 → 0.28.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e30a6a27a48e1a23c1947477068eec74f8ff2ff869bab4592f68c612a728dc76
-  data.tar.gz: c1cdf115062d9ddd903296574420bd75128995460ce26e64e44681cdd6655ed9
+  metadata.gz: 8242f60832e3c23d204132839a832e082eb5115c26afd46eb2082ca4e342f908
+  data.tar.gz: 5798c4e0f853e3aa5a2ae20291a7909cc5f70dd7baaa5e9bd2765cb5cbbe5f4f
 SHA512:
-  metadata.gz: ef4c4ddecf48be58b43ff9d02b1e2743d75b28eabdccf13bc35db32c0225b4a880318a4e35faa7ba8c235a29f77c934ad323f5099a355edd0d29ba18f8293566
-  data.tar.gz: cd6b8331a0ad697ca16e71c7d6eb623d70e050c1fe50332672bd797c01a42509cef2c2ba6738bbebf3dfdff6d1feb07a29ea2f205408390259ea8d809622103a
+  metadata.gz: ec13a779260638ccd8968c9c3be4bd250780fdda06f01ea13b9963553756329cb0c092603a7ee66118e854227bcdb68b2b360e66a1e7830c2cfeffc879bf35d3
+  data.tar.gz: 92dd2c71fd38a9cc31638c77927a21a9a53f6ba8a4f4b81dc876f652dbf91c3d601722b402dd71f108d7530ab96442d1d2c113aadf60684ab767902c1e3ec1c9

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -54,40 +54,12 @@ module CompletionKit
             return render_error("Judge has changed since this run executed. Retry would mix versions in the same run; use POST /api/v1/runs/:id/rerun instead.", status: :conflict)
           end
 
-          scope = @run.responses.where(status: "failed")
-          scope = scope.where(id: params[:only]) if params[:only].present?
-
-          ActiveRecord::Base.transaction do
-            failed_response_ids = scope.pluck(:id)
-            CompletionKit::Review.where(response_id: failed_response_ids, status: "failed").update_all(
-              status: "pending", attempts: 0,
-              error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-              ai_score: nil, passed: nil, ai_feedback: nil
-            )
-            scope.update_all(
-              status: "pending", attempts: 0,
-              error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-              response_text: nil
-            )
-            @run.update!(status: "running")
-            failed_response_ids.each { |rid| CompletionKit::GenerateRowJob.perform_later(@run.id, rid) }
-          end
-
+          @run.retry_failures!(only: params[:only])
           render json: @run.reload, status: :accepted
         end
 
         def rerun
-          new_run = Run.create!(
-            prompt_id: @run.prompt_id,
-            dataset_id: @run.dataset_id,
-            judge_model: @run.judge_model,
-            temperature: @run.temperature,
-            output_column: @run.output_column,
-            expected_column: @run.expected_column,
-            tag_names: @run.tag_names,
-            status: "pending"
-          )
-          new_run.replace_metrics!(@run.metric_ids)
+          new_run = @run.rerun!
           if new_run.start!
             render json: new_run.reload, status: :accepted
           else

data/app/controllers/completion_kit/runs_controller.rb

@@ -6,8 +6,9 @@ module CompletionKit
     before_action :load_form_collections, only: [:new, :edit, :create, :update]
 
     def index
-      scope = Run.includes(:prompt, :dataset, :tags, responses: :reviews).order(created_at: :desc).display_scoped
-      @runs = apply_tag_filter(scope)
+      scope = Run.includes(:prompt, :dataset, :tags).order(created_at: :desc).display_scoped
+      @runs = apply_tag_filter(scope).load
+      Run.preload_summaries(@runs)
     end
 
     RESPONSES_PER_PAGE = 100
@@ -110,17 +111,7 @@ module CompletionKit
     end
 
     def rerun
-      new_run = Run.create!(
-        prompt_id: @run.prompt_id,
-        dataset_id: @run.dataset_id,
-        judge_model: @run.judge_model,
-        temperature: @run.temperature,
-        output_column: @run.output_column,
-        expected_column: @run.expected_column,
-        tag_names: @run.tag_names,
-        status: "pending"
-      )
-      new_run.replace_metrics!(@run.metric_ids)
+      new_run = @run.rerun!
       if new_run.start!
         redirect_to run_path(new_run), notice: "Re-running with the same configuration."
       else
@@ -162,27 +153,7 @@ module CompletionKit
         return
       end
 
-      scope = @run.responses.where(status: "failed")
-      scope = scope.where(id: params[:only]) if params[:only].present?
-
-      ActiveRecord::Base.transaction do
-        failed_response_ids = scope.pluck(:id)
-        Review.where(response_id: failed_response_ids, status: "failed").update_all(
-          status: "pending",
-          attempts: 0,
-          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-          ai_score: nil, passed: nil, ai_feedback: nil
-        )
-        scope.update_all(
-          status: "pending",
-          attempts: 0,
-          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-          response_text: nil
-        )
-        @run.update!(status: "running")
-        failed_response_ids.each { |rid| GenerateRowJob.perform_later(@run.id, rid) }
-      end
-
+      @run.retry_failures!(only: params[:only])
       @run.broadcast_ui
       redirect_to run_path(@run)
     end

data/app/models/completion_kit/run.rb

@@ -14,6 +14,8 @@ module CompletionKit
     has_many :suggestions, dependent: :destroy
     has_many :dashboard_dismissals, as: :dismissable, dependent: :destroy
 
+    attr_writer :avg_score, :check_pass_rate, :metric_averages, :response_count
+
     validates :name, presence: true
     validates :status, inclusion: { in: STATUSES }
     validate :dataset_supplies_prompt_variables
@@ -34,6 +36,58 @@ module CompletionKit
       display_scoped.select(:id)
     end
 
+    # Batch-compute the list-view summaries (response count, avg score, check
+    # pass rate, per-metric averages) for a set of runs in a constant number of
+    # grouped queries, injecting the results so the index never loads a single
+    # response or review object. Mirrors the per-run reader methods exactly.
+    def self.preload_summaries(runs)
+      runs = runs.to_a
+      return runs if runs.empty?
+
+      run_ids = runs.map(&:id)
+      counts = Response.where(run_id: run_ids).group(:run_id).count
+
+      run_col = Arel.sql("completion_kit_responses.run_id")
+      base = Review.joins(:response).where(completion_kit_responses: {run_id: run_ids})
+
+      run_rows = base.group(run_col).pluck(
+        run_col,
+        Arel.sql("AVG(ai_score)"),
+        Arel.sql("COUNT(passed)"),
+        Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
+      )
+      run_stats = run_rows.each_with_object({}) do |(rid, avg, resolved, passed), h|
+        h[rid] = {avg: avg, resolved: resolved.to_i, passed: passed.to_i}
+      end
+
+      metric_rows = base.group(run_col, :metric_name).pluck(
+        run_col,
+        :metric_name,
+        Arel.sql("AVG(ai_score)"),
+        Arel.sql("COUNT(ai_score)"),
+        Arel.sql("COUNT(passed)"),
+        Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
+      )
+      metrics_by_run = metric_rows.group_by(&:first)
+
+      runs.each do |run|
+        run.response_count = counts.fetch(run.id, 0)
+
+        stats = run_stats[run.id]
+        run.avg_score = stats && stats[:avg] ? stats[:avg].to_f.round(2) : nil
+        run.check_pass_rate = stats && stats[:resolved] > 0 ? (stats[:passed].to_f / stats[:resolved]).round(2) : nil
+
+        run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, resolved, passed)|
+          if scored.to_i > 0
+            {name: name, avg: avg.to_f.round(1)}
+          elsif resolved.to_i > 0
+            {name: name, kind: "check", pass_rate: (passed.to_i.to_f / resolved.to_i).round(2)}
+          end
+        end
+      end
+      runs
+    end
+
     # A scoring-only run grades a pre-existing column on the dataset instead of
     # generating new outputs. No prompt is attached; the response text is read
     # from row[output_column]; no LLM generation happens.
@@ -126,7 +180,15 @@ module CompletionKit
         end
     end
 
+    def response_count
+      return @response_count if defined?(@response_count)
+
+      responses.size
+    end
+
     def avg_score
+      return @avg_score if defined?(@avg_score)
+
       scores = reviews_for_summary.map(&:ai_score).compact.map(&:to_f)
       return nil if scores.empty?
 
@@ -134,6 +196,8 @@ module CompletionKit
     end
 
     def metric_averages
+      return @metric_averages if defined?(@metric_averages)
+
       reviews_for_summary.group_by(&:metric_name).filter_map do |name, reviews|
         scored = reviews.select { |r| r.ai_score.present? }
         if scored.any?
@@ -150,6 +214,8 @@ module CompletionKit
     end
 
     def check_pass_rate
+      return @check_pass_rate if defined?(@check_pass_rate)
+
       resolved = reviews_for_summary.reject { |r| r.passed.nil? }
       return nil if resolved.empty?
 
@@ -307,6 +373,45 @@ module CompletionKit
       true
     end
 
+    def rerun!
+      new_run = Run.create!(
+        prompt_id: prompt_id,
+        dataset_id: dataset_id,
+        judge_model: judge_model,
+        temperature: temperature,
+        output_column: output_column,
+        expected_column: expected_column,
+        tag_names: tag_names,
+        status: "pending"
+      )
+      new_run.replace_metrics!(metric_ids)
+      new_run
+    end
+
+    def retry_failures!(only: nil)
+      scope = responses.where(status: "failed")
+      scope = scope.where(id: only) if only.present?
+
+      transaction do
+        failed_response_ids = scope.pluck(:id)
+        Review.where(response_id: failed_response_ids, status: "failed").update_all(
+          status: "pending",
+          attempts: 0,
+          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+          ai_score: nil, passed: nil, ai_feedback: nil
+        )
+        scope.update_all(
+          status: "pending",
+          attempts: 0,
+          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+          response_text: nil
+        )
+        update!(status: "running")
+        failed_response_ids.each { |rid| GenerateRowJob.perform_later(id, rid) }
+      end
+      self
+    end
+
     def progress_snapshot
       generated_done = responses.where(status: "succeeded").count
       generated_failed = responses.where(status: "failed").count

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -24,6 +24,7 @@ module CompletionKit
               output_column: {type: "string", description: "Dataset column to grade when prompt_id is omitted; defaults to \"actual_output\"."},
               expected_column: {type: "string", description: "Dataset column holding each row's answer key / ground truth, graded by checks with compare_to \"expected\" and passed to the judge; defaults to \"expected_output\"."},
               metric_ids: {type: "array", items: {type: "integer"}},
+              metric_group_id: {type: "integer", description: "Attach the metrics belonging to this metric group (its current metric_ids). Ignored when metric_ids is also given."},
               tag_names: {type: "array", items: {type: "string"}}
             },
             required: ["name"]
@@ -40,6 +41,7 @@ module CompletionKit
               output_column: {type: "string"},
               expected_column: {type: "string"},
               metric_ids: {type: "array", items: {type: "integer"}},
+              metric_group_id: {type: "integer", description: "Replace the run's metrics with those belonging to this metric group. Ignored when metric_ids is also given."},
               tag_names: {type: "array", items: {type: "string"}}
             },
             required: ["id"]
@@ -55,6 +57,21 @@ module CompletionKit
           description: "Start a run. Required for every run, including score-only runs (no prompt): generates responses with the prompt when there is one, otherwise copies the graded dataset column and grades it.",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :generate
+        },
+        "runs_regrade" => {
+          description: "Re-grade a run's existing responses with its currently attached metrics, without regenerating. Use after attaching or editing metrics on an already-generated run.",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :regrade
+        },
+        "runs_rerun" => {
+          description: "Create and start a fresh copy of a run with the same prompt, dataset, metrics, and settings. Use when the judge changed and you want a clean run instead of mixing versions.",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :rerun
+        },
+        "runs_retry_failures" => {
+          description: "Re-run only the failed responses of a run, optionally limited to specific response ids via \"only\".",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}, only: {type: "array", items: {type: "integer"}}}, required: ["id"]},
+          handler: :retry_failures
         }
       }.freeze
 
@@ -63,15 +80,15 @@ module CompletionKit
       end
 
       def self.get(args)
-        text_result(Run.find(args["id"]).as_json)
+        text_result(run_payload(Run.find(args["id"])))
       end
 
       def self.create(args)
         run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column", "expected_column"))
         if run.save
-          run.replace_metrics!(args["metric_ids"])
+          run.replace_metrics!(resolve_metric_ids(args))
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
-          text_result(run.reload.as_json)
+          text_result(run_payload(run.reload))
         else
           error_result(run.errors.full_messages.join(", "))
         end
@@ -79,10 +96,10 @@ module CompletionKit
 
       def self.update(args)
         run = Run.find(args["id"])
-        if run.update(args.except("id", "metric_ids", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column"))
-          run.replace_metrics!(args["metric_ids"]) if args.key?("metric_ids")
+        if run.update(args.except("id", "metric_ids", "metric_group_id", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column"))
+          run.replace_metrics!(resolve_metric_ids(args)) if args.key?("metric_ids") || args["metric_group_id"].present?
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
-          text_result(run.reload.as_json)
+          text_result(run_payload(run.reload))
         else
           error_result(run.errors.full_messages.join(", "))
         end
@@ -96,11 +113,53 @@ module CompletionKit
       def self.generate(args)
         run = Run.find(args["id"])
         if run.start!
-          text_result(run.reload.as_json)
+          text_result(run_payload(run.reload))
         else
           text_result(run.failure_summary || run.errors.full_messages.to_sentence)
         end
       end
+
+      def self.regrade(args)
+        run = Run.find(args["id"])
+        if run.regrade!
+          text_result(run_payload(run.reload))
+        else
+          error_result("Nothing to re-grade. The run has no succeeded responses or no metrics attached.")
+        end
+      end
+
+      def self.rerun(args)
+        new_run = Run.find(args["id"]).rerun!
+        if new_run.start!
+          text_result(run_payload(new_run.reload))
+        else
+          error_result(new_run.failure_summary || "Could not start the new run.")
+        end
+      end
+
+      def self.retry_failures(args)
+        run = Run.find(args["id"])
+        if run.stale_review_summary.any?
+          return error_result("Judge has changed since this run executed. Retry would mix versions in the same run; use runs_rerun instead.")
+        end
+
+        run.retry_failures!(only: args["only"])
+        text_result(run_payload(run.reload))
+      end
+
+      def self.resolve_metric_ids(args)
+        return args["metric_ids"] if args.key?("metric_ids")
+        return MetricGroup.find(args["metric_group_id"]).metric_ids if args["metric_group_id"].present?
+
+        nil
+      end
+
+      def self.run_payload(run)
+        json = run.as_json
+        return json unless run.metric_ids.empty?
+
+        json.merge("warning" => "No metrics are attached, so this run judges nothing. Attach metric_ids or a metric_group_id before generating.")
+      end
     end
   end
 end

data/app/views/completion_kit/runs/_row.html.erb

@@ -27,7 +27,7 @@
   </td>
   <td>
     <span class="ck-runs-table__count">
-      <%= run.responses.size %><% if run.dataset %><span class="ck-runs-table__count-of">/<%= run.dataset.row_count %></span><% end %>
+      <%= run.response_count %><% if run.dataset %><span class="ck-runs-table__count-of">/<%= run.dataset.row_count %></span><% end %>
     </span>
   </td>
   <td>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.19"
+  VERSION = "0.28.21"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.19
+  version: 0.28.21
 platform: ruby
 authors:
 - Damien Bastin