completion-kit 0.28.19 → 0.28.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e30a6a27a48e1a23c1947477068eec74f8ff2ff869bab4592f68c612a728dc76
-  data.tar.gz: c1cdf115062d9ddd903296574420bd75128995460ce26e64e44681cdd6655ed9
+  metadata.gz: 6a05f4f94c9f7663a486ca16dcf6d7a60215b3c8017be6576760dd929a6df49d
+  data.tar.gz: d5e26b475bb73c1402d68856f833d91241a6feac0402fdfd0e1ab605a658578f
 SHA512:
-  metadata.gz: ef4c4ddecf48be58b43ff9d02b1e2743d75b28eabdccf13bc35db32c0225b4a880318a4e35faa7ba8c235a29f77c934ad323f5099a355edd0d29ba18f8293566
-  data.tar.gz: cd6b8331a0ad697ca16e71c7d6eb623d70e050c1fe50332672bd797c01a42509cef2c2ba6738bbebf3dfdff6d1feb07a29ea2f205408390259ea8d809622103a
+  metadata.gz: b0153fca04674849516b94f8d12021be77dedec325e8e716e40037699194002102ac28fb80ffccc2f01c1cd08a7ddca498055f352ac6bdb5e00721e3c2a39732
+  data.tar.gz: 6b11f2fbd4e155b191f27c561624b3e2ba7ee0358ffc2f97e6430749ee251c6a0ad628696d9b1f48c5827ebce91001d583ab7a2d07c796dd2c28b695db16a7aa

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -54,40 +54,12 @@ module CompletionKit
             return render_error("Judge has changed since this run executed. Retry would mix versions in the same run; use POST /api/v1/runs/:id/rerun instead.", status: :conflict)
           end
 
-          scope = @run.responses.where(status: "failed")
-          scope = scope.where(id: params[:only]) if params[:only].present?
-
-          ActiveRecord::Base.transaction do
-            failed_response_ids = scope.pluck(:id)
-            CompletionKit::Review.where(response_id: failed_response_ids, status: "failed").update_all(
-              status: "pending", attempts: 0,
-              error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-              ai_score: nil, passed: nil, ai_feedback: nil
-            )
-            scope.update_all(
-              status: "pending", attempts: 0,
-              error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-              response_text: nil
-            )
-            @run.update!(status: "running")
-            failed_response_ids.each { |rid| CompletionKit::GenerateRowJob.perform_later(@run.id, rid) }
-          end
-
+          @run.retry_failures!(only: params[:only])
           render json: @run.reload, status: :accepted
         end
 
         def rerun
-          new_run = Run.create!(
-            prompt_id: @run.prompt_id,
-            dataset_id: @run.dataset_id,
-            judge_model: @run.judge_model,
-            temperature: @run.temperature,
-            output_column: @run.output_column,
-            expected_column: @run.expected_column,
-            tag_names: @run.tag_names,
-            status: "pending"
-          )
-          new_run.replace_metrics!(@run.metric_ids)
+          new_run = @run.rerun!
           if new_run.start!
             render json: new_run.reload, status: :accepted
           else

data/app/controllers/completion_kit/runs_controller.rb

@@ -110,17 +110,7 @@ module CompletionKit
     end
 
     def rerun
-      new_run = Run.create!(
-        prompt_id: @run.prompt_id,
-        dataset_id: @run.dataset_id,
-        judge_model: @run.judge_model,
-        temperature: @run.temperature,
-        output_column: @run.output_column,
-        expected_column: @run.expected_column,
-        tag_names: @run.tag_names,
-        status: "pending"
-      )
-      new_run.replace_metrics!(@run.metric_ids)
+      new_run = @run.rerun!
       if new_run.start!
         redirect_to run_path(new_run), notice: "Re-running with the same configuration."
       else
@@ -162,27 +152,7 @@ module CompletionKit
         return
       end
 
-      scope = @run.responses.where(status: "failed")
-      scope = scope.where(id: params[:only]) if params[:only].present?
-
-      ActiveRecord::Base.transaction do
-        failed_response_ids = scope.pluck(:id)
-        Review.where(response_id: failed_response_ids, status: "failed").update_all(
-          status: "pending",
-          attempts: 0,
-          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-          ai_score: nil, passed: nil, ai_feedback: nil
-        )
-        scope.update_all(
-          status: "pending",
-          attempts: 0,
-          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-          response_text: nil
-        )
-        @run.update!(status: "running")
-        failed_response_ids.each { |rid| GenerateRowJob.perform_later(@run.id, rid) }
-      end
-
+      @run.retry_failures!(only: params[:only])
       @run.broadcast_ui
       redirect_to run_path(@run)
     end

data/app/models/completion_kit/run.rb

@@ -307,6 +307,45 @@ module CompletionKit
       true
     end
 
+    def rerun!
+      new_run = Run.create!(
+        prompt_id: prompt_id,
+        dataset_id: dataset_id,
+        judge_model: judge_model,
+        temperature: temperature,
+        output_column: output_column,
+        expected_column: expected_column,
+        tag_names: tag_names,
+        status: "pending"
+      )
+      new_run.replace_metrics!(metric_ids)
+      new_run
+    end
+
+    def retry_failures!(only: nil)
+      scope = responses.where(status: "failed")
+      scope = scope.where(id: only) if only.present?
+
+      transaction do
+        failed_response_ids = scope.pluck(:id)
+        Review.where(response_id: failed_response_ids, status: "failed").update_all(
+          status: "pending",
+          attempts: 0,
+          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+          ai_score: nil, passed: nil, ai_feedback: nil
+        )
+        scope.update_all(
+          status: "pending",
+          attempts: 0,
+          error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
+          response_text: nil
+        )
+        update!(status: "running")
+        failed_response_ids.each { |rid| GenerateRowJob.perform_later(id, rid) }
+      end
+      self
+    end
+
     def progress_snapshot
       generated_done = responses.where(status: "succeeded").count
       generated_failed = responses.where(status: "failed").count

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -24,6 +24,7 @@ module CompletionKit
               output_column: {type: "string", description: "Dataset column to grade when prompt_id is omitted; defaults to \"actual_output\"."},
               expected_column: {type: "string", description: "Dataset column holding each row's answer key / ground truth, graded by checks with compare_to \"expected\" and passed to the judge; defaults to \"expected_output\"."},
               metric_ids: {type: "array", items: {type: "integer"}},
+              metric_group_id: {type: "integer", description: "Attach the metrics belonging to this metric group (its current metric_ids). Ignored when metric_ids is also given."},
               tag_names: {type: "array", items: {type: "string"}}
             },
             required: ["name"]
@@ -40,6 +41,7 @@ module CompletionKit
               output_column: {type: "string"},
               expected_column: {type: "string"},
               metric_ids: {type: "array", items: {type: "integer"}},
+              metric_group_id: {type: "integer", description: "Replace the run's metrics with those belonging to this metric group. Ignored when metric_ids is also given."},
               tag_names: {type: "array", items: {type: "string"}}
             },
             required: ["id"]
@@ -55,6 +57,21 @@ module CompletionKit
           description: "Start a run. Required for every run, including score-only runs (no prompt): generates responses with the prompt when there is one, otherwise copies the graded dataset column and grades it.",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :generate
+        },
+        "runs_regrade" => {
+          description: "Re-grade a run's existing responses with its currently attached metrics, without regenerating. Use after attaching or editing metrics on an already-generated run.",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :regrade
+        },
+        "runs_rerun" => {
+          description: "Create and start a fresh copy of a run with the same prompt, dataset, metrics, and settings. Use when the judge changed and you want a clean run instead of mixing versions.",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :rerun
+        },
+        "runs_retry_failures" => {
+          description: "Re-run only the failed responses of a run, optionally limited to specific response ids via \"only\".",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}, only: {type: "array", items: {type: "integer"}}}, required: ["id"]},
+          handler: :retry_failures
         }
       }.freeze
 
@@ -63,15 +80,15 @@ module CompletionKit
       end
 
       def self.get(args)
-        text_result(Run.find(args["id"]).as_json)
+        text_result(run_payload(Run.find(args["id"])))
       end
 
       def self.create(args)
         run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column", "expected_column"))
         if run.save
-          run.replace_metrics!(args["metric_ids"])
+          run.replace_metrics!(resolve_metric_ids(args))
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
-          text_result(run.reload.as_json)
+          text_result(run_payload(run.reload))
         else
           error_result(run.errors.full_messages.join(", "))
         end
@@ -79,10 +96,10 @@ module CompletionKit
 
       def self.update(args)
         run = Run.find(args["id"])
-        if run.update(args.except("id", "metric_ids", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column"))
-          run.replace_metrics!(args["metric_ids"]) if args.key?("metric_ids")
+        if run.update(args.except("id", "metric_ids", "metric_group_id", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column"))
+          run.replace_metrics!(resolve_metric_ids(args)) if args.key?("metric_ids") || args["metric_group_id"].present?
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
-          text_result(run.reload.as_json)
+          text_result(run_payload(run.reload))
         else
           error_result(run.errors.full_messages.join(", "))
         end
@@ -96,11 +113,53 @@ module CompletionKit
       def self.generate(args)
         run = Run.find(args["id"])
         if run.start!
-          text_result(run.reload.as_json)
+          text_result(run_payload(run.reload))
         else
           text_result(run.failure_summary || run.errors.full_messages.to_sentence)
         end
       end
+
+      def self.regrade(args)
+        run = Run.find(args["id"])
+        if run.regrade!
+          text_result(run_payload(run.reload))
+        else
+          error_result("Nothing to re-grade. The run has no succeeded responses or no metrics attached.")
+        end
+      end
+
+      def self.rerun(args)
+        new_run = Run.find(args["id"]).rerun!
+        if new_run.start!
+          text_result(run_payload(new_run.reload))
+        else
+          error_result(new_run.failure_summary || "Could not start the new run.")
+        end
+      end
+
+      def self.retry_failures(args)
+        run = Run.find(args["id"])
+        if run.stale_review_summary.any?
+          return error_result("Judge has changed since this run executed. Retry would mix versions in the same run; use runs_rerun instead.")
+        end
+
+        run.retry_failures!(only: args["only"])
+        text_result(run_payload(run.reload))
+      end
+
+      def self.resolve_metric_ids(args)
+        return args["metric_ids"] if args.key?("metric_ids")
+        return MetricGroup.find(args["metric_group_id"]).metric_ids if args["metric_group_id"].present?
+
+        nil
+      end
+
+      def self.run_payload(run)
+        json = run.as_json
+        return json unless run.metric_ids.empty?
+
+        json.merge("warning" => "No metrics are attached, so this run judges nothing. Attach metric_ids or a metric_group_id before generating.")
+      end
     end
   end
 end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.19"
+  VERSION = "0.28.20"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.19
+  version: 0.28.20
 platform: ruby
 authors:
 - Damien Bastin