completion-kit 0.17.1 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e35865036a3c24bd7cea9332a734af90d031b8c9d1b8d6b23b0694322fc248fe
-  data.tar.gz: 16202eef750b6b0e233456eedff06429d23634f325de5bde2a2ff8d9598108e0
+  metadata.gz: 84ae9459a26b612bb68a2f875a83274dc1bd711659b62d230aa7315fb3e7ce66
+  data.tar.gz: ebbe020a987228e1c1f5e2c0c1d6be4caa3c17cd7448141d69e331fc9a207eb3
 SHA512:
-  metadata.gz: a9112f5dcc7419ac0f6f6cc2f375b40bcd2d4d40e9d7fb4ca3b45899a51ff611234ef1ecfa6f64c4eecefbb43ac5e5bb4e2ab9edace0b531d1b3cf963adb2fdc
-  data.tar.gz: 9a3f51f1754b0d60475c7f5b2c72c6bc0f8cdd2bfa00240aca7d2c5afdac91acf105cd2fddd103b29f4d0c0da9c846ddc752f068f409a4b257f1e9f9016199d0
+  metadata.gz: 95ee6ba7cd0db74ea2e27629f3a8d3b83de56b1171f3f26419c6cce72fcdf9cf01dafefb7942bcd06519db2b1b304e1e108156e7b22c9b3a54230cb2b620ac50
+  data.tar.gz: f9883c98aa3e6e4ec4cfaf2dbe7f9f9c3261959de217b20813efd53551ecbe69a41d871d3fc9179771413a5f214c9ffd337ea67775d9d558c8d2ea21dc2e336a

data/app/controllers/completion_kit/agreements_controller.rb

@@ -2,6 +2,7 @@ module CompletionKit
   class AgreementsController < ApplicationController
     before_action :ensure_agreement_enabled
     before_action :set_scope
+    before_action :reject_check_metric, only: [:create]
 
     def create
       created_by = agreement_creator
@@ -60,6 +61,10 @@ module CompletionKit
       head :not_found unless CompletionKit.config.judge_agreement_enabled
     end
 
+    def reject_check_metric
+      head :unprocessable_entity if @metric.check?
+    end
+
     def set_scope
       @run = Run.find(params[:run_id])
       @response = @run.responses.find(params[:response_id])

data/app/controllers/completion_kit/api/v1/agreements_controller.rb

@@ -4,6 +4,7 @@ module CompletionKit
       class AgreementsController < BaseController
         before_action :ensure_agreement_enabled
         before_action :set_nested_scope, only: [:create]
+        before_action :reject_check_metric, only: [:create]
         before_action :load_agreement, only: [:destroy]
 
         def index
@@ -53,6 +54,10 @@ module CompletionKit
           not_found
         end
 
+        def reject_check_metric
+          render_error("Checks have nothing to calibrate", status: :unprocessable_entity) if @metric.check?
+        end
+
         def load_agreement
           @agreement = Agreement.find(params[:id])
         rescue ActiveRecord::RecordNotFound

data/app/controllers/completion_kit/api/v1/metrics_controller.rb

@@ -37,6 +37,11 @@ module CompletionKit
         end
 
         def suggest_variants
+          if @metric.check?
+            render_error("Checks are exact; no variants to suggest.", status: :unprocessable_entity)
+            return
+          end
+
           disagreement_count = Agreement.where(metric_id: @metric.id, verdict: "disagree").count
           if disagreement_count.zero?
             render_error("Mark at least one case as Disagree before asking the model to suggest a change.", status: :unprocessable_entity)
@@ -63,8 +68,10 @@ module CompletionKit
         end
 
         def metric_params
-          params.permit(:name, :instruction,
-            rubric_bands: [:stars, :description], tag_names: [])
+          params.permit(:name, :instruction, :metric_type,
+            rubric_bands: [:stars, :description],
+            check_config: %i[check_kind target target_path value pattern json_path expected min max case_sensitive multiline trim],
+            tag_names: [])
         end
       end
     end

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -62,7 +62,7 @@ module CompletionKit
             CompletionKit::Review.where(response_id: failed_response_ids, status: "failed").update_all(
               status: "pending", attempts: 0,
               error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-              ai_score: nil, ai_feedback: nil
+              ai_score: nil, passed: nil, ai_feedback: nil
             )
             scope.update_all(
               status: "pending", attempts: 0,

data/app/controllers/completion_kit/metrics_controller.rb

@@ -24,7 +24,9 @@ module CompletionKit
       metric = Metric.create!(
         name: starter.name,
         instruction: starter.instruction,
-        rubric_bands: starter.rubric_bands
+        rubric_bands: starter.rubric_bands,
+        metric_type: starter.metric_type || "llm_judge",
+        check_config: starter.check_config
       )
       redirect_to metric_path(metric), notice: "Added the \"#{starter.name}\" starter. Tweak any band before you run a judge against it."
     end
@@ -39,9 +41,14 @@ module CompletionKit
     def show
       @edit_draft = MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").order(created_at: :desc).first
       @suggestion_draft = MetricVersion.drafts.where(metric_id: @metric.id, source: "suggestion").order(created_at: :desc).first
-      @improve_disagreement_count = Agreement.where(metric_id: @metric.id, verdict: "disagree").count
       @versions = MetricVersion.where(metric_id: @metric.id).order(version_number: :desc).to_a
-      @guiding_examples = CompletionKit.config.judge_examples_from_reviews ? MetricAgreementExamples.judge_examples_for(@metric) : []
+      if @metric.check?
+        @improve_disagreement_count = 0
+        @guiding_examples = []
+      else
+        @improve_disagreement_count = Agreement.where(metric_id: @metric.id, verdict: "disagree").count
+        @guiding_examples = CompletionKit.config.judge_examples_from_reviews ? MetricAgreementExamples.judge_examples_for(@metric) : []
+      end
     end
 
     def new
@@ -52,7 +59,7 @@ module CompletionKit
       @suggestion_draft = MetricVersion.drafts.where(metric_id: @metric.id, source: "suggestion").order(created_at: :desc).first
       @edit_draft = MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").order(created_at: :desc).first
       @published_metric_version = MetricVersion.published.where(metric_id: @metric.id, current: true).first
-      @improve_disagreement_count = Agreement.where(metric_id: @metric.id, verdict: "disagree").count
+      @improve_disagreement_count = @metric.check? ? 0 : Agreement.where(metric_id: @metric.id, verdict: "disagree").count
 
       if @edit_draft
         @metric.instruction = @edit_draft.instruction
@@ -71,42 +78,16 @@ module CompletionKit
     end
 
     def update
-      judge_keys = %i[instruction rubric_bands]
-      meta_attrs = metric_params.except(*judge_keys)
-      proposed_instruction = metric_params[:instruction]
-      proposed_rubric = metric_params[:rubric_bands]
+      meta_attrs = metric_params.except(:instruction, :rubric_bands, :check_config)
 
       unless @metric.update(meta_attrs)
         return render(:edit, status: :unprocessable_entity)
       end
 
-      current_instruction = @metric.instruction.to_s
-      current_rubric = @metric.rubric_bands || []
-      normalized_proposed_rubric = normalize_rubric_bands_for_update(proposed_rubric)
-
-      instruction_changed = !proposed_instruction.nil? && proposed_instruction.to_s != current_instruction
-      rubric_changed = !normalized_proposed_rubric.nil? && normalized_proposed_rubric != current_rubric
-
-      unless instruction_changed || rubric_changed
-        return redirect_to(metric_path(@metric), notice: "Metric was successfully updated.")
-      end
-
-      new_instruction = instruction_changed ? proposed_instruction.to_s : current_instruction
-      new_rubric = rubric_changed ? normalized_proposed_rubric : current_rubric
-
-      if @metric.reviews.exists?
-        MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").destroy_all
-        draft = MetricVersion.create!(
-          metric: @metric, instruction: new_instruction, rubric_bands: new_rubric,
-          state: "draft", source: "edit", current: false
-        )
-        redirect_to edit_metric_path(@metric),
-                    notice: "Saved as draft #{draft.version_label}. Publish to make these changes the metric's live version."
+      if @metric.check?
+        update_check_definition
       else
-        @metric.update!(instruction: new_instruction, rubric_bands: new_rubric)
-        current_pub = MetricVersion.published.where(metric_id: @metric.id, current: true).first
-        current_pub&.update!(instruction: @metric.instruction, rubric_bands: @metric.rubric_bands)
-        redirect_to metric_path(@metric), notice: "Metric was successfully updated."
+        update_judge_definition
       end
     end
 
@@ -116,6 +97,11 @@ module CompletionKit
     end
 
     def suggest_variants
+      if @metric.check?
+        redirect_to metric_path(@metric), alert: "Checks are exact, so there is nothing to suggest."
+        return
+      end
+
       target = params[:back_to] == "edit" ? edit_metric_path(@metric) : metric_path(@metric)
       counts = Agreement.where(metric_id: @metric.id, verdict: %w[agree disagree]).group(:verdict).count
       if counts["disagree"].to_i.zero?
@@ -188,13 +174,88 @@ module CompletionKit
       head :not_found unless CompletionKit.config.judge_examples_from_reviews
     end
 
+    def update_judge_definition
+      proposed_instruction = metric_params[:instruction]
+      proposed_rubric = metric_params[:rubric_bands]
+      current_instruction = @metric.instruction.to_s
+      current_rubric = @metric.rubric_bands || []
+      normalized_proposed_rubric = normalize_rubric_bands_for_update(proposed_rubric)
+
+      instruction_changed = !proposed_instruction.nil? && proposed_instruction.to_s != current_instruction
+      rubric_changed = !normalized_proposed_rubric.nil? && normalized_proposed_rubric != current_rubric
+
+      unless instruction_changed || rubric_changed
+        return redirect_to(metric_path(@metric), notice: "Metric was successfully updated.")
+      end
+
+      new_instruction = instruction_changed ? proposed_instruction.to_s : current_instruction
+      new_rubric = rubric_changed ? normalized_proposed_rubric : current_rubric
+
+      if @metric.reviews.exists?
+        MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").destroy_all
+        draft = MetricVersion.create!(
+          metric: @metric, instruction: new_instruction, rubric_bands: new_rubric,
+          state: "draft", source: "edit", current: false
+        )
+        redirect_to edit_metric_path(@metric),
+                    notice: "Saved as draft #{draft.version_label}. Publish to make these changes the metric's live version."
+      else
+        @metric.update!(instruction: new_instruction, rubric_bands: new_rubric)
+        current_pub = MetricVersion.published.where(metric_id: @metric.id, current: true).first
+        current_pub&.update!(instruction: @metric.instruction, rubric_bands: @metric.rubric_bands)
+        redirect_to metric_path(@metric), notice: "Metric was successfully updated."
+      end
+    end
+
+    def update_check_definition
+      raw = metric_params[:check_config]
+      proposed = raw.nil? ? nil : normalize_check_config(raw)
+
+      unless !proposed.nil? && proposed != @metric.check_config
+        return redirect_to(metric_path(@metric), notice: "Metric was successfully updated.")
+      end
+
+      if @metric.reviews.exists?
+        MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").destroy_all
+        draft = MetricVersion.create!(
+          metric: @metric, metric_type: "check", check_config: proposed,
+          state: "draft", source: "edit", current: false
+        )
+        redirect_to edit_metric_path(@metric),
+                    notice: "Saved as draft #{draft.version_label}. Publish to make these changes the metric's live version."
+      else
+        @metric.update!(check_config: proposed)
+        current_pub = MetricVersion.published.where(metric_id: @metric.id, current: true).first
+        current_pub&.update!(metric_type: "check", check_config: proposed)
+        redirect_to metric_path(@metric), notice: "Metric was successfully updated."
+      end
+    end
+
     def set_metric
       @metric = Metric.find(params[:id])
     end
 
     def metric_params
-      params.require(:metric).permit(:name, :instruction,
-        rubric_bands: [:stars, :description], tag_names: [])
+      permitted = params.require(:metric).permit(:name, :instruction, :metric_type,
+        rubric_bands: [:stars, :description],
+        check_config: %i[check_kind target target_path value pattern json_path expected min max case_sensitive multiline trim],
+        tag_names: [])
+      permitted[:check_config] = normalize_check_config(permitted[:check_config]) if permitted.key?(:check_config)
+      permitted
+    end
+
+    def normalize_check_config(config)
+      hash = config.to_unsafe_h.stringify_keys
+      %w[min max].each { |key| hash[key] = hash[key].to_i if hash[key].present? }
+      %w[case_sensitive multiline trim].each { |key| hash[key] = ActiveModel::Type::Boolean.new.cast(hash[key]) if hash.key?(key) }
+      hash["expected"] = coerce_scalar(hash["expected"]) if hash["expected"].present?
+      hash.reject { |_, value| value.nil? || value == "" }
+    end
+
+    def coerce_scalar(value)
+      JSON.parse(value)
+    rescue JSON::ParserError
+      value
     end
 
     def normalize_rubric_bands_for_update(bands)

data/app/controllers/completion_kit/runs_controller.rb

@@ -164,7 +164,7 @@ module CompletionKit
           status: "pending",
           attempts: 0,
           error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-          ai_score: nil, ai_feedback: nil
+          ai_score: nil, passed: nil, ai_feedback: nil
         )
         scope.update_all(
           status: "pending",

data/app/jobs/completion_kit/check_review_job.rb

@@ -0,0 +1,66 @@
+module CompletionKit
+  class CheckReviewJob < ApplicationJob
+    queue_as :default
+
+    rescue_from(StandardError) do |error|
+      Rails.error.report(error, handled: true, context: { job: self.class.name, run_id: @run_id, response_id: @response_id, metric_id: @metric_id })
+      record_terminal_failure!(error)
+      enqueue_completion_check
+    end
+
+    def perform(response_id, metric_id, run_id = nil)
+      @response_id = response_id
+      @metric_id = metric_id
+      @run_id = run_id
+
+      response = Response.find(response_id)
+      metric = Metric.find(metric_id)
+      result = evaluate(response, metric.check_config || {})
+
+      review = response.reviews.find_or_initialize_by(metric_id: metric.id)
+      current_metric_version = MetricVersion.ensure_current_for(metric)
+      review.assign_attributes(
+        metric_name: metric.name,
+        metric_version_id: current_metric_version.id,
+        status: "succeeded",
+        passed: result.passed,
+        ai_score: nil,
+        ai_feedback: result.detail,
+        error_provider: nil, error_class: nil, error_status: nil, error_message: nil
+      )
+      review.save!
+
+      enqueue_completion_check
+    end
+
+    private
+
+    def evaluate(response, config)
+      target_value = Checks::TargetResolver.call(response, config)
+      if target_value.equal?(Checks::TargetResolver::UNRESOLVED)
+        return Checks::Result.new(passed: false, detail: "could not resolve target")
+      end
+
+      Checks::Registry.fetch(config["check_kind"]).call(target_value, config)
+    end
+
+    def record_terminal_failure!(error)
+      response = Response.find_by(id: @response_id)
+      return unless response
+
+      review = response.reviews.find_or_initialize_by(metric_id: @metric_id)
+      review.assign_attributes(
+        metric_name: review.metric_name || Metric.find_by(id: @metric_id)&.name || "(deleted metric)",
+        status: "failed",
+        error_class: error.class.name,
+        error_message: error.message.to_s.truncate(2000)
+      )
+      review.save!(validate: false)
+    end
+
+    def enqueue_completion_check
+      response = Response.find_by(id: @response_id)
+      RunCompletionCheckJob.perform_later(response.run_id) if response
+    end
+  end
+end

data/app/jobs/completion_kit/generate_row_job.rb

@@ -61,11 +61,14 @@ module CompletionKit
         error_provider: nil, error_class: nil, error_status: nil, error_message: nil
       )
 
-      if run.judge_configured?
-        run.metrics.each do |metric|
+      if run.llm_judge_configured?
+        run.llm_metrics.each do |metric|
           JudgeReviewJob.perform_later(response.id, metric.id, run.id)
         end
       end
+      run.check_metrics.each do |metric|
+        CheckReviewJob.perform_later(response.id, metric.id, run.id)
+      end
 
       enqueue_completion_check
     end

data/app/jobs/completion_kit/metric_suggestion_job.rb

@@ -15,6 +15,7 @@ module CompletionKit
     def perform(metric_id)
       @metric = Metric.find_by(id: metric_id)
       return unless @metric
+      return if @metric.check?
 
       MetricVersion.drafts.where(metric_id: @metric.id, source: "suggestion").destroy_all
 

data/app/models/completion_kit/metric.rb

@@ -16,14 +16,20 @@ module CompletionKit
     has_many :reviews, dependent: :nullify
     has_many :dashboard_dismissals, as: :dismissable, dependent: :destroy
 
+    METRIC_TYPES = %w[llm_judge check].freeze
+
     serialize :rubric_bands, coder: JSON
+    serialize :check_config, coder: JSON
 
     validates :name, presence: true
     validates :key, tenant_scoped_uniqueness: { allow_nil: true }
+    validates :metric_type, inclusion: { in: METRIC_TYPES }
+    validate :validate_check_config, if: :check?
+    validate :metric_type_immutable_once_in_use, on: :update
 
     before_validation :generate_key
-    before_validation :normalize_rubric_bands
-    before_validation :set_defaults
+    before_validation :normalize_rubric_bands, if: :llm_judge?
+    before_validation :set_defaults, if: :llm_judge?
 
     def self.default_rubric_bands
       DEFAULT_RUBRIC_BANDS.map(&:dup)
@@ -74,13 +80,29 @@ module CompletionKit
       self.class.rubric_text_for(rubric_bands_for_form)
     end
 
+    def check?
+      metric_type == "check"
+    end
+
+    def llm_judge?
+      !check?
+    end
+
+    def in_use?
+      RunMetric.exists?(metric_id: id) || reviews.exists? || metric_versions.exists?
+    end
+
     def as_json(options = {})
-      {
-        id: id, name: name, key: key, instruction: instruction,
-        rubric_bands: rubric_bands,
+      base = {
+        id: id, name: name, key: key, metric_type: metric_type,
         created_at: created_at, updated_at: updated_at,
         tags: tags.as_json
       }
+      if check?
+        base.merge(check_config: check_config)
+      else
+        base.merge(instruction: instruction, rubric_bands: rubric_bands)
+      end
     end
 
     private
@@ -89,6 +111,70 @@ module CompletionKit
       self.key ||= name.parameterize if name.present?
     end
 
+    def metric_type_immutable_once_in_use
+      return unless metric_type_changed?
+      return unless in_use?
+
+      errors.add(:metric_type, "cannot change once the metric has been used in a run")
+    end
+
+    def validate_check_config
+      config = check_config
+      unless config.is_a?(Hash)
+        errors.add(:check_config, "must be a configuration object")
+        return
+      end
+
+      kind = config["check_kind"]
+      unless CompletionKit::Checks::Registry.kinds.include?(kind)
+        errors.add(:check_config, "check_kind must be one of #{CompletionKit::Checks::Registry.kinds.join(", ")}")
+        return
+      end
+
+      validate_check_target(config)
+      validate_check_required_keys(config, kind)
+      validate_check_kind_rules(config, kind)
+    end
+
+    def validate_check_target(config)
+      target = config["target"].presence || "response_text"
+      unless CompletionKit::Checks::TargetResolver::TARGETS.include?(target)
+        errors.add(:check_config, "target must be one of #{CompletionKit::Checks::TargetResolver::TARGETS.join(", ")}")
+      end
+      if target == "json_path" && config["target_path"].to_s.strip.empty?
+        errors.add(:check_config, "target_path is required when target is json_path")
+      end
+    end
+
+    def validate_check_required_keys(config, kind)
+      CompletionKit::Checks::Registry.required_keys.fetch(kind).each do |required_key|
+        if required_key == "expected"
+          errors.add(:check_config, "expected is required") unless config.key?("expected")
+        elsif config[required_key].to_s.strip.empty?
+          errors.add(:check_config, "#{required_key} is required")
+        end
+      end
+    end
+
+    def validate_check_kind_rules(config, kind)
+      case kind
+      when "regex"
+        begin
+          Regexp.new(config["pattern"].to_s)
+        rescue RegexpError
+          errors.add(:check_config, "pattern is not a valid regular expression")
+        end
+      when "length_bounds"
+        min = config["min"]
+        max = config["max"]
+        if min.nil? && max.nil?
+          errors.add(:check_config, "length_bounds requires at least one of min or max")
+        elsif min && max && min.to_i > max.to_i
+          errors.add(:check_config, "min must be less than or equal to max")
+        end
+      end
+    end
+
     def set_defaults
       self.rubric_bands = self.class.default_rubric_bands if rubric_bands.blank?
     end

data/app/models/completion_kit/metric_version.rb

@@ -6,6 +6,7 @@ module CompletionKit
     has_many :agreements, dependent: :destroy
 
     serialize :rubric_bands, coder: JSON
+    serialize :check_config, coder: JSON
     serialize :validation_summary, coder: JSON
 
     before_validation :assign_version_number, on: :create
@@ -23,12 +24,22 @@ module CompletionKit
         metric: metric,
         instruction: metric.instruction,
         rubric_bands: metric.rubric_bands,
+        metric_type: metric.metric_type,
+        check_config: metric.check_config,
         current: true,
         state: "published",
         published_at: Time.current
       )
     end
 
+    def check?
+      metric_type == "check"
+    end
+
+    def llm_judge?
+      !check?
+    end
+
     def draft?
       state == "draft"
     end
@@ -43,6 +54,7 @@ module CompletionKit
 
     def change_summary_against(previous)
       return nil if previous.nil?
+      return check_change_summary_against(previous) if check?
 
       instruction_changed = previous.instruction.to_s.strip != instruction.to_s.strip
       rubric_changes = rubric_band_change_count(previous)
@@ -75,31 +87,46 @@ module CompletionKit
         self.class.where(metric_id: metric_id).where.not(id: id).update_all(current: false)
         reload
         update!(state: "published", current: true, published_at: published_at || Time.current)
-        metric.update_columns(
-          instruction: instruction,
-          rubric_bands: Array(rubric_bands).to_json
-        )
+        if check?
+          metric.update_columns(metric_type: "check", check_config: check_config)
+        else
+          metric.update_columns(
+            metric_type: "llm_judge",
+            instruction: instruction,
+            rubric_bands: Array(rubric_bands).to_json
+          )
+        end
       end
       self
     end
 
     def as_json(options = {})
-      {
+      base = {
         id: id,
         metric_id: metric_id,
         version_number: version_number,
-        instruction: instruction,
-        rubric_bands: rubric_bands,
+        metric_type: metric_type,
         current: current,
         state: state,
         source: source,
         published_at: published_at,
         created_at: created_at
       }
+      if check?
+        base.merge(check_config: check_config)
+      else
+        base.merge(instruction: instruction, rubric_bands: rubric_bands)
+      end
     end
 
     private
 
+    def check_change_summary_against(previous)
+      return nil if check_config == previous.check_config
+
+      { magnitude: :minor, label: "Check configuration changes" }
+    end
+
     def rubric_band_change_count(previous)
       prev = Metric.normalize_rubric_bands(previous.rubric_bands)
       curr = Metric.normalize_rubric_bands(rubric_bands)

data/app/models/completion_kit/response.rb

@@ -8,7 +8,7 @@ module CompletionKit
 
     delegate :prompt, to: :run
 
-    validates :response_text, presence: true, if: :succeeded?
+    validates :response_text, presence: true, if: :requires_response_text?
 
     before_validation :set_default_status, on: :create
 
@@ -34,7 +34,19 @@ module CompletionKit
     end
 
     def reviewed?
-      reviews.any? { |r| r.ai_score.present? }
+      reviews.any? { |r| r.ai_score.present? || !r.passed.nil? }
+    end
+
+    def checks_total
+      reviews.count { |r| !r.passed.nil? }
+    end
+
+    def checks_passed
+      reviews.count { |r| r.passed == true }
+    end
+
+    def checks_failed
+      reviews.count { |r| r.passed == false }
     end
 
     def fully_reviewed?
@@ -46,6 +58,10 @@ module CompletionKit
 
     private
 
+    def requires_response_text?
+      succeeded? && !run&.judge_only_input_data_checks?
+    end
+
     def broadcast_row_update
       run.broadcast_response_update(self)
     end

data/app/models/completion_kit/review.rb

@@ -16,6 +16,10 @@ module CompletionKit
     after_save_commit :broadcast_parent_row_update, unless: :destroyed?
     after_save_commit :broadcast_run_progress, if: :should_broadcast_progress?
 
+    def check?
+      metric_version&.metric_type == "check"
+    end
+
     def stale_against_current_judge?
       return false unless metric_id && metric_version_id
       current_id = MetricVersion.current.where(metric_id: metric_id).limit(1).pick(:id)
@@ -27,7 +31,7 @@ module CompletionKit
       {
         id: id, response_id: response_id, metric_id: metric_id,
         metric_version_id: metric_version_id,
-        metric_name: metric_name, ai_score: ai_score,
+        metric_name: metric_name, ai_score: ai_score, passed: passed,
         ai_feedback: ai_feedback, status: status, attempts: attempts,
         error: error_payload
       }