completion-kit 0.28.46 → 0.29.0

data/app/jobs/completion_kit/check_review_job.rb

@@ -25,6 +25,7 @@ module CompletionKit
         status: "succeeded",
         passed: result.passed,
         ai_score: nil,
+        check_score: result.score,
         ai_feedback: result.detail,
         error_provider: nil, error_class: nil, error_status: nil, error_message: nil
       )
@@ -36,23 +37,43 @@ module CompletionKit
     private
 
     def evaluate(response, config)
-      target_value = Checks::TargetResolver.call(response, config)
+      kind = config["check_kind"]
+      target_value = resolve_target(response, config, kind)
       if target_value.equal?(Checks::TargetResolver::UNRESOLVED)
-        return Checks::Result.new(passed: false, detail: "could not resolve target")
+        return ungraded(kind, "could not resolve target")
       end
 
-      kind = config["check_kind"]
       if config["compare_to"] == "expected" && Checks::Registry.compares_value?(kind)
-        expected_value = Checks::ExpectedResolver.call(response, config)
+        expected_value = resolve_expected(response, config, kind)
         if expected_value.equal?(Checks::ExpectedResolver::UNRESOLVED)
-          return Checks::Result.new(passed: false, detail: "no expected value for this row")
+          return ungraded(kind, "no expected value for this row")
         end
-        config = config.merge("value" => expected_value)
+        config = config.merge(Checks::Registry.expected_key(kind) => expected_value)
       end
 
       Checks::Registry.fetch(kind).call(target_value, config)
     end
 
+    def ungraded(kind, detail)
+      Checks::Result.new(passed: false, detail: detail, score: Checks::Registry.scores?(kind) ? 0.0 : nil)
+    end
+
+    def resolve_target(response, config, kind)
+      if Checks::Registry.raw_target?(kind)
+        Checks::TargetResolver.call_value(response, config)
+      else
+        Checks::TargetResolver.call(response, config)
+      end
+    end
+
+    def resolve_expected(response, config, kind)
+      if Checks::Registry.raw_expected?(kind)
+        Checks::ExpectedResolver.call_value(response, config)
+      else
+        Checks::ExpectedResolver.call(response, config)
+      end
+    end
+
     def record_terminal_failure!(error)
       response = Response.find_by(id: @response_id)
       return unless response

data/app/models/completion_kit/metric.rb

@@ -30,6 +30,7 @@ module CompletionKit
     validate :metric_type_immutable_once_in_use, on: :update
 
     before_validation :generate_key
+    before_validation :default_check_target, if: :check?
     before_validation :normalize_rubric_bands, if: :llm_judge?
     before_validation :set_defaults, if: :llm_judge?
 
@@ -94,6 +95,10 @@ module CompletionKit
       RunMetric.exists?(metric_id: id) || reviews.exists? || metric_versions.exists?
     end
 
+    def check_error_keys
+      errors.objects.select { |error| error.attribute == :check_config }.flat_map { |error| Array(error.options[:check_keys]) }.uniq
+    end
+
     def as_json(options = {})
       base = {
         id: id, name: name, key: key, metric_type: metric_type,
@@ -129,67 +134,137 @@ module CompletionKit
 
       kind = config["check_kind"]
       unless CompletionKit::Checks::Registry.kinds.include?(kind)
-        errors.add(:check_config, "check_kind must be one of #{CompletionKit::Checks::Registry.kinds.join(", ")}")
+        add_check_error("needs #{check_label("check_kind")} to be one of #{CompletionKit::Checks::Registry.kinds.join(", ")}", "check_kind")
         return
       end
 
       validate_check_target(config)
       validate_check_compare_to(config, kind)
       validate_check_required_keys(config, kind)
-      validate_check_kind_rules(config, kind)
+      if refused_list_value?(config, kind)
+        add_check_error("needs #{check_label("value", kind)} to be a single value, since only list_overlap takes a list", "value")
+      else
+        validate_check_kind_rules(config, kind)
+      end
+    end
+
+    def refused_list_value?(config, kind)
+      config["value"].is_a?(Array) && kind != "list_overlap" && will_save_change_to_check_config?
     end
 
     def validate_check_compare_to(config, kind)
       return unless config.key?("compare_to")
 
       unless %w[constant expected].include?(config["compare_to"])
-        errors.add(:check_config, "compare_to must be constant or expected")
+        add_check_error("needs #{check_label("compare_to")} to be constant or expected", "compare_to")
         return
       end
       if config["compare_to"] == "expected" && !CompletionKit::Checks::Registry.compares_value?(kind)
-        errors.add(:check_config, "compare_to expected only applies to contains, not_contains, or equals")
+        add_check_error("can grade against \"Each row's expected value\" only with #{CompletionKit::Checks::Registry.comparable_kinds.join(", ")}", "compare_to")
       end
     end
 
     def validate_check_target(config)
       target = config["target"].presence || "response_text"
-      unless CompletionKit::Checks::TargetResolver::TARGETS.include?(target)
-        errors.add(:check_config, "target must be one of #{CompletionKit::Checks::TargetResolver::TARGETS.join(", ")}")
-      end
-      if target == "json_path" && config["target_path"].to_s.strip.empty?
-        errors.add(:check_config, "target_path is required when target is json_path")
+      path_given = config["target_path"].present?
+      if !CompletionKit::Checks::TargetResolver::TARGETS.include?(target)
+        add_check_error("needs #{check_label("target")} to be one of #{CompletionKit::Checks::TargetResolver::TARGETS.join(", ")}", "target")
+      elsif target == "json_path" && !path_given
+        add_check_error("needs #{check_label("target_path")} filled in to check a value from the response JSON", "target_path")
+      elsif target != "json_path" && path_given && will_save_change_to_check_config?
+        add_check_error("needs #{check_label("target")} to be \"#{CompletionKit::Checks::Labels.target("json_path")}\" when #{check_label("target_path")} is filled in", "target")
       end
     end
 
+    def default_check_target
+      config = check_config
+      return unless config.is_a?(Hash) && config["target"].blank? && config["target_path"].present? && will_save_change_to_check_config?
+
+      self.check_config = config.merge("target" => "json_path")
+    end
+
     def validate_check_required_keys(config, kind)
       required = CompletionKit::Checks::Registry.required_keys.fetch(kind)
-      required -= %w[value] if config["compare_to"] == "expected" && CompletionKit::Checks::Registry.compares_value?(kind)
+      required -= [CompletionKit::Checks::Registry.expected_key(kind)] if config["compare_to"] == "expected" && CompletionKit::Checks::Registry.compares_value?(kind)
       required.each do |required_key|
-        if required_key == "expected"
-          errors.add(:check_config, "expected is required") unless config.key?("expected")
-        elsif config[required_key].to_s.strip.empty?
-          errors.add(:check_config, "#{required_key} is required")
-        end
+        missing = required_key == "expected" ? !config.key?("expected") : config[required_key].to_s.strip.empty?
+        add_check_error("needs #{check_label(required_key, kind)} filled in", required_key) if missing
       end
     end
 
     def validate_check_kind_rules(config, kind)
       case kind
       when "regex"
-        begin
-          Regexp.new(config["pattern"].to_s)
-        rescue RegexpError
-          errors.add(:check_config, "pattern is not a valid regular expression")
-        end
-      when "length_bounds"
-        min = config["min"]
-        max = config["max"]
-        if min.nil? && max.nil?
-          errors.add(:check_config, "length_bounds requires at least one of min or max")
-        elsif min && max && min.to_i > max.to_i
-          errors.add(:check_config, "min must be less than or equal to max")
-        end
+        validate_check_pattern(config)
+      when "list_overlap"
+        validate_check_list_overlap(config)
+      when "numeric_equals"
+        validate_check_numeric_equals(config)
       end
+      validate_check_bounds(config, kind) if CompletionKit::Checks::Registry.bounded?(kind)
+    end
+
+    def validate_check_pattern(config)
+      Regexp.new(config["pattern"].to_s)
+    rescue RegexpError
+      add_check_error("needs #{check_label("pattern")} to be a valid regular expression", "pattern")
+    end
+
+    def validate_check_bounds(config, kind)
+      min = config["min"].to_s.strip
+      max = config["max"].to_s.strip
+      min_label = check_label("min", kind)
+      max_label = check_label("max", kind)
+      if min.empty? && max.empty?
+        add_check_error("needs #{min_label} or #{max_label} filled in", "min", "max")
+        return
+      end
+
+      min_value = CompletionKit::Checks::NumericValue.parse(min)
+      max_value = CompletionKit::Checks::NumericValue.parse(max)
+      not_numbers = { "min" => !min.empty? && min_value.nil?, "max" => !max.empty? && max_value.nil? }.select { |_key, bad| bad }.keys
+      if not_numbers.any?
+        add_check_error("needs #{min_label} and #{max_label} to be numbers", *not_numbers)
+      elsif min_value && max_value && min_value > max_value
+        add_check_error("needs #{min_label} to be no more than #{max_label}", "min", "max")
+      end
+    end
+
+    def validate_check_list_overlap(config)
+      measures = CompletionKit::Checks::ListOverlap::MEASURES
+      if config["score_by"].present? && !measures.include?(config["score_by"])
+        add_check_error("needs #{check_label("score_by")} to be one of #{measures.join(", ")}", "score_by")
+      end
+      return if config["min"].to_s.strip.empty?
+
+      threshold = CompletionKit::Checks::NumericValue.parse(config["min"])
+      if threshold.nil? || threshold.negative? || threshold > 1
+        add_check_error("needs #{check_label("min", "list_overlap")} to be a number between 0 and 1", "min")
+      end
+    end
+
+    def validate_check_numeric_equals(config)
+      validate_check_tolerance(config)
+      return if config["compare_to"] == "expected" || config["value"].to_s.strip.empty?
+
+      add_check_error("needs #{check_label("value", "numeric_equals")} to be a number", "value") if CompletionKit::Checks::NumericValue.parse(config["value"]).nil?
+    end
+
+    def validate_check_tolerance(config)
+      return if config["tolerance"].to_s.strip.empty?
+
+      tolerance = CompletionKit::Checks::NumericEquals.tolerance_value(config["tolerance"])
+      if tolerance.nil? || tolerance.negative?
+        add_check_error("needs #{check_label("tolerance")} to be a number that is not negative, or a percentage like 2%", "tolerance")
+      end
+    end
+
+    def add_check_error(message, *keys)
+      errors.add(:check_config, message, check_keys: keys)
+    end
+
+    def check_label(key, kind = nil)
+      "\"#{CompletionKit::Checks::Labels.field(key, kind)}\""
     end
 
     def set_defaults

data/app/models/completion_kit/metric_version.rb

@@ -85,12 +85,14 @@ module CompletionKit
     end
 
     def publish!
+      config = publishable_check_config if check?
+      validate_check_config_on_metric!(config) if check?
       MetricVersion.transaction do
         self.class.where(metric_id: metric_id).where.not(id: id).update_all(current: false)
         reload
         update!(state: "published", current: true, published_at: published_at || Time.current)
         if check?
-          metric.update_columns(metric_type: "check", check_config: check_config)
+          metric.update_columns(metric_type: "check", check_config: config)
         else
           metric.update_columns(
             metric_type: "llm_judge",
@@ -123,6 +125,19 @@ module CompletionKit
 
     private
 
+    def publishable_check_config
+      config = check_config.to_h.stringify_keys
+      return config if config["target_path"].blank?
+      return config.merge("target" => "json_path") if config["target"].blank?
+
+      config["target"] == "json_path" ? config : config.except("target_path")
+    end
+
+    def validate_check_config_on_metric!(config)
+      metric.assign_attributes(metric_type: "check", check_config: config)
+      raise ActiveRecord::RecordInvalid, metric unless metric.valid?
+    end
+
     def check_change_summary_against(previous)
       return nil if check_config == previous.check_config
 

data/app/models/completion_kit/review.rb

@@ -10,6 +10,7 @@ module CompletionKit
     validates :metric_name, presence: true
     validates :metric_version, presence: true
     validates :ai_score, numericality: { greater_than_or_equal_to: 1, less_than_or_equal_to: 5 }, allow_nil: true
+    validates :check_score, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true
 
     before_validation :set_default_status
 
@@ -36,6 +37,7 @@ module CompletionKit
         id: id, response_id: response_id, metric_id: metric_id,
         metric_version_id: metric_version_id,
         metric_name: metric_name, ai_score: ai_score, passed: passed,
+        check_score: check_score&.to_f,
         ai_feedback: ai_feedback, status: status, attempts: attempts,
         error: error_payload
       }

data/app/models/completion_kit/run.rb

@@ -10,7 +10,7 @@ module CompletionKit
       status: "pending",
       attempts: 0,
       error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
-      ai_score: nil, passed: nil, ai_feedback: nil
+      ai_score: nil, passed: nil, check_score: nil, ai_feedback: nil
     }.freeze
 
     belongs_to :prompt, optional: true
@@ -79,7 +79,9 @@ module CompletionKit
         Arel.sql("COUNT(ai_score)"),
         Arel.sql("SUM(CASE WHEN ai_score < #{low_score_ceiling} THEN 1 ELSE 0 END)"),
         Arel.sql("COUNT(passed)"),
-        Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
+        Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)"),
+        Arel.sql("AVG(check_score)"),
+        Arel.sql("COUNT(check_score)")
       )
       metrics_by_run = metric_rows.group_by(&:first)
 
@@ -90,12 +92,12 @@ module CompletionKit
         run.avg_score = stats && stats[:avg] ? stats[:avg].to_f.round(2) : nil
         run.check_pass_rate = stats && stats[:resolved] > 0 ? (stats[:passed].to_f / stats[:resolved]).round(2) : nil
 
-        run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed)|
+        run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed, fraction_avg, fraction_count)|
           if scored.to_i > 0
             {name: name, avg: avg.to_f.round(1), count: scored.to_i, low_count: low.to_i}
           elsif resolved.to_i > 0
-            {name: name, kind: "check", pass_rate: (passed.to_i.to_f / resolved.to_i).round(2),
-             count: resolved.to_i, low_count: resolved.to_i - passed.to_i}
+            check_summary(name, passed.to_i, resolved.to_i,
+                          fraction_count.to_i > 0 ? fraction_avg.to_f : nil)
           end
         end
       end
@@ -109,6 +111,14 @@ module CompletionKit
       CompletionKit.config.medium_quality_threshold.to_f
     end
 
+    def self.check_summary(name, passed, resolved, fraction_avg)
+      summary = { name: name, kind: "check", pass_rate: (passed.to_f / resolved).round(2),
+                  count: resolved, low_count: resolved - passed }
+      return summary if fraction_avg.nil?
+
+      summary.merge(avg_check_score: fraction_avg.round(3))
+    end
+
     TOP_SCORE = 5.0
     CEILING_MEAN = 4.8
     CEILING_TOP_SHARE = 0.9
@@ -271,9 +281,9 @@ module CompletionKit
           resolved = reviews.reject { |r| r.passed.nil? }
           next if resolved.empty?
 
-          passed = resolved.count { |r| r.passed == true }
-          { name: name, kind: "check", pass_rate: (passed.to_f / resolved.length).round(2),
-            count: resolved.length, low_count: resolved.length - passed }
+          fractions = reviews.filter_map { |r| r.check_score&.to_f }
+          self.class.check_summary(name, resolved.count { |r| r.passed == true }, resolved.length,
+                                   fractions.any? ? fractions.sum / fractions.length : nil)
         end
       end
     end
@@ -446,6 +456,7 @@ module CompletionKit
           metric_version_id: nil,
           ai_score: nil,
           passed: nil,
+          check_score: nil,
           ai_feedback: nil,
           error_provider: nil,
           error_class: nil,

data/app/services/completion_kit/checks/contains.rb

@@ -2,15 +2,9 @@ module CompletionKit
   module Checks
     class Contains
       def call(target, config)
-        value = config["value"].to_s
-        haystack = target.to_s
-        present = if config["case_sensitive"] == true
-          haystack.include?(value)
-        else
-          haystack.downcase.include?(value.downcase)
-        end
+        value, haystack = Textual.operands(target, config)
 
-        if present
+        if Textual.include?(haystack, value, config)
           Result.new(passed: true, detail: "contains #{value.inspect}")
         else
           Result.new(passed: false, detail: "does not contain #{value.inspect}")

data/app/services/completion_kit/checks/equals.rb

@@ -2,14 +2,9 @@ module CompletionKit
   module Checks
     class Equals
       def call(target, config)
-        actual = target.to_s
-        expected = config["value"].to_s
-        if config["trim"] == true
-          actual = actual.strip
-          expected = expected.strip
-        end
+        expected, actual = Textual.operands(target, config)
 
-        match = if config["case_sensitive"] == true
+        match = if Flag.on?(config, "case_sensitive")
           actual == expected
         else
           actual.casecmp?(expected)

data/app/services/completion_kit/checks/expected_resolver.rb

@@ -4,13 +4,27 @@ module CompletionKit
       UNRESOLVED = TargetResolver::UNRESOLVED
 
       def self.call(response, config)
+        TargetResolver.stringify(resolve(response, config) { |raw| raw.to_s })
+      end
+
+      def self.call_value(response, config)
+        resolve(response, config) { |raw| parse(raw) }
+      end
+
+      def self.resolve(response, config)
         raw = response.expected_output
         return UNRESOLVED if raw.blank?
 
-        path = config["expected_path"].to_s.strip
-        return raw.to_s if path.empty?
+        path = TargetResolver.normalize_path(config["expected_path"])
+        return yield(raw) if path.empty?
+
+        TargetResolver.dig_json(raw.to_s, path)
+      end
 
-        TargetResolver.resolve_json_path(raw.to_s, path)
+      def self.parse(raw)
+        JSON.parse(raw.to_s)
+      rescue JSON::ParserError
+        raw
       end
     end
   end

data/app/services/completion_kit/checks/flag.rb

@@ -0,0 +1,13 @@
+module CompletionKit
+  module Checks
+    module Flag
+      def self.on?(config, key)
+        cast(config[key])
+      end
+
+      def self.cast(value)
+        ActiveModel::Type::Boolean.new.cast(value) == true
+      end
+    end
+  end
+end

data/app/services/completion_kit/checks/json_path_equals.rb

@@ -1,13 +1,10 @@
 module CompletionKit
   module Checks
     class JsonPathEquals
-      MISSING = Object.new
-
       def call(target, config)
-        parsed = JSON.parse(target.to_s)
-        value = dig(parsed, config["json_path"].to_s)
+        value = TargetResolver.walk(JSON.parse(target.to_s), config["json_path"])
 
-        if value.equal?(MISSING)
+        if value.equal?(TargetResolver::UNRESOLVED)
           Result.new(passed: false, detail: "path #{config["json_path"]} not found")
         elsif value == config["expected"]
           Result.new(passed: true, detail: "#{config["json_path"]} == #{config["expected"].inspect}")
@@ -17,16 +14,6 @@ module CompletionKit
       rescue JSON::ParserError
         Result.new(passed: false, detail: "not valid JSON")
       end
-
-      private
-
-      def dig(data, path)
-        path.split(".").reduce(data) do |node, key|
-          return MISSING unless node.is_a?(Hash) && node.key?(key)
-
-          node[key]
-        end
-      end
     end
   end
 end

data/app/services/completion_kit/checks/labels.rb

@@ -0,0 +1,104 @@
+module CompletionKit
+  module Checks
+    module Labels
+      KINDS = {
+        "contains" => "Contains a phrase",
+        "not_contains" => "Does not contain a phrase",
+        "equals" => "Equals exactly",
+        "regex" => "Matches a pattern",
+        "valid_json" => "Is valid JSON",
+        "json_path_equals" => "A JSON field equals a value",
+        "length_bounds" => "Length is within a range",
+        "list_overlap" => "How much of a list was found",
+        "numeric_bounds" => "A number is within a range",
+        "numeric_equals" => "A number is close enough"
+      }.freeze
+
+      TARGETS = {
+        "response_text" => "The response text",
+        "input_data" => "The input row",
+        "json_path" => "A value from the response JSON"
+      }.freeze
+
+      COMPARE_TO = {
+        "constant" => "A value you type",
+        "expected" => "Each row's expected value"
+      }.freeze
+
+      SCORE_BY = {
+        "recall" => "Coverage: how much of the expected list it found",
+        "precision" => "Accuracy: how much of what it returned was expected",
+        "f1" => "Balance: coverage and accuracy weighed equally",
+        "jaccard" => "Overlap: shared items out of every item in either list"
+      }.freeze
+
+      FIELDS = {
+        "check_kind" => "Rule",
+        "target" => "What to check",
+        "target_path" => "Path into the JSON",
+        "compare_to" => "Compare against",
+        "value" => "Text to look for",
+        "pattern" => "Pattern",
+        "json_path" => "JSON path",
+        "expected" => "Expected value",
+        "expected_path" => "Field in the answer key",
+        "score_by" => "How to score it",
+        "min" => "Minimum",
+        "max" => "Maximum",
+        "tolerance" => "How far off is still right",
+        "case_sensitive" => "Case sensitive",
+        "multiline" => "Multiline",
+        "trim" => "Trim whitespace"
+      }.freeze
+
+      FIELDS_BY_KIND = {
+        "length_bounds" => { "min" => "Shortest allowed", "max" => "Longest allowed" },
+        "numeric_bounds" => { "min" => "Lowest allowed", "max" => "Highest allowed" },
+        "list_overlap" => { "min" => "Lowest score that passes", "value" => "Expected list" },
+        "numeric_equals" => { "value" => "Expected number" }
+      }.freeze
+
+      HINTS = {
+        "score_by" => "Coverage ignores extra items, and accuracy ignores missing ones. Balance weighs the two equally. " \
+                      "Overlap divides the shared items by every item in either list, so it never scores higher than the other three.",
+        "min" => "Leave blank for no minimum.",
+        "max" => "Leave blank for no maximum."
+      }.freeze
+
+      HINTS_BY_KIND = {
+        "list_overlap" => { "min" => "Give a value from 0 to 1, like 0.6 for 60%. Leave blank to require a perfect score." }
+      }.freeze
+
+      VALUES = { "compare_to" => COMPARE_TO, "score_by" => SCORE_BY }.freeze
+
+      def self.kind(kind)
+        KINDS.fetch(kind.to_s) { kind.to_s.humanize }
+      end
+
+      def self.target(target)
+        TARGETS.fetch(target.to_s) { target.to_s.humanize }
+      end
+
+      def self.field(key, kind = nil)
+        FIELDS_BY_KIND.dig(kind.to_s, key.to_s) || FIELDS.fetch(key.to_s) { key.to_s.humanize }
+      end
+
+      def self.hint(key, kind = nil)
+        HINTS_BY_KIND.dig(kind.to_s, key.to_s) || HINTS[key.to_s]
+      end
+
+      def self.value(key, value)
+        VALUES.fetch(key.to_s, {}).fetch(value.to_s, value)
+      end
+
+      def self.worded?(key)
+        VALUES.key?(key.to_s)
+      end
+
+      def self.in_form_order(config)
+        order = FIELDS.keys
+        config.sort_by.with_index { |(key, _value), index| [order.index(key) || order.length, index] }.to_h
+      end
+    end
+  end
+end

data/app/services/completion_kit/checks/length_bounds.rb

@@ -3,13 +3,13 @@ module CompletionKit
     class LengthBounds
       def call(target, config)
         length = target.to_s.length
-        min = config["min"] && config["min"].to_i
-        max = config["max"] && config["max"].to_i
+        min = NumericValue.parse(config["min"])
+        max = NumericValue.parse(config["max"])
 
         if min && length < min
-          Result.new(passed: false, detail: "length #{length} below min #{min}")
+          Result.new(passed: false, detail: "length #{length} below min #{NumericValue.format(min)}")
         elsif max && length > max
-          Result.new(passed: false, detail: "length #{length} above max #{max}")
+          Result.new(passed: false, detail: "length #{length} above max #{NumericValue.format(max)}")
         else
           Result.new(passed: true, detail: "length #{length} within bounds")
         end