completion-kit 0.28.46 → 0.29.0

data/app/services/completion_kit/checks/list_overlap.rb

@@ -0,0 +1,96 @@
+module CompletionKit
+  module Checks
+    class ListOverlap
+      MEASURES = %w[recall precision f1 jaccard].freeze
+      DEFAULT_MEASURE = "recall".freeze
+      DEFAULT_MIN = 1.0
+      SEPARATORS = /[,;\n]/
+      DECORATION = /\A(?:["'`]+|[-*•]\s+)|["'`]+\z/
+      PLAIN_NUMBER = /\A-?(?:0|[1-9]\d*)(?:\.\d+)?\z/
+
+      def call(target, config)
+        actual = to_set(target, config)
+        expected = to_set(config["value"], config)
+        measure = MEASURES.include?(config["score_by"]) ? config["score_by"] : DEFAULT_MEASURE
+        overlap = (actual & expected).length
+        score = score_for(measure, overlap, actual.length, expected.length).round(4)
+        threshold = NumericValue.parse(config["min"]) || DEFAULT_MIN
+
+        Result.new(
+          passed: score >= threshold,
+          detail: "found #{overlap} of #{expected.length} expected, #{actual.length} returned (score #{score.round(2)})",
+          score: score
+        )
+      end
+
+      private
+
+      def score_for(measure, overlap, actual_size, expected_size)
+        return 1.0 if actual_size.zero? && expected_size.zero?
+
+        precision = ratio(overlap, actual_size)
+        recall = expected_size.zero? ? 1.0 : ratio(overlap, expected_size)
+        case measure
+        when "precision" then precision
+        when "jaccard" then ratio(overlap, actual_size + expected_size - overlap)
+        when "f1" then f1(precision, recall)
+        else recall
+        end
+      end
+
+      def ratio(numerator, denominator)
+        return 0.0 if denominator.zero?
+
+        numerator.to_f / denominator
+      end
+
+      def f1(precision, recall)
+        return 0.0 if (precision + recall).zero?
+
+        (2 * precision * recall) / (precision + recall)
+      end
+
+      def to_set(value, config)
+        items = coerce_list(value).map { |item| normalize(item, config) }
+        items.reject(&:empty?).uniq
+      end
+
+      def normalize(item, config)
+        return number_text(item.to_s) if item.is_a?(::Numeric)
+
+        text = item.to_s.strip.gsub(DECORATION, "").strip
+        return number_text(text) if PLAIN_NUMBER.match?(text)
+
+        Flag.on?(config, "case_sensitive") ? text : text.downcase
+      end
+
+      def number_text(text)
+        number = BigDecimal(text)
+        number.frac.zero? ? number.to_i.to_s : number.to_s("F")
+      end
+
+      def coerce_list(value)
+        case value
+        when Array then value
+        when Hash, nil then []
+        else
+          text = value.to_s.strip
+          return [] if text.empty?
+
+          from_json(text) || text.split(SEPARATORS)
+        end
+      end
+
+      def from_json(text)
+        parsed = JSON.parse(text)
+        return parsed if parsed.is_a?(Array)
+        return parsed.split(SEPARATORS) if parsed.is_a?(String)
+        return [] if parsed.is_a?(Hash)
+
+        nil
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/app/services/completion_kit/checks/not_contains.rb

@@ -2,15 +2,9 @@ module CompletionKit
   module Checks
     class NotContains
       def call(target, config)
-        value = config["value"].to_s
-        haystack = target.to_s
-        present = if config["case_sensitive"] == true
-          haystack.include?(value)
-        else
-          haystack.downcase.include?(value.downcase)
-        end
+        value, haystack = Textual.operands(target, config)
 
-        if present
+        if Textual.include?(haystack, value, config)
           Result.new(passed: false, detail: "contains #{value.inspect}")
         else
           Result.new(passed: true, detail: "does not contain #{value.inspect}")

data/app/services/completion_kit/checks/numeric_bounds.rb

@@ -0,0 +1,22 @@
+module CompletionKit
+  module Checks
+    class NumericBounds
+      def call(target, config)
+        number = NumericValue.parse(target)
+        return Result.new(passed: false, detail: "#{NumericValue.preview(target)} is not a number") if number.nil?
+
+        min = NumericValue.parse(config["min"])
+        max = NumericValue.parse(config["max"])
+        shown = NumericValue.format(number)
+
+        if min && number < min
+          Result.new(passed: false, detail: "#{shown} is below min #{NumericValue.format(min)}")
+        elsif max && number > max
+          Result.new(passed: false, detail: "#{shown} is above max #{NumericValue.format(max)}")
+        else
+          Result.new(passed: true, detail: "#{shown} is within bounds")
+        end
+      end
+    end
+  end
+end

data/app/services/completion_kit/checks/numeric_equals.rb

@@ -0,0 +1,37 @@
+module CompletionKit
+  module Checks
+    class NumericEquals
+      PERCENT = /\A(.*)%\z/
+
+      def call(target, config)
+        actual = NumericValue.decimal(target)
+        return Result.new(passed: false, detail: "#{NumericValue.preview(target)} is not a number") if actual.nil?
+
+        expected = NumericValue.decimal(config["value"])
+        return Result.new(passed: false, detail: "expected #{NumericValue.preview(config["value"])} is not a number") if expected.nil?
+
+        difference = (actual - expected).abs
+        allowed = self.class.allowance(config["tolerance"], expected)
+        shown = "#{NumericValue.format(actual)} vs #{NumericValue.format(expected)}"
+
+        if difference <= allowed
+          Result.new(passed: true, detail: "#{shown}, within #{NumericValue.format(allowed)}")
+        else
+          Result.new(passed: false, detail: "#{shown}, off by #{NumericValue.format(difference)} (allowed #{NumericValue.format(allowed)})")
+        end
+      end
+
+      def self.allowance(raw, expected)
+        share = PERCENT.match(raw.to_s)
+        return (NumericValue.decimal(share[1]) || BigDecimal("0")) / 100 * expected.abs if share
+
+        NumericValue.decimal(raw) || BigDecimal("0")
+      end
+
+      def self.tolerance_value(raw)
+        share = PERCENT.match(raw.to_s)
+        NumericValue.parse(share ? share[1] : raw)
+      end
+    end
+  end
+end

data/app/services/completion_kit/checks/numeric_value.rb

@@ -0,0 +1,45 @@
+module CompletionKit
+  module Checks
+    module NumericValue
+      STRIPPED = /[\s_   ]/
+      CURRENCY = /\A([+-]?)[$£€]/
+      GROUPED = /\A[+-]?[1-9]\d{0,2}(?:,\d{3})+(?:\.\d*)?(?:[eE][+-]?\d+)?\z/
+      NUMERIC = /\A[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?\z/
+      PREVIEW_LENGTH = 40
+
+      def self.parse(value)
+        return finite(value.to_f) if value.is_a?(::Numeric)
+
+        text = clean(value)
+        return nil unless NUMERIC.match?(text)
+
+        finite(Float(text))
+      end
+
+      def self.decimal(value)
+        return nil if parse(value).nil?
+
+        BigDecimal(value.is_a?(::Numeric) ? value.to_s : clean(value))
+      end
+
+      def self.clean(value)
+        text = value.to_s.gsub(STRIPPED, "").sub(CURRENCY) { ::Regexp.last_match(1) }
+        GROUPED.match?(text) ? text.delete(",") : text
+      end
+
+      def self.finite(number)
+        number.finite? ? number : nil
+      end
+
+      def self.format(number)
+        return number.to_i.to_s if number == number.to_i
+
+        number.to_s
+      end
+
+      def self.preview(value)
+        value.inspect.truncate(PREVIEW_LENGTH)
+      end
+    end
+  end
+end

data/app/services/completion_kit/checks/regex.rb

@@ -3,8 +3,8 @@ module CompletionKit
     class Regex
       def call(target, config)
         options = 0
-        options |= Regexp::IGNORECASE if config["case_sensitive"] == false
-        options |= Regexp::MULTILINE if config["multiline"] == true
+        options |= Regexp::IGNORECASE unless Flag.on?(config, "case_sensitive")
+        options |= Regexp::MULTILINE if Flag.on?(config, "multiline")
         pattern = Regexp.new(config["pattern"].to_s, options)
 
         if pattern.match?(target.to_s)

data/app/services/completion_kit/checks/registry.rb

@@ -8,7 +8,10 @@ module CompletionKit
         "regex" => Regex,
         "valid_json" => ValidJson,
         "json_path_equals" => JsonPathEquals,
-        "length_bounds" => LengthBounds
+        "length_bounds" => LengthBounds,
+        "list_overlap" => ListOverlap,
+        "numeric_bounds" => NumericBounds,
+        "numeric_equals" => NumericEquals
       }.freeze
 
       REQUIRED_KEYS = {
@@ -18,10 +21,43 @@ module CompletionKit
         "regex" => %w[pattern],
         "valid_json" => [],
         "json_path_equals" => %w[json_path expected],
-        "length_bounds" => []
+        "length_bounds" => [],
+        "list_overlap" => %w[value],
+        "numeric_bounds" => [],
+        "numeric_equals" => %w[value]
       }.freeze
 
-      VALUE_COMPARISON_KINDS = %w[contains not_contains equals].freeze
+      CONFIG_KEYS = {
+        "contains" => %w[value case_sensitive trim],
+        "not_contains" => %w[value case_sensitive trim],
+        "equals" => %w[value case_sensitive trim],
+        "regex" => %w[pattern case_sensitive multiline],
+        "valid_json" => [],
+        "json_path_equals" => %w[json_path expected],
+        "length_bounds" => %w[min max],
+        "list_overlap" => %w[value score_by min case_sensitive],
+        "numeric_bounds" => %w[min max],
+        "numeric_equals" => %w[value tolerance]
+      }.freeze
+
+      SHARED_KEYS = %w[check_kind target target_path compare_to expected_path].freeze
+      FLAG_KEYS = %w[case_sensitive multiline trim].freeze
+      NUMBER_KEYS = %w[min max tolerance].freeze
+      DEFAULTS = { "target" => "response_text", "score_by" => ListOverlap::DEFAULT_MEASURE }.freeze
+
+      EXPECTED_KEYS = {
+        "contains" => "value",
+        "not_contains" => "value",
+        "equals" => "value",
+        "json_path_equals" => "expected",
+        "list_overlap" => "value",
+        "numeric_equals" => "value"
+      }.freeze
+
+      RAW_TARGET_KINDS = %w[list_overlap numeric_bounds numeric_equals].freeze
+      RAW_EXPECTED_KINDS = %w[json_path_equals list_overlap numeric_equals].freeze
+      BOUNDED_KINDS = %w[length_bounds numeric_bounds].freeze
+      SCORING_KINDS = %w[list_overlap].freeze
 
       KINDS = CHECKS.keys.freeze
 
@@ -33,13 +69,80 @@ module CompletionKit
         REQUIRED_KEYS
       end
 
+      def self.config_keys(kind)
+        CONFIG_KEYS.fetch(kind, [])
+      end
+
       def self.compares_value?(kind)
-        VALUE_COMPARISON_KINDS.include?(kind)
+        EXPECTED_KEYS.key?(kind)
+      end
+
+      def self.expected_key(kind)
+        EXPECTED_KEYS[kind]
+      end
+
+      def self.comparable_kinds
+        EXPECTED_KEYS.keys
+      end
+
+      def self.raw_target?(kind)
+        RAW_TARGET_KINDS.include?(kind)
+      end
+
+      def self.raw_expected?(kind)
+        RAW_EXPECTED_KINDS.include?(kind)
+      end
+
+      def self.bounded?(kind)
+        BOUNDED_KINDS.include?(kind)
+      end
+
+      def self.scores?(kind)
+        SCORING_KINDS.include?(kind)
       end
 
       def self.fetch(kind)
         CHECKS.fetch(kind).new
       end
+
+      def self.unused_keys(config)
+        kind = config["check_kind"]
+        keys = config["compare_to"] == "expected" && compares_value?(kind) ? [expected_key(kind)] : %w[compare_to expected_path]
+        config["target"] == "json_path" ? keys : keys + %w[target_path]
+      end
+
+      def self.canonical(config)
+        hash = config.to_h.stringify_keys
+        relevant = hash.slice(*SHARED_KEYS, *config_keys(hash["check_kind"])).except(*unused_keys(hash))
+        relevant.to_h { |key, value| [key, canonical_value(key, value)] }.reject { |key, value| omitted?(key, value) }
+      end
+
+      def self.canonical_value(key, value)
+        case key
+        when *FLAG_KEYS then Flag.cast(value)
+        when *NUMBER_KEYS then NumericValue.parse(value) || value
+        when "value" then canonical_operand(value)
+        else value
+        end
+      end
+
+      def self.canonical_operand(value)
+        return value.to_s if value.is_a?(::Numeric)
+        return value unless value.is_a?(String)
+
+        parsed = parse_json(value)
+        parsed.is_a?(Array) ? parsed : value
+      end
+
+      def self.parse_json(text)
+        JSON.parse(text)
+      rescue JSON::ParserError
+        text
+      end
+
+      def self.omitted?(key, value)
+        value.nil? || value == "" || (FLAG_KEYS.include?(key) && !value) || DEFAULTS[key] == value
+      end
     end
   end
 end

data/app/services/completion_kit/checks/result.rb

@@ -1,5 +1,9 @@
 module CompletionKit
   module Checks
-    Result = Data.define(:passed, :detail)
+    Result = Data.define(:passed, :detail, :score) do
+      def initialize(passed:, detail:, score: nil)
+        super
+      end
+    end
   end
 end

data/app/services/completion_kit/checks/target_resolver.rb

@@ -3,28 +3,63 @@ module CompletionKit
     module TargetResolver
       TARGETS = %w[response_text input_data json_path].freeze
       UNRESOLVED = Object.new.freeze
+      ARRAY_INDEX = /\A\d+\z/
+      ROOT = /\A\$(?:\.|\z)/
 
       def self.call(response, config)
+        stringify(call_value(response, config))
+      end
+
+      def self.call_value(response, config)
         case config["target"]
         when "input_data"
           response.input_data
         when "json_path"
-          resolve_json_path(response.response_text, config["target_path"].to_s)
+          dig_json(response.response_text, config["target_path"])
         else
           response.response_text
         end
       end
 
-      def self.resolve_json_path(text, path)
-        parsed = JSON.parse(text.to_s)
-        value = path.split(".").reduce(parsed) do |node, key|
-          return UNRESOLVED unless node.is_a?(Hash) && node.key?(key)
+      def self.dig_json(text, path)
+        walk(JSON.parse(text.to_s), path)
+      rescue JSON::ParserError
+        UNRESOLVED
+      end
 
-          node[key]
+      def self.walk(document, path)
+        normalize_path(path).split(".").reduce(document) do |node, key|
+          case node
+          when Hash
+            return UNRESOLVED unless node.key?(key)
+
+            node[key]
+          when Array
+            index = array_index(node, key)
+            return UNRESOLVED if index.nil?
+
+            node[index]
+          else
+            return UNRESOLVED
+          end
         end
+      end
+
+      def self.normalize_path(path)
+        path.to_s.strip.sub(ROOT, "")
+      end
+
+      def self.array_index(array, key)
+        return nil unless ARRAY_INDEX.match?(key)
+
+        index = key.to_i
+        index < array.length ? index : nil
+      end
+
+      def self.stringify(value)
+        return value if value.equal?(UNRESOLVED)
+
         value.to_s
-      rescue JSON::ParserError
-        UNRESOLVED
       end
     end
   end

data/app/services/completion_kit/checks/textual.rb

@@ -0,0 +1,19 @@
+module CompletionKit
+  module Checks
+    module Textual
+      def self.operands(target, config)
+        value = config["value"].to_s
+        haystack = target.to_s
+        return [value.strip, haystack.strip] if Flag.on?(config, "trim")
+
+        [value, haystack]
+      end
+
+      def self.include?(haystack, value, config)
+        return haystack.include?(value) if Flag.on?(config, "case_sensitive")
+
+        haystack.downcase.include?(value.downcase)
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/metrics.rb

@@ -3,30 +3,44 @@ module CompletionKit
     module Metrics
       extend Base
 
+      CASE_SENSITIVE_NOTE = "A check with no case_sensitive matches case-insensitively, regex and list_overlap included; " \
+        "send case_sensitive: true to match case exactly. Before 0.29.0 a regex check read a missing case_sensitive as case-sensitive."
+
       CHECK_CONFIG_SCHEMA = {
         type: "object",
         properties: {
           check_kind: {type: "string", enum: CompletionKit::Checks::Registry.kinds},
           target: {type: "string", enum: CompletionKit::Checks::TargetResolver::TARGETS},
           target_path: {type: "string"},
-          value: {type: "string"},
+          value: {type: %w[string number array], items: {type: %w[string number]}, description: "Only list_overlap takes a list, as a JSON array or as comma-separated text; every other rule refuses one"},
           pattern: {type: "string"},
           json_path: {type: "string"},
           expected: {},
           compare_to: {type: "string", enum: %w[constant expected]},
           expected_path: {type: "string"},
-          min: {type: "integer"},
-          max: {type: "integer"},
-          case_sensitive: {type: "boolean"},
+          min: {type: "number"},
+          max: {type: "number"},
+          score_by: {type: "string", enum: CompletionKit::Checks::ListOverlap::MEASURES},
+          tolerance: {type: %w[number string], description: "A plain number for a fixed amount, or a percentage string such as \"2%\" for a share of the expected value"},
+          case_sensitive: {type: "boolean", description: CASE_SENSITIVE_NOTE},
           multiline: {type: "boolean"},
           trim: {type: "boolean"}
         }
       }.freeze
 
       CHECK_CONFIG_HINT = "For a deterministic check set metric_type:\"check\" and check_config. Per-kind required keys: " \
-        "value (contains/not_contains/equals), pattern (regex), json_path+expected (json_path_equals), " \
-        "min and/or max (length_bounds); valid_json takes no extra keys. target_path is required when target is json_path. " \
-        "For contains, not_contains, and equals, set compare_to:\"expected\" to grade against each row's own expected_output (ground truth) instead of a constant value (drop value); add expected_path to dig into the expected value when it is JSON."
+        "value (contains/not_contains/equals/list_overlap/numeric_equals), pattern (regex), json_path+expected (json_path_equals), " \
+        "min and/or max (length_bounds/numeric_bounds); valid_json takes no extra keys. target_path is required when target is json_path; " \
+        "a target_path sent with no target sets target to json_path, and one sent with any other target is refused. " \
+        "target_path walks array indices, as in items.0.name; any path may start at the root with $., as in $.items.0.name. " \
+        "list_overlap grades a list-valued field and stores a fraction rather than a verdict: score_by is recall, precision, f1 or jaccard (recall by default), " \
+        "and min is the fraction that counts as a pass (1, a perfect score, by default). " \
+        "numeric_bounds and numeric_equals read a number, ignoring the thousands commas and currency symbol a figure is written with; " \
+        "a comma counts only as a thousands separator, with three digits after it and no decimal point before it, so 1,5 is not a number; " \
+        "numeric_equals takes a tolerance, either a plain number for a fixed amount or a percentage string such as 2% for a share of the expected value. " \
+        "#{CASE_SENSITIVE_NOTE} " \
+        "Set compare_to:\"expected\" on #{CompletionKit::Checks::Registry.comparable_kinds.join(", ")} to grade against each row's own expected_output (ground truth) " \
+        "instead of a constant; drop value (or expected, for json_path_equals) and add expected_path to dig into the answer key when it is JSON."
 
       TOOLS = {
         "metrics_list" => {

data/app/services/completion_kit/promptfoo_importer.rb

@@ -149,17 +149,17 @@ module CompletionKit
     def check_mapping(type, text)
       case type
       when "contains"
-        [{ "check_kind" => "contains", "target" => "response_text", "value" => text }, "Contains #{text.truncate(30).inspect}"]
+        [{ "check_kind" => "contains", "target" => "response_text", "value" => text, "case_sensitive" => true }, "Contains #{text.truncate(30).inspect}"]
       when "icontains"
         [{ "check_kind" => "contains", "target" => "response_text", "value" => text, "case_sensitive" => false }, "Contains #{text.truncate(30).inspect}"]
       when "not-contains"
-        [{ "check_kind" => "not_contains", "target" => "response_text", "value" => text }, "Does not contain #{text.truncate(30).inspect}"]
+        [{ "check_kind" => "not_contains", "target" => "response_text", "value" => text, "case_sensitive" => true }, "Does not contain #{text.truncate(30).inspect}"]
       when "not-icontains"
         [{ "check_kind" => "not_contains", "target" => "response_text", "value" => text, "case_sensitive" => false }, "Does not contain #{text.truncate(30).inspect}"]
       when "equals"
-        [{ "check_kind" => "equals", "target" => "response_text", "value" => text }, "Equals #{text.truncate(30).inspect}"]
+        [{ "check_kind" => "equals", "target" => "response_text", "value" => text, "case_sensitive" => true }, "Equals #{text.truncate(30).inspect}"]
       when "regex"
-        [{ "check_kind" => "regex", "target" => "response_text", "pattern" => text }, "Matches /#{text.truncate(30)}/"]
+        [{ "check_kind" => "regex", "target" => "response_text", "pattern" => text, "case_sensitive" => true }, "Matches /#{text.truncate(30)}/"]
       when "is-json"
         [{ "check_kind" => "valid_json", "target" => "response_text" }, "Valid JSON"]
       end

data/app/services/completion_kit/run_comparison.rb

@@ -9,5 +9,11 @@ module CompletionKit
 
       "same"
     end
+
+    def score_delta(left, right)
+      return nil if left.nil? || right.nil?
+
+      (right.to_f - left.to_f).round(2)
+    end
   end
 end

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -135,7 +135,7 @@
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:id</p>
         <p class="ck-meta-copy">Get a run with status, progress, response count, and average score.</p>
-        <p class="ck-api-params"><strong>Also returns:</strong>&ensp;<code>metric_averages</code>, a per-metric breakdown with each metric's average score (<code>pass_rate</code> for checks), the number of rows it graded, and <code>low_count</code>, how many scored below the medium quality threshold</p>
+        <p class="ck-api-params"><strong>Also returns:</strong>&ensp;<code>metric_averages</code>, a per-metric breakdown with each metric's average score (<code>pass_rate</code> for checks, plus <code>avg_check_score</code> for a <code>list_overlap</code> check, the average fraction from 0 to 1), the number of rows it graded, and <code>low_count</code>, how many scored below the medium quality threshold</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl #{base_url}/api/v1/runs/1 \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <%= render "completion_kit/api_reference/resource_list", title: "Your recent runs",
@@ -166,7 +166,7 @@
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/runs/:id/compare?with=:other_id</p>
-        <p class="ck-meta-copy">Side-by-side comparison against another run. Returns <code>{left_run_id, right_run_id, metric_ids: [...], rows: [...]}</code> with one row per input case, per-metric scores on both sides, and the delta. Rows are keyed by the left run's input cases: a case present only in the left run is returned with the right side nulled out, and a case present only in the right run is omitted.</p>
+        <p class="ck-meta-copy">Side-by-side comparison against another run. Returns <code>{left_run_id, right_run_id, metric_ids: [...], rows: [...]}</code> with one row per input case, per-metric scores on both sides, and the delta. A check reports <code>left_passed</code>, <code>right_passed</code> and <code>result_change</code>, and a <code>list_overlap</code> check also reports <code>left_check_score</code>, <code>right_check_score</code> and <code>check_score_delta</code>, so a fraction that moved shows even when the verdict did not. Rows are keyed by the left run's input cases: a case present only in the left run is returned with the right side nulled out, and a case present only in the right run is omitted.</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl \"#{base_url}/api/v1/runs/1/compare?with=2\" \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <div class="ck-api-endpoint">
@@ -237,8 +237,12 @@
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/metrics</p>
-        <p class="ck-meta-copy">Create a metric. Defaults to an LLM judge; pass <code>metric_type</code> <code>check</code> with a <code>check_config</code> for a deterministic pass/fail check.</p>
+        <p class="ck-meta-copy">Create a metric. Defaults to an LLM judge; pass <code>metric_type</code> <code>check</code> with a <code>check_config</code> for a deterministic check that runs with no model and no cost.</p>
         <p class="ck-api-params"><strong>Required:</strong>&ensp;<code>name</code>&emsp;<strong>Optional:</strong>&ensp;<code>metric_type</code> (llm_judge default, or check), <code>instruction</code> and <code>rubric_bands</code> (array of {stars, description}) for judges, <code>check_config</code> (<code>check_kind</code>, <code>target</code>, plus per-kind keys) for checks</p>
+        <p class="ck-api-params"><strong>Check kinds:</strong>&ensp;<code>contains</code>, <code>not_contains</code>, <code>equals</code> (<code>value</code>); <code>regex</code> (<code>pattern</code>); <code>valid_json</code>; <code>json_path_equals</code> (<code>json_path</code>, <code>expected</code>); <code>length_bounds</code> and <code>numeric_bounds</code> (<code>min</code> and/or <code>max</code>); <code>list_overlap</code> (<code>value</code>, <code>score_by</code>, <code>min</code>); <code>numeric_equals</code> (<code>value</code>, <code>tolerance</code>)</p>
+        <p class="ck-api-params"><strong>Case:</strong>&ensp;A check with no <code>case_sensitive</code> matches case-insensitively, <code>regex</code> and <code>list_overlap</code> included. Send <code>"case_sensitive": true</code> to match case exactly. Before 0.29.0 a <code>regex</code> check read a missing <code>case_sensitive</code> as case-sensitive.</p>
+        <p class="ck-api-params"><strong>Grading against the answer key:</strong>&ensp;set <code>compare_to</code> to <code>expected</code> on <code>contains</code>, <code>not_contains</code>, <code>equals</code>, <code>json_path_equals</code>, <code>list_overlap</code> or <code>numeric_equals</code> to grade each row against its own <code>expected_output</code> rather than a constant. Drop <code>value</code> (or <code>expected</code>, on <code>json_path_equals</code>) and add <code>expected_path</code> to name the field to read out of a JSON answer key. A <code>target_path</code>, an <code>expected_path</code> and a <code>json_path</code> all walk array indices, as in <code>items.0.name</code>, and may start at the root with <code>$.</code>, as in <code>$.items.0.name</code>. A <code>target_path</code> sent with no <code>target</code> sets <code>target</code> to <code>json_path</code>, and one sent with any other <code>target</code> is refused.</p>
+        <p class="ck-api-params"><strong>Scores, not just verdicts:</strong>&ensp;<code>list_overlap</code> grades a list-valued field and stores a fraction on the review as <code>check_score</code>, with <code>score_by</code> one of recall, precision, f1 or jaccard. Its <code>min</code> is the fraction that counts as a pass. <code>numeric_equals</code> takes a <code>tolerance</code> that is a plain number for a fixed amount, or a percentage string such as <code>"2%"</code> for a share of the expected value. Each run averages it per metric as <code>avg_check_score</code> inside <code>metric_averages</code>, so a partial answer has run history and can gate a regression.</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/metrics \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"name\": \"relevance\", \"instruction\": \"Is the response relevant?\"}'" %>
       </div>
       <div class="ck-api-endpoint">

data/app/views/completion_kit/metrics/_check_spec.html.erb

@@ -1,23 +1,17 @@
 <% config = local_assigns.fetch(:config) %>
 <dl class="ck-check-spec">
   <div class="ck-check-spec__row">
-    <dt class="ck-check-spec__term">Rule</dt>
+    <dt class="ck-check-spec__term"><%= ck_check_field_label("check_kind") %></dt>
     <dd class="ck-check-spec__val"><%= ck_check_kind_label(config["check_kind"]) %></dd>
   </div>
   <div class="ck-check-spec__row">
-    <dt class="ck-check-spec__term">Checks</dt>
+    <dt class="ck-check-spec__term"><%= ck_check_field_label("target") %></dt>
     <dd class="ck-check-spec__val"><%= ck_check_target_label(config["target"]) %></dd>
   </div>
-  <% config.except("check_kind", "target").each do |key, value| %>
+  <% ck_check_spec_fields(config).each do |key, value| %>
     <div class="ck-check-spec__row">
-      <dt class="ck-check-spec__term"><%= ck_check_field_label(key) %></dt>
-      <dd class="ck-check-spec__val">
-        <% if key == "compare_to" %>
-          <%= ck_check_field_value(key, value) %>
-        <% else %>
-          <code class="ck-check-spec__code"><%= value %></code>
-        <% end %>
-      </dd>
+      <dt class="ck-check-spec__term"><%= ck_check_field_label(key, config["check_kind"]) %></dt>
+      <dd class="ck-check-spec__val"><%= ck_check_spec_value(key, value) %></dd>
     </div>
   <% end %>
 </dl>