completion-kit 0.21.1 → 0.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fc454c51bfbe7917a7b3f248b055d58b6a147fe7e9325dadc196d9aebf7c762
-  data.tar.gz: 5e96ffb9c4d1e98c4917ec7847a6b734e9153142184d0b540585d8f7c8aba3b6
+  metadata.gz: aa4d0b76c07fb964d0e7db4d26c101653d8ed0e27745e639eabb78d24653dee5
+  data.tar.gz: 61b5c7e4b150b4d0fad6e6d54d28fe6a69d27fa1223d9df2388c9229e039d954
 SHA512:
-  metadata.gz: 11ad299c28fdb4dc4be3ee113c99ebf19ec6cb9d422dbc180cb253292adc7373d2d183aef7fd21c5e7293c1acc017f3c0689f34a5ded2e13b17674a86cacb6e6
-  data.tar.gz: b995b586a1b2c5422be141b9e94476ea0d32928dc81e8aca4210c54c1b4d8ea6cd0d7729621a0155438a2b7819977e6678e1000c343d9160ec09db7120be47e6
+  metadata.gz: 2f7b9f2bb3fe037528736c388eff47dd4032edf9c6a5598d851cfb7c36f7a1ee81a818fd5b94866ec6855af87c5e018ef53dd691bcf8d816b316bfe7cd7793a8
+  data.tar.gz: 14203d5a281eeb6c43ab266d567f9c68803200654d4c8efcd48457936f19a60812eca6c0e40d863f279d66b3c50b674faf8a1f10d011536cf2f68a6a5f47cca2

data/app/assets/javascripts/completion_kit/application.js

@@ -221,20 +221,31 @@ var CK_CHECK_FIELDS = {
   no_refusal: []
 };
 
+var CK_VALUE_KINDS = ["contains", "not_contains", "equals"];
+
 function ckApplyCheckFields(scope) {
   if (!scope) return;
   var kindSelect = scope.querySelector('[name="metric[check_config][check_kind]"]');
   if (!kindSelect) return;
-  var visible = CK_CHECK_FIELDS[kindSelect.value];
+  var kind = kindSelect.value;
+  var visible = (CK_CHECK_FIELDS[kind] || []).slice();
   var targetSelect = scope.querySelector('[name="metric[check_config][target]"]');
   var targetIsJsonPath = !!(targetSelect && targetSelect.value === "json_path");
+
+  if (CK_VALUE_KINDS.indexOf(kind) !== -1) {
+    visible.push("compare_to");
+    var comparison = scope.querySelector('[name="metric[check_config][compare_to]"]:checked');
+    if (comparison && comparison.value === "expected") {
+      visible = visible.filter(function(key) { return key !== "value"; });
+      visible.push("expected_path");
+    }
+  }
+
   scope.querySelectorAll("[data-ck-check-field]").forEach(function(field) {
     var key = field.getAttribute("data-ck-check-field");
     var show;
     if (key === "target_path") {
       show = targetIsJsonPath;
-    } else if (!visible) {
-      show = true;
     } else {
       show = visible.indexOf(key) !== -1;
     }
@@ -274,7 +285,7 @@ document.addEventListener("change", function(e) {
     ckApplyMetricType(group);
     return;
   }
-  if (target.name === "metric[check_config][check_kind]" || target.name === "metric[check_config][target]") {
+  if (target.name === "metric[check_config][check_kind]" || target.name === "metric[check_config][target]" || target.name === "metric[check_config][compare_to]") {
     var scope = target.closest('[data-ck-metric-editor="check"]') || target.closest("form");
     ckApplyCheckFields(scope);
   }

data/app/assets/stylesheets/completion_kit/application.css

@@ -52,6 +52,7 @@
   --ck-sans: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
   --ck-radius: 6px;
   --ck-radius-lg: 10px;
+  --ck-field-gap: 1.15rem;
 }
 
 *,
@@ -1613,10 +1614,32 @@ tr:hover .ck-chip--publish {
 
 .ck-form-card {
   display: grid;
-  gap: 1rem;
+  gap: var(--ck-field-gap);
   max-width: 36rem;
 }
 
+.ck-form-card [data-ck-metric-type],
+.ck-form-card [data-ck-metric-editor] {
+  display: grid;
+  gap: var(--ck-field-gap);
+  align-content: start;
+}
+
+.ck-form-card [data-ck-metric-type] > .ck-section-title + .ck-hint,
+.ck-form-card [data-ck-metric-editor] > .ck-section-title + .ck-hint {
+  margin-top: calc(0.35rem - var(--ck-field-gap));
+}
+
+.ck-form-card [data-ck-metric-type] > .ck-radio,
+.ck-form-card [data-ck-metric-editor] > .ck-checkbox {
+  margin-top: 0;
+}
+
+.ck-form-card [data-ck-metric-type] > .ck-radio + .ck-radio,
+.ck-form-card [data-ck-metric-editor] > .ck-checkbox + .ck-checkbox {
+  margin-top: calc(0.4rem - var(--ck-field-gap));
+}
+
 .ck-version-note {
   margin: 0 0 1rem;
   max-width: 36rem;
@@ -1922,6 +1945,10 @@ label.ck-checkbox input {
   cursor: pointer;
 }
 
+label.ck-checkbox[hidden] {
+  display: none;
+}
+
 .ck-field-row {
   display: flex;
   gap: 1rem;

data/app/controllers/completion_kit/api/v1/metrics_controller.rb

@@ -70,7 +70,7 @@ module CompletionKit
         def metric_params
           params.permit(:name, :instruction, :metric_type,
             rubric_bands: [:stars, :description],
-            check_config: %i[check_kind target target_path value pattern json_path expected min max case_sensitive multiline trim],
+            check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max case_sensitive multiline trim],
             tag_names: [])
         end
       end

data/app/controllers/completion_kit/metrics_controller.rb

@@ -247,7 +247,7 @@ module CompletionKit
     def metric_params
       permitted = params.require(:metric).permit(:name, :instruction, :metric_type,
         rubric_bands: [:stars, :description],
-        check_config: %i[check_kind target target_path value pattern json_path expected min max case_sensitive multiline trim],
+        check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max case_sensitive multiline trim],
         tag_names: [])
       permitted[:check_config] = normalize_check_config(permitted[:check_config]) if permitted.key?(:check_config)
       permitted
@@ -258,6 +258,12 @@ module CompletionKit
       %w[min max].each { |key| hash[key] = hash[key].to_i if hash[key].present? }
       %w[case_sensitive multiline trim].each { |key| hash[key] = ActiveModel::Type::Boolean.new.cast(hash[key]) if hash.key?(key) }
       hash["expected"] = coerce_scalar(hash["expected"]) if hash["expected"].present?
+      if hash["compare_to"] == "expected"
+        hash.delete("value")
+      else
+        hash.delete("compare_to")
+        hash.delete("expected_path")
+      end
       hash.reject { |_, value| value.nil? || value == "" }
     end
 

data/app/helpers/completion_kit/application_helper.rb

@@ -180,6 +180,11 @@ module CompletionKit
       "no_refusal" => "Is not a refusal"
     }.freeze
 
+    CHECK_COMPARE_TO_LABELS = {
+      "constant" => "A value you type",
+      "expected" => "Each row's expected value"
+    }.freeze
+
     CHECK_TARGET_LABELS = {
       "response_text" => "The response text",
       "input_data" => "The input row",
@@ -191,6 +196,8 @@ module CompletionKit
       "pattern" => "Pattern",
       "json_path" => "JSON path",
       "expected" => "Expected value",
+      "expected_path" => "Field in the answer key",
+      "compare_to" => "Compare against",
       "target_path" => "Path into the JSON",
       "min" => "Shortest allowed",
       "max" => "Longest allowed",
@@ -211,6 +218,12 @@ module CompletionKit
       CHECK_FIELD_LABELS.fetch(key.to_s) { key.to_s.humanize }
     end
 
+    def ck_check_field_value(key, value)
+      return CHECK_COMPARE_TO_LABELS.fetch(value.to_s, value) if key.to_s == "compare_to"
+
+      value
+    end
+
     def ck_result_change_badge(change)
       case change
       when "broke"

data/app/jobs/completion_kit/check_review_job.rb

@@ -41,7 +41,16 @@ module CompletionKit
         return Checks::Result.new(passed: false, detail: "could not resolve target")
       end
 
-      Checks::Registry.fetch(config["check_kind"]).call(target_value, config)
+      kind = config["check_kind"]
+      if config["compare_to"] == "expected" && Checks::Registry.compares_value?(kind)
+        expected_value = Checks::ExpectedResolver.call(response, config)
+        if expected_value.equal?(Checks::ExpectedResolver::UNRESOLVED)
+          return Checks::Result.new(passed: false, detail: "no expected value for this row")
+        end
+        config = config.merge("value" => expected_value)
+      end
+
+      Checks::Registry.fetch(kind).call(target_value, config)
     end
 
     def record_terminal_failure!(error)

data/app/models/completion_kit/metric.rb

@@ -132,10 +132,23 @@ module CompletionKit
       end
 
       validate_check_target(config)
+      validate_check_compare_to(config, kind)
       validate_check_required_keys(config, kind)
       validate_check_kind_rules(config, kind)
     end
 
+    def validate_check_compare_to(config, kind)
+      return unless config.key?("compare_to")
+
+      unless %w[constant expected].include?(config["compare_to"])
+        errors.add(:check_config, "compare_to must be constant or expected")
+        return
+      end
+      if config["compare_to"] == "expected" && !CompletionKit::Checks::Registry.compares_value?(kind)
+        errors.add(:check_config, "compare_to expected only applies to contains, not_contains, or equals")
+      end
+    end
+
     def validate_check_target(config)
       target = config["target"].presence || "response_text"
       unless CompletionKit::Checks::TargetResolver::TARGETS.include?(target)
@@ -147,7 +160,9 @@ module CompletionKit
     end
 
     def validate_check_required_keys(config, kind)
-      CompletionKit::Checks::Registry.required_keys.fetch(kind).each do |required_key|
+      required = CompletionKit::Checks::Registry.required_keys.fetch(kind)
+      required -= %w[value] if config["compare_to"] == "expected" && CompletionKit::Checks::Registry.compares_value?(kind)
+      required.each do |required_key|
         if required_key == "expected"
           errors.add(:check_config, "expected is required") unless config.key?("expected")
         elsif config[required_key].to_s.strip.empty?

data/app/services/completion_kit/checks/expected_resolver.rb

@@ -0,0 +1,17 @@
+module CompletionKit
+  module Checks
+    module ExpectedResolver
+      UNRESOLVED = TargetResolver::UNRESOLVED
+
+      def self.call(response, config)
+        raw = response.expected_output
+        return UNRESOLVED if raw.blank?
+
+        path = config["expected_path"].to_s.strip
+        return raw.to_s if path.empty?
+
+        TargetResolver.resolve_json_path(raw.to_s, path)
+      end
+    end
+  end
+end

data/app/services/completion_kit/checks/registry.rb

@@ -23,6 +23,8 @@ module CompletionKit
         "no_refusal" => []
       }.freeze
 
+      VALUE_COMPARISON_KINDS = %w[contains not_contains equals].freeze
+
       KINDS = CHECKS.keys.freeze
 
       def self.kinds
@@ -33,6 +35,10 @@ module CompletionKit
         REQUIRED_KEYS
       end
 
+      def self.compares_value?(kind)
+        VALUE_COMPARISON_KINDS.include?(kind)
+      end
+
       def self.fetch(kind)
         CHECKS.fetch(kind).new
       end

data/app/services/completion_kit/mcp_tools/metrics.rb

@@ -13,6 +13,8 @@ module CompletionKit
           pattern: {type: "string"},
           json_path: {type: "string"},
           expected: {},
+          compare_to: {type: "string", enum: %w[constant expected]},
+          expected_path: {type: "string"},
           min: {type: "integer"},
           max: {type: "integer"},
           case_sensitive: {type: "boolean"},
@@ -23,7 +25,8 @@ module CompletionKit
 
       CHECK_CONFIG_HINT = "For a deterministic check set metric_type:\"check\" and check_config. Per-kind required keys: " \
         "value (contains/not_contains/equals), pattern (regex), json_path+expected (json_path_equals), " \
-        "min and/or max (length_bounds); valid_json and no_refusal take no extra keys. target_path is required when target is json_path."
+        "min and/or max (length_bounds); valid_json and no_refusal take no extra keys. target_path is required when target is json_path. " \
+        "For contains, not_contains, and equals, set compare_to:\"expected\" to grade against each row's own expected_output (ground truth) instead of a constant value (drop value); add expected_path to dig into the expected value when it is JSON."
 
       TOOLS = {
         "metrics_list" => {

data/app/services/completion_kit/starter_metrics.rb

@@ -96,6 +96,14 @@ module CompletionKit
         catches: "A required marker, citation, or keyword the output must always include. Set the value to the token you require.",
         metric_type: "check",
         check_config: { "check_kind" => "contains", "target" => "response_text", "value" => "REQUIRED" }
+      ),
+      Starter.new(
+        key: "matches_expected",
+        name: "Matches the answer key",
+        description: "Does the output equal this row's own expected value?",
+        catches: "Extraction and classification against ground truth, where every row has its own correct answer in the expected_output column. Exact match, deterministic, no judge call.",
+        metric_type: "check",
+        check_config: { "check_kind" => "equals", "target" => "response_text", "compare_to" => "expected" }
       )
     ].freeze
 

data/app/views/completion_kit/metrics/_check_spec.html.erb

@@ -11,7 +11,13 @@
   <% config.except("check_kind", "target").each do |key, value| %>
     <div class="ck-check-spec__row">
       <dt class="ck-check-spec__term"><%= ck_check_field_label(key) %></dt>
-      <dd class="ck-check-spec__val"><code class="ck-check-spec__code"><%= value %></code></dd>
+      <dd class="ck-check-spec__val">
+        <% if key == "compare_to" %>
+          <%= ck_check_field_value(key, value) %>
+        <% else %>
+          <code class="ck-check-spec__code"><%= value %></code>
+        <% end %>
+      </dd>
     </div>
   <% end %>
 </dl>

data/app/views/completion_kit/metrics/_form.html.erb

@@ -183,6 +183,19 @@
         <input type="text" name="metric[check_config][target_path]" id="metric_check_target_path" class="ck-input" value="<%= check["target_path"] %>">
       </div>
 
+      <div class="ck-field" data-ck-check-field="compare_to">
+        <p class="ck-label">Compare against</p>
+        <p class="ck-hint">Check every output against one fixed value, or against each row's own expected value (its answer key).</p>
+        <label class="ck-radio">
+          <input type="radio" name="metric[check_config][compare_to]" value="constant" <%= "checked" unless check["compare_to"] == "expected" %>>
+          <span>A value I type</span>
+        </label>
+        <label class="ck-radio">
+          <input type="radio" name="metric[check_config][compare_to]" value="expected" <%= "checked" if check["compare_to"] == "expected" %>>
+          <span>Each row's expected value</span>
+        </label>
+      </div>
+
       <div class="ck-field" data-ck-check-field="value">
         <label class="ck-label" for="metric_check_value">Text to look for</label>
         <input type="text" name="metric[check_config][value]" id="metric_check_value" class="ck-input" value="<%= check["value"] %>">
@@ -206,6 +219,12 @@
         <input type="text" name="metric[check_config][expected]" id="metric_check_expected" class="ck-input" value="<%= check["expected"] %>">
       </div>
 
+      <div class="ck-field" data-ck-check-field="expected_path">
+        <label class="ck-label" for="metric_check_expected_path">Field in the answer key</label>
+        <p class="ck-hint">Optional. Each row is compared to its own expected value. If that answer key is JSON, dig into it here, like vin. Leave blank to use the whole expected value.</p>
+        <input type="text" name="metric[check_config][expected_path]" id="metric_check_expected_path" class="ck-input" value="<%= check["expected_path"] %>">
+      </div>
+
       <div class="ck-field-row">
         <div class="ck-field" data-ck-check-field="min">
           <label class="ck-label" for="metric_check_min">Shortest allowed</label>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.21.1"
+  VERSION = "0.22.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: 0.22.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -302,6 +302,7 @@ files:
 - app/services/completion_kit/api_config.rb
 - app/services/completion_kit/checks/contains.rb
 - app/services/completion_kit/checks/equals.rb
+- app/services/completion_kit/checks/expected_resolver.rb
 - app/services/completion_kit/checks/json_path_equals.rb
 - app/services/completion_kit/checks/length_bounds.rb
 - app/services/completion_kit/checks/no_refusal.rb