completion-kit 0.17.1 → 0.18.0

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -224,8 +224,8 @@
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/metrics</p>
-        <p class="ck-meta-copy">Create a metric.</p>
-        <p class="ck-api-params"><strong>Required:</strong>&ensp;<code>name</code>&emsp;<strong>Optional:</strong>&ensp;<code>instruction</code>, <code>rubric_bands</code> (array of {stars, description})</p>
+        <p class="ck-meta-copy">Create a metric. Defaults to an LLM judge; pass <code>metric_type</code> <code>check</code> with a <code>check_config</code> for a deterministic pass/fail check.</p>
+        <p class="ck-api-params"><strong>Required:</strong>&ensp;<code>name</code>&emsp;<strong>Optional:</strong>&ensp;<code>metric_type</code> (llm_judge default, or check), <code>instruction</code> and <code>rubric_bands</code> (array of {stars, description}) for judges, <code>check_config</code> (<code>check_kind</code>, <code>target</code>, plus per-kind keys) for checks</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/metrics \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"name\": \"relevance\", \"instruction\": \"Is the response relevant?\"}'" %>
       </div>
       <div class="ck-api-endpoint">
@@ -259,11 +259,11 @@
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span> /api/v1/metrics/:metric_id/metric_versions/:id</p>
-        <p class="ck-meta-copy">Get a single version with its instruction, rubric bands, state, and source.</p>
+        <p class="ck-meta-copy">Get a single version with its instruction and rubric bands (or check_config for checks), state, and source.</p>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/metrics/:metric_id/metric_versions/:id/publish</p>
-        <p class="ck-meta-copy">Publish the version as current. Works for a draft (promote) or a superseded published version (revert). Copies the version's instruction and rubric back onto the metric.</p>
+        <p class="ck-meta-copy">Publish the version as current. Works for a draft (promote) or a superseded published version (revert). Copies the version's instruction and rubric (or check_config for checks) back onto the metric.</p>
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip" style="color: var(--ck-danger);">DELETE</span> /api/v1/metrics/:metric_id/metric_versions/:id</p>

data/app/views/completion_kit/metrics/_check_spec.html.erb

@@ -0,0 +1,17 @@
+<% config = local_assigns.fetch(:config) %>
+<dl class="ck-check-spec">
+  <div class="ck-check-spec__row">
+    <dt class="ck-check-spec__term">Kind</dt>
+    <dd class="ck-check-spec__val"><code class="ck-check-spec__code"><%= config["check_kind"] %></code></dd>
+  </div>
+  <div class="ck-check-spec__row">
+    <dt class="ck-check-spec__term">Target</dt>
+    <dd class="ck-check-spec__val"><code class="ck-check-spec__code"><%= config["target"] %></code></dd>
+  </div>
+  <% config.except("check_kind", "target").each do |key, value| %>
+    <div class="ck-check-spec__row">
+      <dt class="ck-check-spec__term"><%= key.humanize %></dt>
+      <dd class="ck-check-spec__val"><code class="ck-check-spec__code"><%= value %></code></dd>
+    </div>
+  <% end %>
+</dl>

data/app/views/completion_kit/metrics/_form.html.erb

@@ -66,7 +66,29 @@
       <%= ck_field_error(form, :name) %>
     </div>
 
-    <div class="ck-field ck-field--spacious">
+    <% editing = metric.persisted? %>
+    <% show_judge = !editing || metric.llm_judge? %>
+    <% show_check = !editing || metric.check? %>
+
+    <% if editing %>
+      <%= form.hidden_field :metric_type %>
+    <% else %>
+      <div class="ck-field" data-ck-metric-type>
+        <p class="ck-section-title">Metric type</p>
+        <p class="ck-hint">An LLM judge scores each output on a 1-5 rubric. A deterministic check passes or fails with no model call.</p>
+        <label class="ck-radio">
+          <%= form.radio_button :metric_type, "llm_judge", checked: !metric.check? %>
+          <span>LLM judge (1-5)</span>
+        </label>
+        <label class="ck-radio">
+          <%= form.radio_button :metric_type, "check", checked: metric.check? %>
+          <span>Deterministic check</span>
+        </label>
+      </div>
+    <% end %>
+
+    <% if show_judge %>
+    <div class="ck-field ck-field--spacious" data-ck-metric-editor="llm_judge">
       <p class="ck-section-title">Instruction</p>
       <p class="ck-hint">What should the judge assess? This instruction is sent to the LLM judge when scoring outputs.</p>
       <%= form.text_area :instruction, rows: 8, class: "ck-input ck-input--area", placeholder: "Evaluate whether the output...", **ck_field_aria(form, :instruction) %>
@@ -129,6 +151,87 @@
         <% end %>
       </div>
     </div>
+    <% end %>
+
+    <% if show_check %>
+    <% check = metric.check_config || {} %>
+    <div class="ck-field ck-field--spacious" data-ck-metric-editor="check">
+      <p class="ck-section-title">Check</p>
+      <p class="ck-hint">A deterministic pass/fail rule. Fill only the fields the chosen kind needs.</p>
+
+      <div class="ck-field">
+        <label class="ck-label" for="metric_check_kind">Check kind</label>
+        <select name="metric[check_config][check_kind]" id="metric_check_kind" class="ck-input">
+          <% CompletionKit::Checks::Registry.kinds.each do |kind| %>
+            <option value="<%= kind %>"<%= " selected" if check["check_kind"] == kind %>><%= kind %></option>
+          <% end %>
+        </select>
+      </div>
+
+      <div class="ck-field">
+        <label class="ck-label" for="metric_check_target">Target</label>
+        <select name="metric[check_config][target]" id="metric_check_target" class="ck-input">
+          <% CompletionKit::Checks::TargetResolver::TARGETS.each do |target| %>
+            <option value="<%= target %>"<%= " selected" if check["target"] == target %>><%= target %></option>
+          <% end %>
+        </select>
+      </div>
+
+      <div class="ck-field">
+        <label class="ck-label" for="metric_check_target_path">Target path</label>
+        <p class="ck-hint">Used when target is json_path, e.g. data.items.0.name.</p>
+        <input type="text" name="metric[check_config][target_path]" id="metric_check_target_path" class="ck-input" value="<%= check["target_path"] %>">
+      </div>
+
+      <div class="ck-field">
+        <label class="ck-label" for="metric_check_value">Value</label>
+        <p class="ck-hint">The substring or exact string for contains, not_contains, or equals.</p>
+        <input type="text" name="metric[check_config][value]" id="metric_check_value" class="ck-input" value="<%= check["value"] %>">
+      </div>
+
+      <div class="ck-field">
+        <label class="ck-label" for="metric_check_pattern">Pattern</label>
+        <p class="ck-hint">A regular expression for the regex kind.</p>
+        <input type="text" name="metric[check_config][pattern]" id="metric_check_pattern" class="ck-input" value="<%= check["pattern"] %>">
+      </div>
+
+      <div class="ck-field">
+        <label class="ck-label" for="metric_check_json_path">JSON path</label>
+        <p class="ck-hint">Dotted path into parsed JSON for json_path_equals.</p>
+        <input type="text" name="metric[check_config][json_path]" id="metric_check_json_path" class="ck-input" value="<%= check["json_path"] %>">
+      </div>
+
+      <div class="ck-field">
+        <label class="ck-label" for="metric_check_expected">Expected</label>
+        <p class="ck-hint">The value the JSON path must equal.</p>
+        <input type="text" name="metric[check_config][expected]" id="metric_check_expected" class="ck-input" value="<%= check["expected"] %>">
+      </div>
+
+      <div class="ck-field-row">
+        <div class="ck-field">
+          <label class="ck-label" for="metric_check_min">Min length</label>
+          <input type="number" name="metric[check_config][min]" id="metric_check_min" class="ck-input" value="<%= check["min"] %>">
+        </div>
+        <div class="ck-field">
+          <label class="ck-label" for="metric_check_max">Max length</label>
+          <input type="number" name="metric[check_config][max]" id="metric_check_max" class="ck-input" value="<%= check["max"] %>">
+        </div>
+      </div>
+
+      <label class="ck-checkbox">
+        <input type="checkbox" name="metric[check_config][case_sensitive]" value="true"<%= " checked" if check["case_sensitive"] %>>
+        <span>Case sensitive</span>
+      </label>
+      <label class="ck-checkbox">
+        <input type="checkbox" name="metric[check_config][multiline]" value="true"<%= " checked" if check["multiline"] %>>
+        <span>Multiline</span>
+      </label>
+      <label class="ck-checkbox">
+        <input type="checkbox" name="metric[check_config][trim]" value="true"<%= " checked" if check["trim"] %>>
+        <span>Trim whitespace</span>
+      </label>
+    </div>
+    <% end %>
 
     <%= render "completion_kit/tags/picker", record: metric, param_namespace: :metric %>
 

data/app/views/completion_kit/metrics/index.html.erb

@@ -1,7 +1,7 @@
 <section class="ck-page-header">
   <div>
     <h1 class="ck-title">Metrics</h1>
-    <p class="ck-lead">Scoring dimensions the judge uses to evaluate each response. Each metric defines one thing to assess and a 1-5 scale.</p>
+    <p class="ck-lead">Scoring dimensions used to evaluate each response. Each defines one thing to assess, by an LLM judge on a 1-5 scale or a deterministic check.</p>
   </div>
   <div class="ck-actions">
     <%= link_to "New metric", new_metric_path, class: ck_button_classes(:dark) %>
@@ -29,6 +29,7 @@
         <tr onclick="window.location='<%= metric_path(metric) %>'" style="cursor: pointer;">
           <td>
             <%= link_to metric_path(metric), class: "ck-record-name" do %><strong><%= metric.name %></strong><% end %>
+            <span class="ck-chip ck-chip--soft ck-metric-type-chip"><%= metric.check? ? "Check" : "Judge" %></span>
             <% if metric.tags.any? %>
               <div class="tag-marks-row">
                 <%= render "completion_kit/tags/marks", tags: metric.tags %>
@@ -69,7 +70,7 @@
   <% if @available_starters.any? %>
     <section class="ck-starter-row">
       <p class="ck-kicker">Skip the blank page</p>
-      <p class="ck-meta-copy">Pre-written rubrics for the dimensions most teams score against. Click a card to preview before it's created.</p>
+      <p class="ck-meta-copy">Pre-written metrics for the dimensions most teams score against, both LLM judges and deterministic checks. Click a card to preview before it's created.</p>
       <div class="ck-starter-grid">
         <% @available_starters.each do |starter| %>
           <%= render "starter_card", starter: starter %>
@@ -85,7 +86,7 @@
   <% if @available_starters.any? %>
     <section class="ck-starter-row ck-starter-row--empty-state">
       <h2 class="ck-title ck-title--sm">Skip the blank page</h2>
-      <p class="ck-lead">Five rubrics we've worked through for common evaluation dimensions. Adopt one to drop in a pre-written 1&ndash;5 scale, edit anything after. Or <%= link_to "write your own from scratch", new_metric_path, class: "ck-link" %>.</p>
+      <p class="ck-lead">Starter metrics we've worked through for common evaluation dimensions, both LLM judges and deterministic checks. Adopt one to drop in a pre-written metric, edit anything after. Or <%= link_to "write your own from scratch", new_metric_path, class: "ck-link" %>.</p>
       <div class="ck-starter-grid">
         <% @available_starters.each do |starter| %>
           <%= render "starter_card", starter: starter %>

data/app/views/completion_kit/metrics/show.html.erb

@@ -24,6 +24,13 @@
 </section>
 <% end %>
 
+<% if @metric.check? %>
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">Check</p>
+  <p class="ck-meta-copy">A deterministic pass/fail rule. No judge call.</p>
+  <%= render "completion_kit/metrics/check_spec", config: @metric.check_config %>
+</section>
+<% else %>
 <section class="ck-card ck-card--spaced">
   <p class="ck-kicker">Rubric<%= render "completion_kit/metrics/rubric_hint" %></p>
   <div class="ck-rubric-display">
@@ -41,6 +48,7 @@
     <% end %>
   </div>
 </section>
+<% end %>
 
 <% if CompletionKit.config.judge_agreement_enabled && @versions.any? %>
   <% predecessor_of = @versions.index_with { |v| @versions.detect { |o| o.version_number < v.version_number } } %>
@@ -184,7 +192,7 @@
   <section class="ck-card ck-card--spaced">
     <div class="ck-prompt-preview__header">
       <p class="ck-kicker">Agreement</p>
-      <% if draft.nil? && @improve_disagreement_count.positive? %>
+      <% if !@metric.check? && draft.nil? && @improve_disagreement_count.positive? %>
         <%= button_to suggest_variants_metric_path(@metric),
               method: :post, form_class: "inline-block",
               class: ck_button_classes(:light, variant: :outline) + " ck-button--sm",
@@ -194,19 +202,23 @@
         <% end %>
       <% end %>
     </div>
-    <%= turbo_stream_from "metric_#{@metric.id}_suggestion" %>
-    <div id="ck-suggestion-status-<%= @metric.id %>" class="ck-suggestion-status"></div>
-    <p class="ck-meta-copy">How often the judge lands on the same score you would. Review its scores to build that signal, and improve the metric to raise it.</p>
-    <%= render "completion_kit/agreements/trust_panel",
-          stats: CompletionKit::MetricAgreementStats.for(@metric),
-          metric: @metric %>
-    <% if CompletionKit.config.judge_examples_from_reviews %>
-      <%= render "completion_kit/metrics/guiding_examples", metric: @metric, examples: @guiding_examples %>
-    <% end %>
-    <% if draft %>
-      <div class="ck-cal-foot">
-        <span class="ck-cal-foot__note">A draft improvement (<%= draft.version_label %>) is waiting in the Versions table above. Open its change to compare, then Publish to use it.</span>
-      </div>
+    <% if @metric.check? %>
+      <p class="ck-meta-copy">This is a deterministic check, so there is nothing to calibrate.</p>
+    <% else %>
+      <%= turbo_stream_from "metric_#{@metric.id}_suggestion" %>
+      <div id="ck-suggestion-status-<%= @metric.id %>" class="ck-suggestion-status"></div>
+      <p class="ck-meta-copy">How often the judge lands on the same score you would. Review its scores to build that signal, and improve the metric to raise it.</p>
+      <%= render "completion_kit/agreements/trust_panel",
+            stats: CompletionKit::MetricAgreementStats.for(@metric),
+            metric: @metric %>
+      <% if CompletionKit.config.judge_examples_from_reviews %>
+        <%= render "completion_kit/metrics/guiding_examples", metric: @metric, examples: @guiding_examples %>
+      <% end %>
+      <% if draft %>
+        <div class="ck-cal-foot">
+          <span class="ck-cal-foot__note">A draft improvement (<%= draft.version_label %>) is waiting in the Versions table above. Open its change to compare, then Publish to use it.</span>
+        </div>
+      <% end %>
     <% end %>
   </section>
 <% end %>

data/app/views/completion_kit/metrics/starter_preview.html.erb

@@ -14,6 +14,13 @@
   <p class="ck-copy"><%= @starter.catches %></p>
 </section>
 
+<% if @starter.metric_type == "check" %>
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">Check</p>
+  <p class="ck-meta-copy">A deterministic pass/fail rule. No judge call.</p>
+  <%= render "completion_kit/metrics/check_spec", config: @starter.check_config %>
+</section>
+<% else %>
 <section class="ck-card ck-card--spaced">
   <p class="ck-kicker">Judge instruction</p>
   <p class="ck-copy"><%= @starter.instruction %></p>
@@ -36,6 +43,7 @@
     <% end %>
   </div>
 </section>
+<% end %>
 
 <div class="ck-starter-actions">
   <%= link_to "Cancel", metrics_path, class: ck_button_classes(:light, variant: :outline) %>

data/app/views/completion_kit/responses/show.html.erb

@@ -124,7 +124,7 @@
           <% if review.ai_feedback.present? %>
             <p class="ck-review-card__feedback"><%= review.ai_feedback %></p>
           <% end %>
-          <% if CompletionKit.config.judge_agreement_enabled && review.metric && review.ai_score %>
+          <% if CompletionKit.config.judge_agreement_enabled && review.metric && review.ai_score && review.metric_version&.llm_judge? %>
             <% existing = CompletionKit::Agreement.find_by(
                  response_id: @response.id, metric_id: review.metric_id,
                  created_by: CompletionKit.config.username.presence || "operator"

data/db/migrate/20260629000001_add_check_type_to_completion_kit_metrics.rb

@@ -0,0 +1,6 @@
+class AddCheckTypeToCompletionKitMetrics < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_metrics, :metric_type, :string, null: false, default: "llm_judge"
+    add_column :completion_kit_metrics, :check_config, :text
+  end
+end

data/db/migrate/20260629000002_add_check_type_to_completion_kit_metric_versions.rb

@@ -0,0 +1,6 @@
+class AddCheckTypeToCompletionKitMetricVersions < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_metric_versions, :metric_type, :string, null: false, default: "llm_judge"
+    add_column :completion_kit_metric_versions, :check_config, :text
+  end
+end

data/db/migrate/20260629000003_add_passed_to_completion_kit_reviews.rb

@@ -0,0 +1,5 @@
+class AddPassedToCompletionKitReviews < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_reviews, :passed, :boolean
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.17.1"
+  VERSION = "0.18.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.18.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -264,6 +264,7 @@ files:
 - app/controllers/concerns/completion_kit/tag_filtering.rb
 - app/helpers/completion_kit/application_helper.rb
 - app/jobs/completion_kit/application_job.rb
+- app/jobs/completion_kit/check_review_job.rb
 - app/jobs/completion_kit/generate_row_job.rb
 - app/jobs/completion_kit/judge_review_job.rb
 - app/jobs/completion_kit/metric_suggestion_job.rb
@@ -296,6 +297,17 @@ files:
 - app/services/completion_kit/agreement_math.rb
 - app/services/completion_kit/anthropic_client.rb
 - app/services/completion_kit/api_config.rb
+- app/services/completion_kit/checks/contains.rb
+- app/services/completion_kit/checks/equals.rb
+- app/services/completion_kit/checks/json_path_equals.rb
+- app/services/completion_kit/checks/length_bounds.rb
+- app/services/completion_kit/checks/no_refusal.rb
+- app/services/completion_kit/checks/not_contains.rb
+- app/services/completion_kit/checks/regex.rb
+- app/services/completion_kit/checks/registry.rb
+- app/services/completion_kit/checks/result.rb
+- app/services/completion_kit/checks/target_resolver.rb
+- app/services/completion_kit/checks/valid_json.rb
 - app/services/completion_kit/csv_processor.rb
 - app/services/completion_kit/dashboard_stats.rb
 - app/services/completion_kit/judge_service.rb
@@ -354,6 +366,7 @@ files:
 - app/views/completion_kit/metric_groups/index.html.erb
 - app/views/completion_kit/metric_groups/new.html.erb
 - app/views/completion_kit/metric_groups/show.html.erb
+- app/views/completion_kit/metrics/_check_spec.html.erb
 - app/views/completion_kit/metrics/_form.html.erb
 - app/views/completion_kit/metrics/_guiding_examples.html.erb
 - app/views/completion_kit/metrics/_rubric_diff.html.erb
@@ -449,6 +462,9 @@ files:
 - db/migrate/20260531000003_add_metric_version_fk_to_reviews.rb
 - db/migrate/20260531000004_rename_calibrations_to_agreements.rb
 - db/migrate/20260611000001_add_validation_to_completion_kit_suggestions.rb
+- db/migrate/20260629000001_add_check_type_to_completion_kit_metrics.rb
+- db/migrate/20260629000002_add_check_type_to_completion_kit_metric_versions.rb
+- db/migrate/20260629000003_add_passed_to_completion_kit_reviews.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb