completion-kit 0.5.33 → 0.5.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df835ff1e701f30b2d0b2889040f1d5fb0f09e5414ce4b50b8c2aeb0729a09b0
-  data.tar.gz: b104c726a3aacac06bfaf7332da69845a5d441b60c6d97cb70a3cf69ce6d50ed
+  metadata.gz: 7ec9f216056f47d007b8a512a009dea02aeaad328a9c4832ac0c1e122b816b15
+  data.tar.gz: 413f1b8e8ca28ed2c14e55210299a28bef42acb1a437a3ee9c3bd88b62d03bf9
 SHA512:
-  metadata.gz: 63a12da9635ce4bda787581edd983c4544a51ee0d2dd5816e0473107dc5a46e1b04d751e93dbb8452536daaf4c53961576996fe478fc2f59354fc4d4935dc749
-  data.tar.gz: b7a6b8b01d28e6dbbd0511ae85f9b3e19c813d623c6e0e8470ec82eae14310b30bf9bae3a0fd22adfac7a3414dda9bdb2c0e220d6a4bfc4952eb1c1acc4bfe38
+  metadata.gz: c837fd6ddc33c5799145bac2a0b1dba4ca1df807365c621330b619c47607e666a30325b47ce6d34e2ecb93e13d4e14a9538d597e0d56e47b9f7a1a836432be0d
+  data.tar.gz: 06b9ac7200883cd19d11ff127898d52d940b3f7a110cd5e7744bfcb2abab797fa05e923ba76121f6c00c6aeb8ac5d7b5d5e8b79b07d773ad0c0d5650ec41b90d

data/app/assets/stylesheets/completion_kit/application.css.erb

@@ -5088,3 +5088,88 @@ a.tag-mark {
   letter-spacing: 0.04em;
   text-transform: uppercase;
 }
+
+.ck-record-name {
+  color: inherit;
+  text-decoration: none;
+}
+.ck-record-name:hover,
+.ck-record-name:focus-visible {
+  color: var(--ck-accent);
+}
+
+.ck-field-error {
+  color: var(--ck-error, #d93232);
+  font-size: 0.85rem;
+  margin: 4px 0 0;
+}
+
+.ck-visually-hidden {
+  position: absolute;
+  width: 1px;
+  height: 1px;
+  padding: 0;
+  margin: -1px;
+  overflow: hidden;
+  clip: rect(0 0 0 0);
+  white-space: nowrap;
+  border: 0;
+}
+.tag-mark:focus-within {
+  outline: 2px solid var(--ck-accent);
+  outline-offset: 2px;
+}
+
+.ck-calibration {
+  margin-top: 12px;
+  padding-top: 12px;
+  border-top: 1px dashed rgba(255, 255, 255, 0.08);
+}
+.ck-calibration__prompt {
+  font-size: 0.8rem;
+  color: var(--ck-dim);
+  margin: 0 0 8px;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+.ck-calibration__count {
+  font-size: 0.75rem;
+  color: var(--ck-accent);
+}
+.ck-calibration__buttons {
+  display: flex;
+  gap: 8px;
+  flex-wrap: wrap;
+}
+.ck-calibration__pill {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  padding: 6px 12px;
+  border-radius: 999px;
+  font-size: 0.85rem;
+  background: transparent;
+  border: 1px solid rgba(255, 255, 255, 0.18);
+  color: inherit;
+  cursor: pointer;
+}
+.ck-calibration__pill:hover,
+.ck-calibration__pill:focus-visible {
+  border-color: var(--ck-accent);
+}
+.ck-calibration__pill.is-active {
+  background: var(--ck-accent);
+  color: #0b1320;
+  border-color: var(--ck-accent);
+}
+.ck-calibration__detail {
+  margin-top: 10px;
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+}
+.ck-calibration__value {
+  color: var(--ck-accent);
+  font-weight: 600;
+}

data/app/controllers/completion_kit/api/v1/calibrations_controller.rb

@@ -0,0 +1,57 @@
+module CompletionKit
+  module Api
+    module V1
+      class CalibrationsController < BaseController
+        before_action :ensure_calibration_enabled
+        before_action :set_scope
+
+        def index
+          render json: scope_calibrations
+        end
+
+        def create
+          calibration = scope_calibrations.find_or_initialize_by(created_by: created_by_param)
+          calibration.assign_attributes(
+            run: @run,
+            response: @response,
+            metric: @metric,
+            judge_version: JudgeVersion.ensure_current_for(@metric),
+            **calibration_params
+          )
+
+          if calibration.save
+            render json: calibration, status: calibration.previously_new_record? ? :created : :ok
+          else
+            render json: { errors: calibration.errors }, status: :unprocessable_entity
+          end
+        end
+
+        private
+
+        def ensure_calibration_enabled
+          render(json: { error: "Calibration disabled" }, status: :not_found) unless CompletionKit.config.judge_calibration_enabled
+        end
+
+        def set_scope
+          @run = Run.find(params[:run_id])
+          @response = @run.responses.find(params[:response_id])
+          @metric = Metric.find(params[:metric_id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def scope_calibrations
+          Calibration.where(run_id: @run.id, response_id: @response.id, metric_id: @metric.id)
+        end
+
+        def calibration_params
+          params.permit(:verdict, :corrected_score, :note).to_h.symbolize_keys
+        end
+
+        def created_by_param
+          params[:created_by].presence || "api"
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/calibrations_controller.rb

@@ -0,0 +1,50 @@
+module CompletionKit
+  class CalibrationsController < ApplicationController
+    before_action :ensure_calibration_enabled
+    before_action :set_scope
+
+    def create
+      created_by = calibration_creator
+      calibration = Calibration.find_or_initialize_by(
+        run_id: @run.id, response_id: @response.id, metric_id: @metric.id, created_by: created_by
+      )
+      calibration.assign_attributes(
+        judge_version: JudgeVersion.ensure_current_for(@metric),
+        verdict: params[:verdict],
+        corrected_score: params[:corrected_score].presence,
+        note: params[:note].presence
+      )
+
+      if calibration.save
+        render turbo_stream: turbo_stream.replace(
+          "calibration_#{@response.id}_#{@metric.id}",
+          partial: "completion_kit/calibrations/buttons",
+          locals: { review: review_for_metric, calibration: calibration, run: @run, response_row: @response, metric: @metric }
+        )
+      else
+        flash[:alert] = calibration.errors.full_messages.to_sentence
+        redirect_to run_response_path(@run, @response)
+      end
+    end
+
+    private
+
+    def ensure_calibration_enabled
+      head :not_found unless CompletionKit.config.judge_calibration_enabled
+    end
+
+    def set_scope
+      @run = Run.find(params[:run_id])
+      @response = @run.responses.find(params[:response_id])
+      @metric = Metric.find(params[:metric_id])
+    end
+
+    def review_for_metric
+      @response.reviews.find_by(metric_id: @metric.id)
+    end
+
+    def calibration_creator
+      request.env["HTTP_X_REMOTE_USER"].presence || CompletionKit.config.username.presence || "operator"
+    end
+  end
+end

data/app/helpers/completion_kit/application_helper.rb

@@ -218,6 +218,20 @@ module CompletionKit
       "#{base_path}?#{{ tag: next_set.map(&:name) }.to_query}"
     end
 
+    def ck_field_aria(form, field)
+      return {} unless form.object.errors[field].any?
+      { "aria-invalid" => "true", "aria-describedby" => ck_field_error_id(form, field) }
+    end
+
+    def ck_field_error(form, field)
+      return nil unless form.object.errors[field].any?
+      content_tag(:p, form.object.errors[field].first, class: "ck-field-error", id: ck_field_error_id(form, field))
+    end
+
+    def ck_field_error_id(form, field)
+      "#{form.object.model_name.param_key}_#{field}_error"
+    end
+
     private
 
     def diff_tokens(old_text, new_text, side)

data/app/models/completion_kit/calibration.rb

@@ -0,0 +1,47 @@
+module CompletionKit
+  class Calibration < ApplicationRecord
+    VERDICTS = %w[agree disagree borderline].freeze
+
+    belongs_to :run
+    belongs_to :response
+    belongs_to :metric
+    belongs_to :judge_version
+
+    validates :verdict, presence: true, inclusion: { in: VERDICTS }
+    validates :response_id,
+              uniqueness: { scope: [:metric_id, :created_by] }
+    validate :corrected_score_required_when_disagreeing
+    validate :corrected_score_within_rubric
+
+    scope :for_run, ->(run_id) { where(run_id: run_id) }
+    scope :for_metric, ->(metric_id) { where(metric_id: metric_id) }
+
+    def as_json(options = {})
+      {
+        id: id,
+        run_id: run_id,
+        response_id: response_id,
+        metric_id: metric_id,
+        judge_version_id: judge_version_id,
+        verdict: verdict,
+        corrected_score: corrected_score,
+        note: note,
+        created_by: created_by,
+        created_at: created_at
+      }
+    end
+
+    private
+
+    def corrected_score_required_when_disagreeing
+      return unless verdict == "disagree"
+      errors.add(:corrected_score, "must be set when disagreeing with the judge") if corrected_score.blank?
+    end
+
+    def corrected_score_within_rubric
+      return if corrected_score.blank?
+      score = corrected_score.to_f
+      errors.add(:corrected_score, "must be between 1 and 5") unless score >= 1 && score <= 5
+    end
+  end
+end

data/app/models/completion_kit/judge_version.rb

@@ -0,0 +1,32 @@
+module CompletionKit
+  class JudgeVersion < ApplicationRecord
+    belongs_to :metric
+    has_many :calibrations, dependent: :destroy
+
+    serialize :rubric_bands, coder: JSON
+
+    validates :metric_id, presence: true
+
+    scope :current, -> { where(current: true) }
+
+    def self.ensure_current_for(metric)
+      current.find_by(metric_id: metric.id) || create!(
+        metric: metric,
+        instruction: metric.instruction,
+        rubric_bands: metric.rubric_bands,
+        current: true
+      )
+    end
+
+    def as_json(options = {})
+      {
+        id: id,
+        metric_id: metric_id,
+        instruction: instruction,
+        rubric_bands: rubric_bands,
+        current: current,
+        created_at: created_at
+      }
+    end
+  end
+end

data/app/models/completion_kit/provider_credential.rb

@@ -1,6 +1,3 @@
-require "ipaddr"
-require "resolv"
-
 module CompletionKit
   class ProviderCredential < ApplicationRecord
     include Turbo::Broadcastable
@@ -8,7 +5,7 @@ module CompletionKit
     PROVIDER_LABELS = {
       "openai" => "OpenAI",
       "anthropic" => "Anthropic",
-      "ollama" => "Ollama / local endpoint",
+      "ollama" => "Ollama / OpenAI-compatible endpoint",
       "openrouter" => "OpenRouter"
     }.freeze
 
@@ -139,29 +136,12 @@ module CompletionKit
     def api_endpoint_not_internal
       return if api_endpoint.blank?
 
-      uri = safe_http_uri(api_endpoint)
-      unless uri
+      issues = ProviderEndpoint.validate(api_endpoint)
+      if issues.include?(:invalid_url)
         errors.add(:api_endpoint, "must be a valid http or https URL")
-        return
-      end
-
-      if endpoint_addresses(uri.host).any? { |ip| ip.private? || ip.link_local? }
+      elsif issues.include?(:unsafe_host)
         errors.add(:api_endpoint, "must not point at a private or internal address")
       end
     end
-
-    def safe_http_uri(value)
-      uri = URI.parse(value.to_s.strip)
-      uri if uri.is_a?(URI::HTTP) && uri.host.present?
-    rescue URI::InvalidURIError
-      nil
-    end
-
-    def endpoint_addresses(host)
-      bare = host.delete_prefix("[").delete_suffix("]")
-      [IPAddr.new(bare)]
-    rescue IPAddr::InvalidAddressError
-      Resolv.getaddresses(host).map { |addr| IPAddr.new(addr) }
-    end
   end
 end

data/app/services/completion_kit/api_config.rb

@@ -1,5 +1,7 @@
 module CompletionKit
   class ApiConfig
+    PROVIDERS = %w[openai anthropic ollama openrouter].freeze
+
     def self.for_model(model_name)
       provider = provider_for_model(model_name)
       provider ? for_provider(provider) : {}
@@ -9,21 +11,25 @@ module CompletionKit
       provider = provider_name.to_s
       stored = ProviderCredential.find_by(provider: provider)&.config_hash || {}
 
-      defaults = case provider
-                 when "openai"
-                   { provider: "openai", api_key: CompletionKit.config.openai_api_key || ENV["OPENAI_API_KEY"] }
-                 when "anthropic"
-                   { provider: "anthropic", api_key: CompletionKit.config.anthropic_api_key || ENV["ANTHROPIC_API_KEY"] }
-                 when "ollama"
-                   {
-                     provider: "ollama",
-                     api_key: CompletionKit.config.ollama_api_key || ENV["OLLAMA_API_KEY"],
-                     api_endpoint: CompletionKit.config.ollama_api_endpoint || ENV["OLLAMA_API_ENDPOINT"]
-                   }
-                 when "openrouter"
-                   { provider: "openrouter", api_key: ENV["OPENROUTER_API_KEY"] }
+      defaults = if CompletionKit.config.tenant_scope
+                   PROVIDERS.include?(provider) ? { provider: provider } : {}
                  else
-                   {}
+                   case provider
+                   when "openai"
+                     { provider: "openai", api_key: CompletionKit.config.openai_api_key || ENV["OPENAI_API_KEY"] }
+                   when "anthropic"
+                     { provider: "anthropic", api_key: CompletionKit.config.anthropic_api_key || ENV["ANTHROPIC_API_KEY"] }
+                   when "ollama"
+                     {
+                       provider: "ollama",
+                       api_key: CompletionKit.config.ollama_api_key || ENV["OLLAMA_API_KEY"],
+                       api_endpoint: CompletionKit.config.ollama_api_endpoint || ENV["OLLAMA_API_ENDPOINT"]
+                     }
+                   when "openrouter"
+                     { provider: "openrouter", api_key: ENV["OPENROUTER_API_KEY"] }
+                   else
+                     {}
+                   end
                  end
 
       defaults.merge(stored.compact)

data/app/services/completion_kit/mcp_dispatcher.rb

@@ -33,7 +33,8 @@ module CompletionKit
         McpTools::Metrics.definitions +
         McpTools::MetricGroups.definitions +
         McpTools::ProviderCredentials.definitions +
-        McpTools::Tags.definitions
+        McpTools::Tags.definitions +
+        McpTools::Calibrations.definitions
     end
 
     def self.call_tool(name, arguments)
@@ -46,6 +47,7 @@ module CompletionKit
       when /\Ametric_groups_/        then McpTools::MetricGroups.call(name, arguments)
       when /\Aprovider_credentials_/ then McpTools::ProviderCredentials.call(name, arguments)
       when /\Atags_/                 then McpTools::Tags.call(name, arguments)
+      when /\Acalibrations_/         then McpTools::Calibrations.call(name, arguments)
       else raise MethodNotFound, "Unknown tool: #{name}"
       end
     end

data/app/services/completion_kit/mcp_tools/calibrations.rb

@@ -0,0 +1,73 @@
+module CompletionKit
+  module McpTools
+    module Calibrations
+      extend Base
+
+      TOOLS = {
+        "calibrations_list" => {
+          description: "List calibrations. Filter by run_id, response_id, metric_id, or created_by.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              run_id: {type: "integer"},
+              response_id: {type: "integer"},
+              metric_id: {type: "integer"},
+              created_by: {type: "string"}
+            },
+            required: []
+          },
+          handler: :list
+        },
+        "calibrations_create" => {
+          description: "Upsert a calibration for (run, response, metric, created_by). Verdict is one of agree, disagree, borderline. corrected_score (1..5) is required when verdict is 'disagree'.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              run_id: {type: "integer"},
+              response_id: {type: "integer"},
+              metric_id: {type: "integer"},
+              verdict: {type: "string", enum: %w[agree disagree borderline]},
+              corrected_score: {type: "number"},
+              note: {type: "string"},
+              created_by: {type: "string"}
+            },
+            required: ["run_id", "response_id", "metric_id", "verdict"]
+          },
+          handler: :create
+        }
+      }.freeze
+
+      def self.list(args)
+        scope = CompletionKit::Calibration.all
+        scope = scope.where(run_id: args["run_id"]) if args["run_id"]
+        scope = scope.where(response_id: args["response_id"]) if args["response_id"]
+        scope = scope.where(metric_id: args["metric_id"]) if args["metric_id"]
+        scope = scope.where(created_by: args["created_by"]) if args["created_by"]
+        text_result(scope.order(:created_at).map(&:as_json))
+      end
+
+      def self.create(args)
+        run = CompletionKit::Run.find(args["run_id"])
+        response = run.responses.find(args["response_id"])
+        metric = CompletionKit::Metric.find(args["metric_id"])
+        created_by = args["created_by"].presence || "mcp"
+
+        calibration = CompletionKit::Calibration.find_or_initialize_by(
+          run_id: run.id, response_id: response.id, metric_id: metric.id, created_by: created_by
+        )
+        calibration.assign_attributes(
+          judge_version: CompletionKit::JudgeVersion.ensure_current_for(metric),
+          verdict: args["verdict"],
+          corrected_score: args["corrected_score"],
+          note: args["note"]
+        )
+
+        if calibration.save
+          text_result(calibration.as_json)
+        else
+          error_result(calibration.errors.full_messages.join(", "))
+        end
+      end
+    end
+  end
+end

data/app/services/completion_kit/ollama_client.rb

@@ -7,6 +7,7 @@ module CompletionKit
     def generate_completion(prompt, options = {})
       @temperature_dropped = false
       return "Error: API endpoint not configured" unless configured?
+      return "Error: API endpoint resolves to a private address" unless ProviderEndpoint.safe?(api_endpoint)
 
       model = options[:model]
       max_tokens = options[:max_tokens] || 1000
@@ -44,6 +45,7 @@ module CompletionKit
 
     def available_models
       return [] unless configured?
+      return [] unless ProviderEndpoint.safe?(api_endpoint)
 
       response = build_connection(api_endpoint).get("/v1/models") do |req|
         req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
@@ -74,7 +76,9 @@ module CompletionKit
     end
 
     def api_endpoint
-      (@config[:api_endpoint] || ENV["OLLAMA_API_ENDPOINT"] || "http://localhost:11434/v1").to_s.delete_suffix("/")
+      raw = @config[:api_endpoint] || ENV["OLLAMA_API_ENDPOINT"]
+      raw ||= "http://localhost:11434/v1" if CompletionKit.config.allow_loopback_endpoints
+      raw.to_s.delete_suffix("/")
     end
 
     def post_completion(model:, prompt:, max_tokens:, temperature:)

data/app/services/completion_kit/provider_endpoint.rb

@@ -0,0 +1,47 @@
+require "ipaddr"
+require "resolv"
+
+module CompletionKit
+  module ProviderEndpoint
+    ZERO_NET = IPAddr.new("0.0.0.0/8").freeze
+
+    module_function
+
+    def validate(url)
+      uri = parse(url)
+      return [:invalid_url] unless uri
+      addrs = addresses(uri.host)
+      return [:unresolvable] if addrs.empty?
+      return [:unsafe_host] if addrs.any? { |ip| unsafe?(ip) }
+      []
+    end
+
+    def safe?(url)
+      errors = validate(url)
+      errors.empty? || errors == [:unresolvable]
+    end
+
+    def parse(value)
+      uri = URI.parse(value.to_s.strip)
+      uri if uri.is_a?(URI::HTTP) && uri.host.present?
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def addresses(host)
+      bare = host.delete_prefix("[").delete_suffix("]")
+      [IPAddr.new(bare)]
+    rescue IPAddr::InvalidAddressError
+      Resolv.getaddresses(host).map { |addr| IPAddr.new(addr) }
+    end
+
+    def unsafe?(ip)
+      return true if ip.private?
+      return true if ip.link_local?
+      return true if ip.to_i.zero?
+      return true if ip.ipv4? && ZERO_NET.include?(ip)
+      return true if ip.loopback? && !CompletionKit.config.allow_loopback_endpoints
+      false
+    end
+  end
+end

data/app/views/completion_kit/calibrations/_buttons.html.erb

@@ -0,0 +1,50 @@
+<div id="calibration_<%= response_row.id %>_<%= metric.id %>" class="ck-calibration">
+  <% current_verdict = calibration&.verdict %>
+  <% verdict_count = CompletionKit::Calibration.where(response_id: response_row.id, metric_id: metric.id).count %>
+  <p class="ck-calibration__prompt">
+    How does this score feel?
+    <% if verdict_count > 0 %>
+      <span class="ck-calibration__count"><%= pluralize(verdict_count, "verdict") %> collected</span>
+    <% end %>
+  </p>
+  <div class="ck-calibration__buttons">
+    <% CompletionKit::Calibration::VERDICTS.each do |verdict| %>
+      <%= button_to run_response_calibrations_path(run, response_row, metric_id: metric.id, verdict: verdict),
+              method: :post,
+              form: { data: { turbo: "true" } },
+              class: "ck-calibration__pill ck-calibration__pill--#{verdict}#{' is-active' if verdict == current_verdict}",
+              "aria-pressed": (verdict == current_verdict).to_s do %>
+        <% case verdict
+           when "agree" %>👍 Agree<% when "disagree" %>👎 Disagree<% else %>🤔 Borderline<% end %>
+      <% end %>
+    <% end %>
+  </div>
+
+  <% if current_verdict == "disagree" %>
+    <%= form_with url: run_response_calibrations_path(run, response_row),
+                  method: :post, local: false,
+                  class: "ck-calibration__detail" do |f| %>
+      <%= hidden_field_tag :metric_id, metric.id %>
+      <%= hidden_field_tag :verdict, "disagree" %>
+      <label class="ck-label">
+        Your score
+        <span class="ck-calibration__value" data-calibration-value><%= calibration.corrected_score || review&.ai_score || 3 %></span>
+      </label>
+      <input type="range" name="corrected_score" min="1" max="5" step="0.5"
+             value="<%= calibration.corrected_score || review&.ai_score || 3 %>"
+             oninput="this.closest('.ck-calibration__detail').querySelector('[data-calibration-value]').textContent = this.value"
+             class="ck-slider">
+      <textarea name="note" rows="2" class="ck-input ck-input--area" placeholder="Why? (optional)"><%= calibration.note %></textarea>
+      <%= f.submit "Save", class: ck_button_classes(:dark) %>
+    <% end %>
+  <% elsif current_verdict == "borderline" %>
+    <%= form_with url: run_response_calibrations_path(run, response_row),
+                  method: :post, local: false,
+                  class: "ck-calibration__detail" do |f| %>
+      <%= hidden_field_tag :metric_id, metric.id %>
+      <%= hidden_field_tag :verdict, "borderline" %>
+      <textarea name="note" rows="2" class="ck-input ck-input--area" placeholder="What made this borderline? (optional)"><%= calibration.note %></textarea>
+      <%= f.submit "Save", class: ck_button_classes(:dark) %>
+    <% end %>
+  <% end %>
+</div>

data/app/views/completion_kit/datasets/_form.html.erb

@@ -1,6 +1,6 @@
 <%= form_with(model: dataset, local: true) do |form| %>
   <% if dataset.errors.any? %>
-    <div class="ck-flash ck-flash--alert">
+    <div class="ck-flash ck-flash--alert" role="alert">
       <p class="ck-flash__title"><%= pluralize(dataset.errors.count, "problem") %> prevented this dataset from being saved.</p>
       <ul class="ck-error-list">
         <% dataset.errors.full_messages.each do |message| %>
@@ -13,12 +13,14 @@
   <div class="ck-card ck-form-card">
     <div class="ck-field">
       <%= form.label :name, "Name", class: "ck-label" %>
-      <%= form.text_field :name, class: "ck-input", placeholder: "Customer support tickets" %>
+      <%= form.text_field :name, class: "ck-input", placeholder: "Customer support tickets", **ck_field_aria(form, :name) %>
+      <%= ck_field_error(form, :name) %>
     </div>
 
     <div class="ck-field">
       <%= form.label :csv_data, "CSV data", class: "ck-label" %>
-      <%= form.text_area :csv_data, rows: 12, class: "ck-input ck-input--area ck-input--code", placeholder: "content,audience\nFirst ticket text,internal\nSecond ticket text,customer" %>
+      <%= form.text_area :csv_data, rows: 12, class: "ck-input ck-input--area ck-input--code", placeholder: "content,audience\nFirst ticket text,internal\nSecond ticket text,customer", **ck_field_aria(form, :csv_data) %>
+      <%= ck_field_error(form, :csv_data) %>
     </div>
 
     <%= render "completion_kit/tags/picker", record: dataset, param_namespace: :dataset %>

data/app/views/completion_kit/datasets/index.html.erb

@@ -17,18 +17,18 @@
   <table class="ck-results-table ck-datasets-table">
     <thead>
       <tr>
-        <th>Name</th>
-        <th>Rows</th>
-        <th>Used in</th>
-        <th>Created</th>
-        <th></th>
+        <th scope="col">Name</th>
+        <th scope="col">Rows</th>
+        <th scope="col">Used in</th>
+        <th scope="col">Created</th>
+        <th scope="col"></th>
       </tr>
     </thead>
     <tbody>
       <% @datasets.each do |dataset| %>
         <tr onclick="window.location='<%= dataset_path(dataset) %>'" style="cursor: pointer;">
           <td>
-            <strong><%= dataset.name %></strong>
+            <%= link_to dataset_path(dataset), class: "ck-record-name" do %><strong><%= dataset.name %></strong><% end %>
             <% if dataset.tags.any? %>
               <div class="tag-marks-row">
                 <%= render "completion_kit/tags/marks", tags: dataset.tags %>

data/app/views/completion_kit/metric_groups/index.html.erb

@@ -22,17 +22,17 @@
   <table class="ck-results-table ck-metric-groups-table">
     <thead>
       <tr>
-        <th>Name</th>
-        <th>Description</th>
-        <th>Members</th>
-        <th></th>
+        <th scope="col">Name</th>
+        <th scope="col">Description</th>
+        <th scope="col">Members</th>
+        <th scope="col"></th>
       </tr>
     </thead>
     <tbody>
       <% @metric_groups.each do |metric_group| %>
         <tr onclick="window.location='<%= metric_group_path(metric_group) %>'" style="cursor: pointer;">
           <td>
-            <strong><%= metric_group.name %></strong>
+            <%= link_to metric_group_path(metric_group), class: "ck-record-name" do %><strong><%= metric_group.name %></strong><% end %>
             <% if metric_group.tags.any? %>
               <div class="tag-marks-row">
                 <%= render "completion_kit/tags/marks", tags: metric_group.tags %>

data/app/views/completion_kit/metrics/_form.html.erb

@@ -1,6 +1,6 @@
 <%= form_with(model: metric, local: true) do |form| %>
   <% if metric.errors.any? %>
-    <div class="ck-flash ck-flash--alert">
+    <div class="ck-flash ck-flash--alert" role="alert">
       <p class="ck-flash__title"><%= pluralize(metric.errors.count, "problem") %> prevented this metric from being saved.</p>
       <ul class="ck-error-list">
         <% metric.errors.full_messages.each do |message| %>
@@ -13,13 +13,15 @@
   <div class="ck-card ck-form-card">
     <div class="ck-field">
       <%= form.label :name, "Metric name", class: "ck-label" %>
-      <%= form.text_field :name, class: "ck-input", placeholder: "Helpfulness" %>
+      <%= form.text_field :name, class: "ck-input", placeholder: "Helpfulness", **ck_field_aria(form, :name) %>
+      <%= ck_field_error(form, :name) %>
     </div>
 
     <div class="ck-field ck-field--spacious">
       <p class="ck-section-title">Instruction</p>
       <p class="ck-hint">What should the judge assess? This instruction is sent to the LLM judge when scoring outputs.</p>
-      <%= form.text_area :instruction, rows: 8, class: "ck-input ck-input--area", placeholder: "Evaluate whether the output..." %>
+      <%= form.text_area :instruction, rows: 8, class: "ck-input ck-input--area", placeholder: "Evaluate whether the output...", **ck_field_aria(form, :instruction) %>
+      <%= ck_field_error(form, :instruction) %>
     </div>
 
     <div class="ck-field ck-field--spacious">

data/app/views/completion_kit/metrics/index.html.erb

@@ -17,17 +17,17 @@
   <table class="ck-results-table ck-metrics-table">
     <thead>
       <tr>
-        <th>Name</th>
-        <th>Instruction</th>
-        <th>In groups</th>
-        <th></th>
+        <th scope="col">Name</th>
+        <th scope="col">Instruction</th>
+        <th scope="col">In groups</th>
+        <th scope="col"></th>
       </tr>
     </thead>
     <tbody>
       <% @metrics.each do |metric| %>
         <tr onclick="window.location='<%= metric_path(metric) %>'" style="cursor: pointer;">
           <td>
-            <strong><%= metric.name %></strong>
+            <%= link_to metric_path(metric), class: "ck-record-name" do %><strong><%= metric.name %></strong><% end %>
             <% if metric.tags.any? %>
               <div class="tag-marks-row">
                 <%= render "completion_kit/tags/marks", tags: metric.tags %>

data/app/views/completion_kit/prompts/_form.html.erb

@@ -1,6 +1,6 @@
 <%= form_with(model: prompt, local: true) do |form| %>
   <% if prompt.errors.any? %>
-    <div class="ck-flash ck-flash--alert">
+    <div class="ck-flash ck-flash--alert" role="alert">
       <p class="ck-flash__title"><%= pluralize(prompt.errors.count, "problem") %> prevented this prompt from being saved.</p>
       <ul class="ck-error-list">
         <% prompt.errors.full_messages.each do |message| %>
@@ -17,17 +17,20 @@
   <div class="ck-card ck-form-card">
     <div class="ck-field">
       <%= form.label :name, "Name", class: "ck-label" %>
-      <%= form.text_field :name, class: "ck-input", placeholder: "Support summary", autocomplete: "off", data: { "1p-ignore": "" } %>
+      <%= form.text_field :name, class: "ck-input", placeholder: "Support summary", autocomplete: "off", data: { "1p-ignore": "" }, **ck_field_aria(form, :name) %>
+      <%= ck_field_error(form, :name) %>
     </div>
 
     <div class="ck-field">
       <%= form.label :description, class: "ck-label" %>
-      <%= form.text_area :description, rows: 3, class: "ck-input ck-input--area", placeholder: "Short note about this prompt." %>
+      <%= form.text_area :description, rows: 3, class: "ck-input ck-input--area", placeholder: "Short note about this prompt.", **ck_field_aria(form, :description) %>
+      <%= ck_field_error(form, :description) %>
     </div>
 
     <div class="ck-field">
       <%= form.label :template, "Prompt text", class: "ck-label" %>
-      <%= form.text_area :template, rows: 12, class: "ck-input ck-input--area ck-input--code", placeholder: "Summarize {{content}} for {{audience}}" %>
+      <%= form.text_area :template, rows: 12, class: "ck-input ck-input--area ck-input--code", placeholder: "Summarize {{content}} for {{audience}}", **ck_field_aria(form, :template) %>
+      <%= ck_field_error(form, :template) %>
       <p class="ck-hint">Use <code>{{variable}}</code>. Match your dataset column names.</p>
     </div>
 

data/app/views/completion_kit/prompts/index.html.erb

@@ -17,19 +17,19 @@
   <table class="ck-results-table ck-prompts-table">
     <thead>
       <tr>
-        <th>Name</th>
-        <th>Version</th>
-        <th>Model</th>
-        <th>Best score</th>
-        <th>Runs</th>
-        <th></th>
+        <th scope="col">Name</th>
+        <th scope="col">Version</th>
+        <th scope="col">Model</th>
+        <th scope="col">Best score</th>
+        <th scope="col">Runs</th>
+        <th scope="col"></th>
       </tr>
     </thead>
     <tbody>
       <% @prompts.each do |prompt| %>
         <tr onclick="window.location='<%= prompt_path(prompt) %>'" style="cursor: pointer;">
           <td>
-            <strong><%= prompt.name %></strong>
+            <%= link_to prompt_path(prompt), class: "ck-record-name" do %><strong><%= prompt.name %></strong><% end %>
             <% if prompt.description.present? %>
               <p class="ck-prompts-table__desc"><%= truncate(prompt.description, length: 120) %></p>
             <% end %>

data/app/views/completion_kit/provider_credentials/_form.html.erb

@@ -1,6 +1,6 @@
 <%= form_with(model: provider_credential, local: true) do |form| %>
   <% if provider_credential.errors.any? %>
-    <div class="ck-flash ck-flash--alert">
+    <div class="ck-flash ck-flash--alert" role="alert">
       <p class="ck-flash__title"><%= pluralize(provider_credential.errors.count, "problem") %> prevented this provider credential from being saved.</p>
       <ul class="ck-error-list">
         <% provider_credential.errors.full_messages.each do |message| %>
@@ -18,12 +18,14 @@
 
     <div class="ck-field">
       <%= form.label :api_key, "API key", class: "ck-label" %>
-      <%= form.text_area :api_key, rows: 3, class: "ck-input ck-input--area ck-input--code", placeholder: "Paste the provider API key" %>
+      <%= form.text_area :api_key, rows: 3, class: "ck-input ck-input--area ck-input--code", placeholder: "Paste the provider API key", **ck_field_aria(form, :api_key) %>
+      <%= ck_field_error(form, :api_key) %>
     </div>
 
     <div class="ck-field">
       <%= form.label :api_endpoint, "API endpoint", class: "ck-label" %>
-      <%= form.text_field :api_endpoint, class: "ck-input", placeholder: "Only needed for Ollama or custom OpenAI-compatible endpoints" %>
+      <%= form.text_field :api_endpoint, class: "ck-input", placeholder: "Only needed for Ollama or custom OpenAI-compatible endpoints", **ck_field_aria(form, :api_endpoint) %>
+      <%= ck_field_error(form, :api_endpoint) %>
     </div>
 
     <div class="ck-actions">

data/app/views/completion_kit/responses/show.html.erb

@@ -1,4 +1,4 @@
-<ol class="ck-breadcrumb">
+<ol class="ck-breadcrumb" aria-label="Breadcrumb">
   <% if @run.prompt %>
     <li><%= link_to "Prompts", prompts_path %></li>
     <li><%= link_to @run.prompt.name, prompt_path(@run.prompt) %></li>
@@ -116,6 +116,15 @@
               <div class="ck-note-box"><%= review.ai_feedback %></div>
             </div>
           <% end %>
+          <% if CompletionKit.config.judge_calibration_enabled && review.metric && review.ai_score %>
+            <% existing = CompletionKit::Calibration.find_by(
+                 response_id: @response.id, metric_id: review.metric_id,
+                 created_by: CompletionKit.config.username.presence || "operator"
+               ) %>
+            <%= render "completion_kit/calibrations/buttons",
+                  review: review, calibration: existing, run: @run,
+                  response_row: @response, metric: review.metric %>
+          <% end %>
         </div>
       <% end %>
     </div>

data/app/views/completion_kit/runs/_form.html.erb

@@ -1,6 +1,6 @@
 <%= form_with(model: run, local: true) do |form| %>
   <% if run.errors.any? %>
-    <div class="ck-flash ck-flash--alert">
+    <div class="ck-flash ck-flash--alert" role="alert">
       <p class="ck-flash__title"><%= pluralize(run.errors.count, "problem") %> prevented this run from being saved.</p>
       <ul class="ck-error-list">
         <% run.errors.full_messages.each do |message| %>
@@ -13,7 +13,8 @@
   <div class="ck-card ck-form-card">
     <div class="ck-field">
       <%= form.label :name, "Name (auto-generated if blank)", class: "ck-label" %>
-      <%= form.text_field :name, class: "ck-input", placeholder: run.name.presence || "Auto-generated from prompt + version + timestamp" %>
+      <%= form.text_field :name, class: "ck-input", placeholder: run.name.presence || "Auto-generated from prompt + version + timestamp", **ck_field_aria(form, :name) %>
+      <%= ck_field_error(form, :name) %>
     </div>
 
     <div class="ck-field">
@@ -56,7 +57,8 @@
 
     <div class="ck-field" id="output-column-field" hidden>
       <%= form.label :output_column, "Output column", class: "ck-label" %>
-      <%= form.text_field :output_column, value: run.output_column.presence || "actual_output", class: "ck-input", id: "run_output_column", placeholder: "actual_output" %>
+      <%= form.text_field :output_column, value: run.output_column.presence || "actual_output", class: "ck-input", id: "run_output_column", placeholder: "actual_output", **ck_field_aria(form, :output_column) %>
+      <%= ck_field_error(form, :output_column) %>
       <p class="ck-field-hint">Name of the dataset column whose value will be graded as the response. Defaults to <code>actual_output</code>.</p>
     </div>
 

data/app/views/completion_kit/runs/_response_row.html.erb

@@ -1,6 +1,6 @@
 <% clickable = response.succeeded? %>
 <tr id="response_<%= response.id %>"<% if clickable %> onclick="window.location='<%= run_response_path(run, response, sort: params[:sort]) %>'" style="cursor: pointer;"<% end %>>
-  <td class="ck-response-cell__index"><%= index %></td>
+  <td class="ck-response-cell__index"><% if clickable %><%= link_to index, run_response_path(run, response, sort: params[:sort]), class: "ck-record-name" %><% else %><%= index %><% end %></td>
   <td class="ck-response-cell__text">
     <% if response.status == "failed" %>
       <% err = response.error_payload %>

data/app/views/completion_kit/runs/_row.html.erb

@@ -3,7 +3,7 @@
     <div class="ck-runs-table__identity">
       <span class="ck-run-name">
         <span class="<%= ck_run_dot(run) %>"></span>
-        <strong><%= run.name %></strong>
+        <%= link_to ck_run_path(run), class: "ck-record-name" do %><strong><%= run.name %></strong><% end %>
       </span>
       <div class="ck-runs-table__config">
         <% if run.prompt %>

data/app/views/completion_kit/runs/_status_header.html.erb

@@ -1,6 +1,6 @@
-<div id="run_status_header">
+<div id="run_status_header" aria-live="polite">
   <% if run.status == "failed" %>
-    <div class="ck-flash ck-flash--alert">
+    <div class="ck-flash ck-flash--alert" role="alert">
       <%= run.failure_summary.presence || run.error_message.presence || "Run failed." %>
     </div>
   <% end %>

data/app/views/completion_kit/runs/_status_panel.html.erb

@@ -1,5 +1,5 @@
 <% snap = run.progress_snapshot %>
-<div id="run_status_panel">
+<div id="run_status_panel" aria-live="polite" aria-atomic="true">
   <% if run.status.in?(%w[running completed]) && snap[:generated_total] > 0 %>
     <% failed_count = snap[:generated_failed] + snap[:judged_failed] %>
     <% has_judge = snap[:judged_total] > 0 || run.judge_configured? %>

data/app/views/completion_kit/runs/_table.html.erb

@@ -3,12 +3,12 @@
 <table class="ck-results-table ck-runs-table" style="margin-top: 0.5rem;">
   <thead>
     <tr>
-      <th>Run</th>
-      <th>Responses</th>
-      <th>Metrics</th>
-      <th>Avg score</th>
-      <th>When</th>
-      <th></th>
+      <th scope="col">Run</th>
+      <th scope="col">Responses</th>
+      <th scope="col">Metrics</th>
+      <th scope="col">Avg score</th>
+      <th scope="col">When</th>
+      <th scope="col"></th>
     </tr>
   </thead>
   <tbody>

data/app/views/completion_kit/suggestions/show.html.erb

@@ -1,4 +1,4 @@
-<ol class="ck-breadcrumb">
+<ol class="ck-breadcrumb" aria-label="Breadcrumb">
   <% if @from == "run" %>
     <li><%= link_to "Runs", runs_path %></li>
     <li><%= link_to @run.name, run_path(@run) %></li>

data/app/views/completion_kit/tags/_picker.html.erb

@@ -7,8 +7,8 @@
     <% all_tags.each do |tag| %>
       <% checked = selected_ids.include?(tag.id) %>
       <label class="tag-mark" style="--mark-color: var(--tag-<%= tag.color %>);">
-        <%= check_box_tag "#{param_namespace}[tag_names][]", tag.name, checked, hidden: true %>
-        <%= tag.name %>
+        <%= check_box_tag "#{param_namespace}[tag_names][]", tag.name, checked, class: "ck-visually-hidden", "aria-label": "Tag: #{tag.name}" %>
+        <span aria-hidden="true"><%= tag.name %></span>
       </label>
     <% end %>
     <%= text_field_tag "#{param_namespace}[tag_names][]", "",

data/app/views/completion_kit/tags/index.html.erb

@@ -14,9 +14,9 @@
   <table class="ck-results-table ck-tags-table">
     <thead>
       <tr>
-        <th>Tag</th>
-        <th>Applied to</th>
-        <th></th>
+        <th scope="col">Tag</th>
+        <th scope="col">Applied to</th>
+        <th scope="col"></th>
       </tr>
     </thead>
     <tbody>
@@ -25,7 +25,7 @@
         <% by_type = @tagging_by_type.select { |(tid, _), _| tid == tag.id } %>
         <% breakdown = by_type.map { |(_, type), n| pluralize(n, type.demodulize.titleize.downcase) }.join(" · ") %>
         <tr onclick="window.location='<%= edit_tag_path(tag) %>'" style="cursor: pointer;">
-          <td><span class="tag-mark tag-mark--lg" style="--mark-color: var(--tag-<%= tag.color %>);"><%= tag.name %></span></td>
+          <td><%= link_to edit_tag_path(tag), class: "ck-record-name" do %><span class="tag-mark tag-mark--lg" style="--mark-color: var(--tag-<%= tag.color %>);"><%= tag.name %></span><% end %></td>
           <td data-label="Applied to" class="ck-meta-copy">
             <% if count.zero? %>
               <span class="ck-tags-table__unused">Not used yet</span>

data/config/routes.rb

@@ -26,7 +26,9 @@ CompletionKit::Engine.routes.draw do
       post :rerun
       get :refresh_status
     end
-    resources :responses, only: [:show]
+    resources :responses, only: [:show] do
+      resources :calibrations, only: [:create]
+    end
   end
 
   resources :suggestions, only: [:show] do
@@ -54,7 +56,11 @@ CompletionKit::Engine.routes.draw do
           post :generate
           post :retry_failures
         end
-        resources :responses, only: [:index, :show]
+        resources :responses, only: [:index, :show] do
+          resources :metrics, only: [] do
+            resources :calibrations, only: [:index, :create]
+          end
+        end
       end
       resources :datasets
       resources :metrics

data/db/migrate/20260522000001_create_completion_kit_judge_versions.rb

@@ -0,0 +1,28 @@
+class CreateCompletionKitJudgeVersions < ActiveRecord::Migration[8.1]
+  def change
+    create_table :completion_kit_judge_versions do |t|
+      t.references :metric,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_metrics, on_delete: :cascade },
+                   index: { name: "index_ck_judge_versions_on_metric_id" }
+      t.text :instruction
+      t.text :rubric_bands
+      t.boolean :current, null: false, default: true
+      t.timestamps
+    end
+
+    add_index :completion_kit_judge_versions,
+              [:metric_id, :current],
+              name: "index_ck_judge_versions_on_metric_current"
+
+    reversible do |dir|
+      dir.up do
+        metric_model = Class.new(ActiveRecord::Base) { self.table_name = "completion_kit_metrics" }
+        jv_model = Class.new(ActiveRecord::Base) { self.table_name = "completion_kit_judge_versions" }
+        metric_model.find_each do |m|
+          jv_model.create!(metric_id: m.id, instruction: m["instruction"], rubric_bands: m["rubric_bands"], current: true)
+        end
+      end
+    end
+  end
+end

data/db/migrate/20260522000002_create_completion_kit_calibrations.rb

@@ -0,0 +1,32 @@
+class CreateCompletionKitCalibrations < ActiveRecord::Migration[8.1]
+  def change
+    create_table :completion_kit_calibrations do |t|
+      t.references :run,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_runs, on_delete: :cascade },
+                   index: { name: "index_ck_calibrations_on_run_id" }
+      t.references :response,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_responses, on_delete: :cascade },
+                   index: { name: "index_ck_calibrations_on_response_id" }
+      t.references :metric,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_metrics, on_delete: :cascade },
+                   index: { name: "index_ck_calibrations_on_metric_id" }
+      t.references :judge_version,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_judge_versions, on_delete: :cascade },
+                   index: { name: "index_ck_calibrations_on_judge_version_id" }
+      t.string :verdict, null: false
+      t.string :created_by
+      t.decimal :corrected_score, precision: 4, scale: 1
+      t.text :note
+      t.timestamps
+    end
+
+    add_index :completion_kit_calibrations,
+              [:response_id, :metric_id, :created_by],
+              unique: true,
+              name: "index_ck_calibrations_on_response_metric_user"
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.33"
+  VERSION = "0.5.35"
 end

data/lib/completion_kit.rb

@@ -11,6 +11,8 @@ module CompletionKit
     attr_accessor :tenant_scope, :tenant_scope_columns
     attr_accessor :api_reference_authentication_partial
     attr_accessor :api_rate_limit, :web_rate_limit
+    attr_accessor :allow_loopback_endpoints
+    attr_accessor :judge_calibration_enabled
 
     def initialize
       @openai_api_key = ENV['OPENAI_API_KEY']
@@ -25,6 +27,9 @@ module CompletionKit
       @api_rate_limit = 120
       @web_rate_limit = 300
 
+      @allow_loopback_endpoints = true
+      @judge_calibration_enabled = true
+
       @api_reference_authentication_partial = "completion_kit/api_reference/authentication"
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.33
+  version: 0.5.35
 platform: ruby
 authors:
 - Damien Bastin
@@ -235,6 +235,7 @@ files:
 - app/assets/javascripts/completion_kit/application.js
 - app/assets/stylesheets/completion_kit/application.css.erb
 - app/controllers/completion_kit/api/v1/base_controller.rb
+- app/controllers/completion_kit/api/v1/calibrations_controller.rb
 - app/controllers/completion_kit/api/v1/datasets_controller.rb
 - app/controllers/completion_kit/api/v1/metric_groups_controller.rb
 - app/controllers/completion_kit/api/v1/metrics_controller.rb
@@ -245,6 +246,7 @@ files:
 - app/controllers/completion_kit/api/v1/tags_controller.rb
 - app/controllers/completion_kit/api_reference_controller.rb
 - app/controllers/completion_kit/application_controller.rb
+- app/controllers/completion_kit/calibrations_controller.rb
 - app/controllers/completion_kit/dashboard_controller.rb
 - app/controllers/completion_kit/dashboard_dismissals_controller.rb
 - app/controllers/completion_kit/datasets_controller.rb
@@ -267,8 +269,10 @@ files:
 - app/jobs/completion_kit/run_completion_check_job.rb
 - app/mailers/completion_kit/application_mailer.rb
 - app/models/completion_kit/application_record.rb
+- app/models/completion_kit/calibration.rb
 - app/models/completion_kit/dashboard_dismissal.rb
 - app/models/completion_kit/dataset.rb
+- app/models/completion_kit/judge_version.rb
 - app/models/completion_kit/mcp_session.rb
 - app/models/completion_kit/metric.rb
 - app/models/completion_kit/metric_group.rb
@@ -292,6 +296,7 @@ files:
 - app/services/completion_kit/llm_client.rb
 - app/services/completion_kit/mcp_dispatcher.rb
 - app/services/completion_kit/mcp_tools/base.rb
+- app/services/completion_kit/mcp_tools/calibrations.rb
 - app/services/completion_kit/mcp_tools/datasets.rb
 - app/services/completion_kit/mcp_tools/metric_groups.rb
 - app/services/completion_kit/mcp_tools/metrics.rb
@@ -308,6 +313,7 @@ files:
 - app/services/completion_kit/open_ai_client.rb
 - app/services/completion_kit/open_router_client.rb
 - app/services/completion_kit/prompt_improvement_service.rb
+- app/services/completion_kit/provider_endpoint.rb
 - app/services/completion_kit/worker_health.rb
 - app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb
 - app/views/completion_kit/api_reference/_authentication.html.erb
@@ -316,6 +322,7 @@ files:
 - app/views/completion_kit/api_reference/_resource_card.html.erb
 - app/views/completion_kit/api_reference/_resource_list.html.erb
 - app/views/completion_kit/api_reference/index.html.erb
+- app/views/completion_kit/calibrations/_buttons.html.erb
 - app/views/completion_kit/dashboard/_eye_icon.html.erb
 - app/views/completion_kit/dashboard/_eye_off_icon.html.erb
 - app/views/completion_kit/dashboard/_failures_card.html.erb
@@ -398,6 +405,8 @@ files:
 - db/migrate/20260513000001_create_completion_kit_mcp_sessions.rb
 - db/migrate/20260514000001_allow_judge_only_runs.rb
 - db/migrate/20260516000001_create_completion_kit_dashboard_dismissals.rb
+- db/migrate/20260522000001_create_completion_kit_judge_versions.rb
+- db/migrate/20260522000002_create_completion_kit_calibrations.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb