completion-kit 0.5.34 → 0.5.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6a67c2ef67eff4762711f38e46cb5df85b9136a6de59a82d962a54ef2c1466c
-  data.tar.gz: 01f86625fec3d18ee712e61d0d601687571125e8333e5f468d3c151f9714651b
+  metadata.gz: 7ec9f216056f47d007b8a512a009dea02aeaad328a9c4832ac0c1e122b816b15
+  data.tar.gz: 413f1b8e8ca28ed2c14e55210299a28bef42acb1a437a3ee9c3bd88b62d03bf9
 SHA512:
-  metadata.gz: 33335e4a89da8b1eb264663c50570841a7afeda8b48bcad1178b2185ceee5ec2148e1e57ea4248835e550ca936a5bccd4ece6819070cde7f44c3ef312aa40970
-  data.tar.gz: 39b1e8845d2dea0e2ac6bed5679336d639ea501796b0e15242eeef42db3e9313eaeccaa59268d11a2fcb52ce7c110634374f16fa4d7721494c6474caa524b97e
+  metadata.gz: c837fd6ddc33c5799145bac2a0b1dba4ca1df807365c621330b619c47607e666a30325b47ce6d34e2ecb93e13d4e14a9538d597e0d56e47b9f7a1a836432be0d
+  data.tar.gz: 06b9ac7200883cd19d11ff127898d52d940b3f7a110cd5e7744bfcb2abab797fa05e923ba76121f6c00c6aeb8ac5d7b5d5e8b79b07d773ad0c0d5650ec41b90d

data/app/assets/stylesheets/completion_kit/application.css.erb

@@ -5119,3 +5119,57 @@ a.tag-mark {
   outline: 2px solid var(--ck-accent);
   outline-offset: 2px;
 }
+
+.ck-calibration {
+  margin-top: 12px;
+  padding-top: 12px;
+  border-top: 1px dashed rgba(255, 255, 255, 0.08);
+}
+.ck-calibration__prompt {
+  font-size: 0.8rem;
+  color: var(--ck-dim);
+  margin: 0 0 8px;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+.ck-calibration__count {
+  font-size: 0.75rem;
+  color: var(--ck-accent);
+}
+.ck-calibration__buttons {
+  display: flex;
+  gap: 8px;
+  flex-wrap: wrap;
+}
+.ck-calibration__pill {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  padding: 6px 12px;
+  border-radius: 999px;
+  font-size: 0.85rem;
+  background: transparent;
+  border: 1px solid rgba(255, 255, 255, 0.18);
+  color: inherit;
+  cursor: pointer;
+}
+.ck-calibration__pill:hover,
+.ck-calibration__pill:focus-visible {
+  border-color: var(--ck-accent);
+}
+.ck-calibration__pill.is-active {
+  background: var(--ck-accent);
+  color: #0b1320;
+  border-color: var(--ck-accent);
+}
+.ck-calibration__detail {
+  margin-top: 10px;
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+}
+.ck-calibration__value {
+  color: var(--ck-accent);
+  font-weight: 600;
+}

data/app/controllers/completion_kit/api/v1/calibrations_controller.rb

@@ -0,0 +1,57 @@
+module CompletionKit
+  module Api
+    module V1
+      class CalibrationsController < BaseController
+        before_action :ensure_calibration_enabled
+        before_action :set_scope
+
+        def index
+          render json: scope_calibrations
+        end
+
+        def create
+          calibration = scope_calibrations.find_or_initialize_by(created_by: created_by_param)
+          calibration.assign_attributes(
+            run: @run,
+            response: @response,
+            metric: @metric,
+            judge_version: JudgeVersion.ensure_current_for(@metric),
+            **calibration_params
+          )
+
+          if calibration.save
+            render json: calibration, status: calibration.previously_new_record? ? :created : :ok
+          else
+            render json: { errors: calibration.errors }, status: :unprocessable_entity
+          end
+        end
+
+        private
+
+        def ensure_calibration_enabled
+          render(json: { error: "Calibration disabled" }, status: :not_found) unless CompletionKit.config.judge_calibration_enabled
+        end
+
+        def set_scope
+          @run = Run.find(params[:run_id])
+          @response = @run.responses.find(params[:response_id])
+          @metric = Metric.find(params[:metric_id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def scope_calibrations
+          Calibration.where(run_id: @run.id, response_id: @response.id, metric_id: @metric.id)
+        end
+
+        def calibration_params
+          params.permit(:verdict, :corrected_score, :note).to_h.symbolize_keys
+        end
+
+        def created_by_param
+          params[:created_by].presence || "api"
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/calibrations_controller.rb

@@ -0,0 +1,50 @@
+module CompletionKit
+  class CalibrationsController < ApplicationController
+    before_action :ensure_calibration_enabled
+    before_action :set_scope
+
+    def create
+      created_by = calibration_creator
+      calibration = Calibration.find_or_initialize_by(
+        run_id: @run.id, response_id: @response.id, metric_id: @metric.id, created_by: created_by
+      )
+      calibration.assign_attributes(
+        judge_version: JudgeVersion.ensure_current_for(@metric),
+        verdict: params[:verdict],
+        corrected_score: params[:corrected_score].presence,
+        note: params[:note].presence
+      )
+
+      if calibration.save
+        render turbo_stream: turbo_stream.replace(
+          "calibration_#{@response.id}_#{@metric.id}",
+          partial: "completion_kit/calibrations/buttons",
+          locals: { review: review_for_metric, calibration: calibration, run: @run, response_row: @response, metric: @metric }
+        )
+      else
+        flash[:alert] = calibration.errors.full_messages.to_sentence
+        redirect_to run_response_path(@run, @response)
+      end
+    end
+
+    private
+
+    def ensure_calibration_enabled
+      head :not_found unless CompletionKit.config.judge_calibration_enabled
+    end
+
+    def set_scope
+      @run = Run.find(params[:run_id])
+      @response = @run.responses.find(params[:response_id])
+      @metric = Metric.find(params[:metric_id])
+    end
+
+    def review_for_metric
+      @response.reviews.find_by(metric_id: @metric.id)
+    end
+
+    def calibration_creator
+      request.env["HTTP_X_REMOTE_USER"].presence || CompletionKit.config.username.presence || "operator"
+    end
+  end
+end

data/app/models/completion_kit/calibration.rb

@@ -0,0 +1,47 @@
+module CompletionKit
+  class Calibration < ApplicationRecord
+    VERDICTS = %w[agree disagree borderline].freeze
+
+    belongs_to :run
+    belongs_to :response
+    belongs_to :metric
+    belongs_to :judge_version
+
+    validates :verdict, presence: true, inclusion: { in: VERDICTS }
+    validates :response_id,
+              uniqueness: { scope: [:metric_id, :created_by] }
+    validate :corrected_score_required_when_disagreeing
+    validate :corrected_score_within_rubric
+
+    scope :for_run, ->(run_id) { where(run_id: run_id) }
+    scope :for_metric, ->(metric_id) { where(metric_id: metric_id) }
+
+    def as_json(options = {})
+      {
+        id: id,
+        run_id: run_id,
+        response_id: response_id,
+        metric_id: metric_id,
+        judge_version_id: judge_version_id,
+        verdict: verdict,
+        corrected_score: corrected_score,
+        note: note,
+        created_by: created_by,
+        created_at: created_at
+      }
+    end
+
+    private
+
+    def corrected_score_required_when_disagreeing
+      return unless verdict == "disagree"
+      errors.add(:corrected_score, "must be set when disagreeing with the judge") if corrected_score.blank?
+    end
+
+    def corrected_score_within_rubric
+      return if corrected_score.blank?
+      score = corrected_score.to_f
+      errors.add(:corrected_score, "must be between 1 and 5") unless score >= 1 && score <= 5
+    end
+  end
+end

data/app/models/completion_kit/judge_version.rb

@@ -0,0 +1,32 @@
+module CompletionKit
+  class JudgeVersion < ApplicationRecord
+    belongs_to :metric
+    has_many :calibrations, dependent: :destroy
+
+    serialize :rubric_bands, coder: JSON
+
+    validates :metric_id, presence: true
+
+    scope :current, -> { where(current: true) }
+
+    def self.ensure_current_for(metric)
+      current.find_by(metric_id: metric.id) || create!(
+        metric: metric,
+        instruction: metric.instruction,
+        rubric_bands: metric.rubric_bands,
+        current: true
+      )
+    end
+
+    def as_json(options = {})
+      {
+        id: id,
+        metric_id: metric_id,
+        instruction: instruction,
+        rubric_bands: rubric_bands,
+        current: current,
+        created_at: created_at
+      }
+    end
+  end
+end

data/app/services/completion_kit/mcp_dispatcher.rb

@@ -33,7 +33,8 @@ module CompletionKit
         McpTools::Metrics.definitions +
         McpTools::MetricGroups.definitions +
         McpTools::ProviderCredentials.definitions +
-        McpTools::Tags.definitions
+        McpTools::Tags.definitions +
+        McpTools::Calibrations.definitions
     end
 
     def self.call_tool(name, arguments)
@@ -46,6 +47,7 @@ module CompletionKit
       when /\Ametric_groups_/        then McpTools::MetricGroups.call(name, arguments)
       when /\Aprovider_credentials_/ then McpTools::ProviderCredentials.call(name, arguments)
       when /\Atags_/                 then McpTools::Tags.call(name, arguments)
+      when /\Acalibrations_/         then McpTools::Calibrations.call(name, arguments)
       else raise MethodNotFound, "Unknown tool: #{name}"
       end
     end

data/app/services/completion_kit/mcp_tools/calibrations.rb

@@ -0,0 +1,73 @@
+module CompletionKit
+  module McpTools
+    module Calibrations
+      extend Base
+
+      TOOLS = {
+        "calibrations_list" => {
+          description: "List calibrations. Filter by run_id, response_id, metric_id, or created_by.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              run_id: {type: "integer"},
+              response_id: {type: "integer"},
+              metric_id: {type: "integer"},
+              created_by: {type: "string"}
+            },
+            required: []
+          },
+          handler: :list
+        },
+        "calibrations_create" => {
+          description: "Upsert a calibration for (run, response, metric, created_by). Verdict is one of agree, disagree, borderline. corrected_score (1..5) is required when verdict is 'disagree'.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              run_id: {type: "integer"},
+              response_id: {type: "integer"},
+              metric_id: {type: "integer"},
+              verdict: {type: "string", enum: %w[agree disagree borderline]},
+              corrected_score: {type: "number"},
+              note: {type: "string"},
+              created_by: {type: "string"}
+            },
+            required: ["run_id", "response_id", "metric_id", "verdict"]
+          },
+          handler: :create
+        }
+      }.freeze
+
+      def self.list(args)
+        scope = CompletionKit::Calibration.all
+        scope = scope.where(run_id: args["run_id"]) if args["run_id"]
+        scope = scope.where(response_id: args["response_id"]) if args["response_id"]
+        scope = scope.where(metric_id: args["metric_id"]) if args["metric_id"]
+        scope = scope.where(created_by: args["created_by"]) if args["created_by"]
+        text_result(scope.order(:created_at).map(&:as_json))
+      end
+
+      def self.create(args)
+        run = CompletionKit::Run.find(args["run_id"])
+        response = run.responses.find(args["response_id"])
+        metric = CompletionKit::Metric.find(args["metric_id"])
+        created_by = args["created_by"].presence || "mcp"
+
+        calibration = CompletionKit::Calibration.find_or_initialize_by(
+          run_id: run.id, response_id: response.id, metric_id: metric.id, created_by: created_by
+        )
+        calibration.assign_attributes(
+          judge_version: CompletionKit::JudgeVersion.ensure_current_for(metric),
+          verdict: args["verdict"],
+          corrected_score: args["corrected_score"],
+          note: args["note"]
+        )
+
+        if calibration.save
+          text_result(calibration.as_json)
+        else
+          error_result(calibration.errors.full_messages.join(", "))
+        end
+      end
+    end
+  end
+end

data/app/views/completion_kit/calibrations/_buttons.html.erb

@@ -0,0 +1,50 @@
+<div id="calibration_<%= response_row.id %>_<%= metric.id %>" class="ck-calibration">
+  <% current_verdict = calibration&.verdict %>
+  <% verdict_count = CompletionKit::Calibration.where(response_id: response_row.id, metric_id: metric.id).count %>
+  <p class="ck-calibration__prompt">
+    How does this score feel?
+    <% if verdict_count > 0 %>
+      <span class="ck-calibration__count"><%= pluralize(verdict_count, "verdict") %> collected</span>
+    <% end %>
+  </p>
+  <div class="ck-calibration__buttons">
+    <% CompletionKit::Calibration::VERDICTS.each do |verdict| %>
+      <%= button_to run_response_calibrations_path(run, response_row, metric_id: metric.id, verdict: verdict),
+              method: :post,
+              form: { data: { turbo: "true" } },
+              class: "ck-calibration__pill ck-calibration__pill--#{verdict}#{' is-active' if verdict == current_verdict}",
+              "aria-pressed": (verdict == current_verdict).to_s do %>
+        <% case verdict
+           when "agree" %>👍 Agree<% when "disagree" %>👎 Disagree<% else %>🤔 Borderline<% end %>
+      <% end %>
+    <% end %>
+  </div>
+
+  <% if current_verdict == "disagree" %>
+    <%= form_with url: run_response_calibrations_path(run, response_row),
+                  method: :post, local: false,
+                  class: "ck-calibration__detail" do |f| %>
+      <%= hidden_field_tag :metric_id, metric.id %>
+      <%= hidden_field_tag :verdict, "disagree" %>
+      <label class="ck-label">
+        Your score
+        <span class="ck-calibration__value" data-calibration-value><%= calibration.corrected_score || review&.ai_score || 3 %></span>
+      </label>
+      <input type="range" name="corrected_score" min="1" max="5" step="0.5"
+             value="<%= calibration.corrected_score || review&.ai_score || 3 %>"
+             oninput="this.closest('.ck-calibration__detail').querySelector('[data-calibration-value]').textContent = this.value"
+             class="ck-slider">
+      <textarea name="note" rows="2" class="ck-input ck-input--area" placeholder="Why? (optional)"><%= calibration.note %></textarea>
+      <%= f.submit "Save", class: ck_button_classes(:dark) %>
+    <% end %>
+  <% elsif current_verdict == "borderline" %>
+    <%= form_with url: run_response_calibrations_path(run, response_row),
+                  method: :post, local: false,
+                  class: "ck-calibration__detail" do |f| %>
+      <%= hidden_field_tag :metric_id, metric.id %>
+      <%= hidden_field_tag :verdict, "borderline" %>
+      <textarea name="note" rows="2" class="ck-input ck-input--area" placeholder="What made this borderline? (optional)"><%= calibration.note %></textarea>
+      <%= f.submit "Save", class: ck_button_classes(:dark) %>
+    <% end %>
+  <% end %>
+</div>

data/app/views/completion_kit/responses/show.html.erb

@@ -116,6 +116,15 @@
               <div class="ck-note-box"><%= review.ai_feedback %></div>
             </div>
           <% end %>
+          <% if CompletionKit.config.judge_calibration_enabled && review.metric && review.ai_score %>
+            <% existing = CompletionKit::Calibration.find_by(
+                 response_id: @response.id, metric_id: review.metric_id,
+                 created_by: CompletionKit.config.username.presence || "operator"
+               ) %>
+            <%= render "completion_kit/calibrations/buttons",
+                  review: review, calibration: existing, run: @run,
+                  response_row: @response, metric: review.metric %>
+          <% end %>
         </div>
       <% end %>
     </div>

data/config/routes.rb

@@ -26,7 +26,9 @@ CompletionKit::Engine.routes.draw do
       post :rerun
       get :refresh_status
     end
-    resources :responses, only: [:show]
+    resources :responses, only: [:show] do
+      resources :calibrations, only: [:create]
+    end
   end
 
   resources :suggestions, only: [:show] do
@@ -54,7 +56,11 @@ CompletionKit::Engine.routes.draw do
           post :generate
           post :retry_failures
         end
-        resources :responses, only: [:index, :show]
+        resources :responses, only: [:index, :show] do
+          resources :metrics, only: [] do
+            resources :calibrations, only: [:index, :create]
+          end
+        end
       end
       resources :datasets
       resources :metrics

data/db/migrate/20260522000001_create_completion_kit_judge_versions.rb

@@ -0,0 +1,28 @@
+class CreateCompletionKitJudgeVersions < ActiveRecord::Migration[8.1]
+  def change
+    create_table :completion_kit_judge_versions do |t|
+      t.references :metric,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_metrics, on_delete: :cascade },
+                   index: { name: "index_ck_judge_versions_on_metric_id" }
+      t.text :instruction
+      t.text :rubric_bands
+      t.boolean :current, null: false, default: true
+      t.timestamps
+    end
+
+    add_index :completion_kit_judge_versions,
+              [:metric_id, :current],
+              name: "index_ck_judge_versions_on_metric_current"
+
+    reversible do |dir|
+      dir.up do
+        metric_model = Class.new(ActiveRecord::Base) { self.table_name = "completion_kit_metrics" }
+        jv_model = Class.new(ActiveRecord::Base) { self.table_name = "completion_kit_judge_versions" }
+        metric_model.find_each do |m|
+          jv_model.create!(metric_id: m.id, instruction: m["instruction"], rubric_bands: m["rubric_bands"], current: true)
+        end
+      end
+    end
+  end
+end

data/db/migrate/20260522000002_create_completion_kit_calibrations.rb

@@ -0,0 +1,32 @@
+class CreateCompletionKitCalibrations < ActiveRecord::Migration[8.1]
+  def change
+    create_table :completion_kit_calibrations do |t|
+      t.references :run,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_runs, on_delete: :cascade },
+                   index: { name: "index_ck_calibrations_on_run_id" }
+      t.references :response,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_responses, on_delete: :cascade },
+                   index: { name: "index_ck_calibrations_on_response_id" }
+      t.references :metric,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_metrics, on_delete: :cascade },
+                   index: { name: "index_ck_calibrations_on_metric_id" }
+      t.references :judge_version,
+                   null: false,
+                   foreign_key: { to_table: :completion_kit_judge_versions, on_delete: :cascade },
+                   index: { name: "index_ck_calibrations_on_judge_version_id" }
+      t.string :verdict, null: false
+      t.string :created_by
+      t.decimal :corrected_score, precision: 4, scale: 1
+      t.text :note
+      t.timestamps
+    end
+
+    add_index :completion_kit_calibrations,
+              [:response_id, :metric_id, :created_by],
+              unique: true,
+              name: "index_ck_calibrations_on_response_metric_user"
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.34"
+  VERSION = "0.5.35"
 end

data/lib/completion_kit.rb

@@ -12,6 +12,7 @@ module CompletionKit
     attr_accessor :api_reference_authentication_partial
     attr_accessor :api_rate_limit, :web_rate_limit
     attr_accessor :allow_loopback_endpoints
+    attr_accessor :judge_calibration_enabled
 
     def initialize
       @openai_api_key = ENV['OPENAI_API_KEY']
@@ -27,6 +28,7 @@ module CompletionKit
       @web_rate_limit = 300
 
       @allow_loopback_endpoints = true
+      @judge_calibration_enabled = true
 
       @api_reference_authentication_partial = "completion_kit/api_reference/authentication"
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.34
+  version: 0.5.35
 platform: ruby
 authors:
 - Damien Bastin
@@ -235,6 +235,7 @@ files:
 - app/assets/javascripts/completion_kit/application.js
 - app/assets/stylesheets/completion_kit/application.css.erb
 - app/controllers/completion_kit/api/v1/base_controller.rb
+- app/controllers/completion_kit/api/v1/calibrations_controller.rb
 - app/controllers/completion_kit/api/v1/datasets_controller.rb
 - app/controllers/completion_kit/api/v1/metric_groups_controller.rb
 - app/controllers/completion_kit/api/v1/metrics_controller.rb
@@ -245,6 +246,7 @@ files:
 - app/controllers/completion_kit/api/v1/tags_controller.rb
 - app/controllers/completion_kit/api_reference_controller.rb
 - app/controllers/completion_kit/application_controller.rb
+- app/controllers/completion_kit/calibrations_controller.rb
 - app/controllers/completion_kit/dashboard_controller.rb
 - app/controllers/completion_kit/dashboard_dismissals_controller.rb
 - app/controllers/completion_kit/datasets_controller.rb
@@ -267,8 +269,10 @@ files:
 - app/jobs/completion_kit/run_completion_check_job.rb
 - app/mailers/completion_kit/application_mailer.rb
 - app/models/completion_kit/application_record.rb
+- app/models/completion_kit/calibration.rb
 - app/models/completion_kit/dashboard_dismissal.rb
 - app/models/completion_kit/dataset.rb
+- app/models/completion_kit/judge_version.rb
 - app/models/completion_kit/mcp_session.rb
 - app/models/completion_kit/metric.rb
 - app/models/completion_kit/metric_group.rb
@@ -292,6 +296,7 @@ files:
 - app/services/completion_kit/llm_client.rb
 - app/services/completion_kit/mcp_dispatcher.rb
 - app/services/completion_kit/mcp_tools/base.rb
+- app/services/completion_kit/mcp_tools/calibrations.rb
 - app/services/completion_kit/mcp_tools/datasets.rb
 - app/services/completion_kit/mcp_tools/metric_groups.rb
 - app/services/completion_kit/mcp_tools/metrics.rb
@@ -317,6 +322,7 @@ files:
 - app/views/completion_kit/api_reference/_resource_card.html.erb
 - app/views/completion_kit/api_reference/_resource_list.html.erb
 - app/views/completion_kit/api_reference/index.html.erb
+- app/views/completion_kit/calibrations/_buttons.html.erb
 - app/views/completion_kit/dashboard/_eye_icon.html.erb
 - app/views/completion_kit/dashboard/_eye_off_icon.html.erb
 - app/views/completion_kit/dashboard/_failures_card.html.erb
@@ -399,6 +405,8 @@ files:
 - db/migrate/20260513000001_create_completion_kit_mcp_sessions.rb
 - db/migrate/20260514000001_allow_judge_only_runs.rb
 - db/migrate/20260516000001_create_completion_kit_dashboard_dismissals.rb
+- db/migrate/20260522000001_create_completion_kit_judge_versions.rb
+- db/migrate/20260522000002_create_completion_kit_calibrations.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb