completion-kit 0.28.43 → 0.28.45

data/app/views/completion_kit/agreements/_trust_panel.html.erb

@@ -3,8 +3,7 @@
 <% anchor = metric&.name&.parameterize %>
 <% current_metric_version = metric && CompletionKit::MetricVersion.current.find_by(metric_id: metric.id) %>
 <% target_response = if (stats.sample_size.zero? || stats.counter_only?) && metric && current_metric_version
-     created_by = CompletionKit.config.username.presence || "operator"
-     verdicted_ids = CompletionKit::Agreement.where(metric_id: metric.id, created_by: created_by, metric_version_id: current_metric_version.id, run_id: CompletionKit::Run.visible_run_ids).pluck(:response_id)
+     verdicted_ids = CompletionKit::Agreement.where(metric_id: metric.id, created_by: ck_current_reviewer, metric_version_id: current_metric_version.id, run_id: CompletionKit::Run.visible_run_ids).pluck(:response_id)
      CompletionKit::Response.where(run_id: CompletionKit::Run.visible_run_ids).joins(:reviews)
        .where(reviews: { metric_id: metric.id, metric_version_id: current_metric_version.id })
        .where.not(reviews: { ai_score: nil })

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -145,7 +145,7 @@
             } %>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:id/generate</p>
-        <p class="ck-meta-copy">Start generating responses. Returns 202 Accepted. Poll the run to check progress.</p>
+        <p class="ck-meta-copy">Start generating responses. Returns 202 Accepted. Poll the run to check progress. Returns 422 with the reason when the run cannot start, for example when no configured provider serves the prompt's model or the judge model.</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs/1/generate \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <div class="ck-api-endpoint">
@@ -161,7 +161,7 @@
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs/:id/regrade</p>
-        <p class="ck-meta-copy">Re-judge the existing successful responses against the current metric versions without regenerating model output. Returns 202 Accepted, or 422 if no responses are eligible.</p>
+        <p class="ck-meta-copy">Re-judge the existing successful responses against the current metric versions without regenerating model output. Returns 202 Accepted, or 422 if no responses are eligible or no configured provider serves the judge model.</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs/1/regrade \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
       <div class="ck-api-endpoint">
@@ -258,7 +258,7 @@
       </div>
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/metrics/:id/suggest_variants</p>
-        <p class="ck-meta-copy">Generate draft metric versions from the current disagreements. Only rubric-style (llm_judge) metrics can suggest variants. Returns 201 with the new draft versions, or 422 if the metric is a check (checks are exact and have no variants to suggest), if no disagreements exist, or if the model produced nothing usable.</p>
+        <p class="ck-meta-copy">Generate draft metric versions from the current disagreements. Only rubric-style (llm_judge) metrics can suggest variants. Returns 201 with the new draft versions, or 422 if the metric is a check (checks are exact and have no variants to suggest), if no disagreements exist, if no configured provider serves the model, or if the model produced nothing usable.</p>
         <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>count</code>, <code>model</code></p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/metrics/1/suggest_variants \\\n  -H \"Authorization: Bearer #{token}\"" %>
       </div>
@@ -395,6 +395,7 @@
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">GET</span>&ensp;<span class="ck-chip ck-chip--soft">PATCH</span>&ensp;<span class="ck-chip" style="color: var(--ck-danger);">DELETE</span> /api/v1/provider_credentials/:id</p>
         <p class="ck-meta-copy">Get, update, or delete a provider credential. DELETE returns 422 (not 204) when the provider is still in use by prompts or judge runs; remove those references first.</p>
+        <p class="ck-meta-copy">Changing <code>provider</code> or <code>api_endpoint</code> on a credential with a stored key requires <code>api_key</code> in the same request, otherwise PATCH returns 422 and nothing changes. Send <code>"api_key": null</code> to remove the key instead.</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X PATCH #{base_url}/api/v1/provider_credentials/1 \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"api_key\": \"sk-new...\"}'" %>
       </div>
       <%= render "completion_kit/api_reference/resource_list", title: "Your providers",

data/app/views/completion_kit/dashboard/show.html.erb

@@ -130,7 +130,7 @@
   <% if @recent_runs.any? %>
     <%= render "completion_kit/runs/table", runs: @recent_runs %>
   <% elsif CompletionKit::Run.exists? %>
-    <div class="ck-empty">Your older runs are hidden.</div>
+    <div class="ck-empty">No runs to show.</div>
   <% else %>
     <div class="ck-empty">No runs yet.&ensp;<%= link_to "Create your first run →", new_run_path, class: "ck-link" %></div>
   <% end %>

data/app/views/completion_kit/datasets/show.html.erb

@@ -71,6 +71,11 @@
     <p class="ck-kicker">Runs</p>
     <%= render "completion_kit/runs/table", runs: @runs %>
   </section>
+<% elsif @dataset.runs.exists? %>
+  <section class="ck-card--spaced">
+    <p class="ck-kicker">Runs</p>
+    <div class="ck-empty">No runs to show.</div>
+  </section>
 <% end %>
 
 <%= ck_runs_display_footer(@runs) %>

data/app/views/completion_kit/metric_groups/_form.html.erb

@@ -45,7 +45,7 @@
       <div class="ck-metric-checkboxes" data-metric-checkboxes role="group" aria-labelledby="mg_metrics_label">
         <% @metrics.each do |metric| %>
           <label class="ck-checkbox-label" data-metric-tags="<%= metric.tag_names.join(",") %>">
-            <%= check_box_tag "metric_group[metric_ids][]", metric.id, metric_group.metrics.exists?(metric.id), class: "ck-checkbox", id: "metric_group_metric_#{metric.id}" %>
+            <%= check_box_tag "metric_group[metric_ids][]", metric.id, metric_group.selected_metric_ids.include?(metric.id), class: "ck-checkbox", id: "metric_group_metric_#{metric.id}" %>
             <span class="ck-checkbox-label__box" aria-hidden="true"></span>
             <span class="ck-checkbox-label__body">
               <span class="ck-checkbox-label__text"><%= metric.name %></span>
@@ -61,6 +61,9 @@
           </label>
         <% end %>
       </div>
+      <% if metric_group.errors[:metric_ids].any? %>
+        <p class="ck-field-error" id="<%= ck_field_error_id(form, :metric_ids) %>"><%= metric_group.errors.full_messages_for(:metric_ids).to_sentence %></p>
+      <% end %>
       <%= hidden_field_tag "metric_group[metric_ids][]", "" %>
     </div>
 

data/app/views/completion_kit/prompts/show.html.erb

@@ -162,6 +162,11 @@
     <p class="ck-kicker">Runs</p>
     <%= render "completion_kit/runs/table", runs: @runs %>
   </section>
+<% elsif CompletionKit::Run.where(prompt_id: @prompt.family_versions.select(:id)).exists? %>
+  <section class="ck-card--spaced">
+    <p class="ck-kicker">Runs</p>
+    <div class="ck-empty">No runs to show.</div>
+  </section>
 <% end %>
 
 <%= ck_runs_display_footer(@runs) %>

data/app/views/completion_kit/provider_credentials/_form.html.erb

@@ -22,10 +22,33 @@
 
     <div class="ck-field">
       <%= form.label :api_key, "API key", class: "ck-label" %>
-      <%= form.text_area :api_key, rows: 3, class: "ck-input ck-input--area ck-input--code", placeholder: "Paste the provider API key", **ck_field_aria(form, :api_key) %>
+      <%= form.text_area :api_key, value: (provider_credential.persisted? ? "" : provider_credential.api_key), rows: 3, class: "ck-input ck-input--area ck-input--code", placeholder: "Paste the provider API key", **ck_field_aria(form, :api_key) %>
       <%= ck_field_error(form, :api_key) %>
+      <% if provider_credential.api_key_stored? %>
+        <p class="ck-field-hint">
+          <% if provider_credential.api_key_ending %>
+            Leave blank to keep the current key ending in <code><%= provider_credential.api_key_ending %></code>.
+          <% else %>
+            Leave blank to keep the current key.
+          <% end %>
+          Enter it again if you change the provider or endpoint.
+        </p>
+      <% end %>
     </div>
 
+    <% if provider_credential.api_key_stored? %>
+      <div class="ck-field">
+        <label class="ck-checkbox-label">
+          <%= check_box_tag "provider_credential[remove_api_key]", "1", local_assigns[:remove_api_key], id: "provider_credential_remove_api_key", class: "ck-checkbox" %>
+          <span class="ck-checkbox-label__box" aria-hidden="true"></span>
+          <span class="ck-checkbox-label__body">
+            <span class="ck-checkbox-label__text">Remove the stored key</span>
+            <span class="ck-checkbox-label__hint">For endpoints that run without a key, such as a local Ollama server.</span>
+          </span>
+        </label>
+      </div>
+    <% end %>
+
     <div class="ck-field">
       <%= form.label :api_endpoint, "API endpoint", class: "ck-label" %>
       <%= form.text_field :api_endpoint, class: "ck-input", placeholder: "Needed for Ollama, Azure AI Foundry, or custom OpenAI-compatible endpoints", **ck_field_aria(form, :api_endpoint) %>

data/app/views/completion_kit/provider_credentials/edit.html.erb

@@ -10,4 +10,4 @@
   </div>
 </section>
 
-<%= render "form", provider_credential: @provider_credential %>
+<%= render "form", provider_credential: @provider_credential, remove_api_key: @remove_api_key %>

data/app/views/completion_kit/provider_credentials/index.html.erb

@@ -19,11 +19,10 @@
   <% default_endpoints = {"openai" => "https://api.openai.com", "anthropic" => "https://api.anthropic.com"} %>
   <div class="ck-provider-list" data-ck-statuses-url="<%= statuses_provider_credentials_path %>">
     <% @provider_credentials.each do |provider_credential| %>
-      <% masked_key = provider_credential.api_key.present? ? "#{provider_credential.api_key[0..6]}#{'•' * 12}#{provider_credential.api_key[-4..]}" : "—" %>
       <a href="<%= edit_provider_credential_path(provider_credential) %>" class="ck-provider-card">
         <div class="ck-provider-card__header">
           <span class="ck-provider-card__name"><%= provider_credential.display_provider %></span>
-          <span class="ck-provider-card__key"><%= masked_key %></span>
+          <span class="ck-provider-card__key"><%= provider_credential.api_key_stored? ? "#{'•' * 12}#{provider_credential.api_key_ending}" : "No key" %></span>
         </div>
 
         <div class="ck-provider-card__meta">

data/app/views/completion_kit/responses/show.html.erb

@@ -139,7 +139,7 @@
           <% if CompletionKit.config.judge_agreement_enabled && review.metric && review.ai_score && review.metric_version&.llm_judge? %>
             <% existing = CompletionKit::Agreement.find_by(
                  response_id: @response.id, metric_id: review.metric_id,
-                 created_by: CompletionKit.config.username.presence || "operator"
+                 created_by: ck_current_reviewer
                ) %>
             <%= render "completion_kit/agreements/buttons",
                   review: review, agreement: existing, run: @run,

data/app/views/completion_kit/runs/_form.html.erb

@@ -160,7 +160,7 @@
         <div class="ck-metric-checkboxes" data-metric-checkboxes role="group" aria-labelledby="run_metrics_label">
           <% @all_metrics.each do |metric| %>
             <label class="ck-checkbox-label" data-metric-tags="<%= metric.tag_names.join(",") %>" data-compare-expected="<%= metric.check? && metric.check_config.to_h["compare_to"] == "expected" ? "1" : "0" %>">
-              <%= check_box_tag "run[metric_ids][]", metric.id, run.metric_ids.include?(metric.id), class: "ck-checkbox", id: "run_metric_#{metric.id}" %>
+              <%= check_box_tag "run[metric_ids][]", metric.id, run.selected_metric_ids.include?(metric.id), class: "ck-checkbox", id: "run_metric_#{metric.id}" %>
               <span class="ck-checkbox-label__box" aria-hidden="true"></span>
               <span class="ck-checkbox-label__body">
                 <span class="ck-checkbox-label__text"><%= metric.name %></span>

data/app/views/completion_kit/runs/index.html.erb

@@ -21,7 +21,7 @@
     <p>No runs match these tags. <%= link_to "Clear filters", runs_path, class: "ck-link" %>.</p>
   </div>
 <% elsif CompletionKit::Run.exists? %>
-  <div class="ck-empty">Your older runs are hidden.</div>
+  <div class="ck-empty">No runs to show.</div>
 <% else %>
   <div class="ck-empty">No runs yet.&ensp;<%= link_to "Create your first run →", new_run_path, class: "ck-link" %></div>
 <% end %>

data/config/locales/en.yml

@@ -0,0 +1,12 @@
+en:
+  activerecord:
+    attributes:
+      completion_kit/prompt:
+        llm_model: Model
+      completion_kit/run:
+        judge_model: Judge model
+        metric_ids: Metrics
+      completion_kit/metric_group:
+        metric_ids: Metrics
+      completion_kit/provider_credential:
+        api_key: API key

data/lib/completion_kit/errors.rb

@@ -1,8 +1,38 @@
+require "faraday"
+require "json"
+
 module CompletionKit
   class Error < StandardError; end
 
   class ConfigurationError < Error; end
 
+  class UnknownModelError < ConfigurationError
+    attr_reader :model_name
+
+    def initialize(model_name = nil)
+      @model_name = model_name
+      super(%(No configured provider serves model "#{model_name}"))
+    end
+  end
+
+  class EndpointError < Error
+    CONNECTION_MESSAGE = "Could not connect to the endpoint."
+    TIMEOUT_MESSAGE = "The endpoint did not respond in time."
+    NOT_JSON_MESSAGE = "The endpoint did not return JSON."
+
+    def self.from(error)
+      message = case error
+                when Faraday::TimeoutError then TIMEOUT_MESSAGE
+                when Faraday::Error then CONNECTION_MESSAGE
+                when JSON::ParserError then NOT_JSON_MESSAGE
+                end
+      return error unless message
+
+      Rails.logger.warn("[CompletionKit] endpoint request failed: #{error.class}: #{error.message}")
+      new(message)
+    end
+  end
+
   class RateLimitError < Error
     attr_reader :provider, :status, :retry_after
 

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.43"
+  VERSION = "0.28.45"
 end

data/lib/completion_kit.rb

@@ -7,7 +7,7 @@ module CompletionKit
   class Configuration
     attr_accessor :openai_api_key, :anthropic_api_key, :ollama_api_key, :ollama_api_endpoint
     attr_accessor :judge_model, :high_quality_threshold, :medium_quality_threshold
-    attr_accessor :username, :password, :auth_strategy, :api_token
+    attr_accessor :username, :password, :auth_strategy, :api_token, :current_reviewer
     attr_accessor :tenant_scope, :tenant_scope_columns
     attr_accessor :api_reference_authentication_partial
     attr_accessor :runs_display_scope, :runs_display_footer_partial

data/lib/generators/completion_kit/templates/initializer.rb

@@ -45,6 +45,15 @@ CompletionKit.configure do |config|
   # config.username = "admin"
   # config.password = ENV['COMPLETION_KIT_PASSWORD']
 
+  # Reviewer name for verdicts recorded in the web UI. The callable receives the
+  # controller. Each reviewer keeps one verdict per response and metric, so return
+  # a String unique to the signed-in person. Blank or nil falls back to username,
+  # then "operator". Any other return type raises CompletionKit::ConfigurationError.
+  # config.current_reviewer = ->(controller) { controller.current_user&.email }
+  # Behind a proxy that sets X-Remote-User. Only use this when a trusted proxy
+  # always overwrites the header, or any client can claim any name:
+  # config.current_reviewer = ->(controller) { controller.request.headers["X-Remote-User"] }
+
   # Rate limiting (requests per minute, per IP)
   # The REST API and MCP endpoint share api_rate_limit; the web UI uses
   # web_rate_limit. Both are enforced through Rails.cache.

data/lib/tasks/completion_kit_runs.rake

@@ -1,7 +1,7 @@
 namespace :completion_kit do
-  desc "Mark in-flight runs as failed (for use after the queue adapter cutover)"
+  desc "Mark in-flight runs in every tenant as failed (for use after the queue adapter cutover)"
   task mark_interrupted_runs_failed: :environment do
-    scope = CompletionKit::Run.where(status: "running")
+    scope = CompletionKit::Run.unscoped.where(status: "running")
     count = scope.count
     scope.update_all(
       status: "failed",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.43
+  version: 0.28.45
 platform: ruby
 authors:
 - Damien Bastin
@@ -238,7 +238,6 @@ files:
 - app/models/completion_kit/application_record.rb
 - app/models/completion_kit/dashboard_dismissal.rb
 - app/models/completion_kit/dataset.rb
-- app/models/completion_kit/mcp_session.rb
 - app/models/completion_kit/metric.rb
 - app/models/completion_kit/metric_group.rb
 - app/models/completion_kit/metric_group_membership.rb
@@ -256,6 +255,7 @@ files:
 - app/models/completion_kit/tag.rb
 - app/models/completion_kit/tagging.rb
 - app/models/concerns/completion_kit/has_job_status.rb
+- app/models/concerns/completion_kit/metric_selection.rb
 - app/models/concerns/completion_kit/taggable.rb
 - app/services/completion_kit/agreement_math.rb
 - app/services/completion_kit/anthropic_client.rb
@@ -274,6 +274,7 @@ files:
 - app/services/completion_kit/checks/valid_json.rb
 - app/services/completion_kit/csv_processor.rb
 - app/services/completion_kit/dashboard_stats.rb
+- app/services/completion_kit/json_object.rb
 - app/services/completion_kit/judge_service.rb
 - app/services/completion_kit/llm_client.rb
 - app/services/completion_kit/mcp_dispatcher.rb
@@ -398,6 +399,7 @@ files:
 - app/views/completion_kit/tags/index.html.erb
 - app/views/completion_kit/tags/new.html.erb
 - app/views/layouts/completion_kit/application.html.erb
+- config/locales/en.yml
 - config/routes.rb
 - db/migrate/20260311000001_create_completion_kit_tables.rb
 - db/migrate/20260326000001_rename_criteria_to_instruction_on_metrics_and_reviews.rb

data/app/models/completion_kit/mcp_session.rb

@@ -1,54 +0,0 @@
-module CompletionKit
-  # MCP session marker — one row per active client session, kept in the
-  # database so sessions survive Puma restarts, deploys, and Rails.cache
-  # eviction.
-  #
-  # Two things to know about why every query goes through `unscoped`:
-  #
-  # 1. CompletionKit::ApplicationRecord applies the host app's tenant_scope as
-  #    a default_scope. MCP sessions are per-CONNECTION, not per-tenant — the
-  #    table has no organization_id column. Letting the tenant_scope into a
-  #    session lookup turns "is this session live?" into either a SQL error
-  #    (no such column) or a false negative (`WHERE 1=0`), which surfaces to
-  #    the client as a spurious "Session not initialized" right after init.
-  # 2. `active?` slides expires_at forward when a session is more than halfway
-  #    through its TTL, so an MCP connection that keeps making calls stays
-  #    alive instead of expiring on the original 1-hour wall clock.
-  #
-  # Expired rows are opportunistically pruned on every new session start, so
-  # the table stays bounded by recent activity.
-  class McpSession < ApplicationRecord
-    self.table_name = "completion_kit_mcp_sessions"
-
-    SESSION_TTL = 1.hour
-
-    def self.start!
-      prune_expired!
-      unscoped.create!(session_id: SecureRandom.uuid, expires_at: SESSION_TTL.from_now).session_id
-    end
-
-    def self.active?(session_id)
-      return false if session_id.blank?
-
-      row = unscoped.where(session_id: session_id).where("expires_at > ?", Time.current).first
-      return false unless row
-
-      slide_expiry(row)
-      true
-    end
-
-    def self.destroy_session(session_id)
-      unscoped.where(session_id: session_id).delete_all
-    end
-
-    def self.prune_expired!
-      unscoped.where("expires_at < ?", Time.current).delete_all
-    end
-
-    def self.slide_expiry(row)
-      half_ttl_from_now = (SESSION_TTL / 2).from_now
-      return if row.expires_at > half_ttl_from_now
-      row.update_column(:expires_at, SESSION_TTL.from_now)
-    end
-  end
-end