completion-kit 0.1.0.rc1 → 0.1.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc1
+  version: 0.1.0
 platform: ruby
 authors:
 - Damien Bastin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-04-16 00:00:00.000000000 Z
+date: 2026-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -198,12 +198,11 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.22'
-description: 'CompletionKit is a mountable Rails engine for rigorous prompt testing:
-  run prompts against datasets, score outputs with an LLM-as-judge against custom
-  metrics and rubrics, track versions, and iterate with AI-assisted improvement suggestions.
-  Every resource is accessible through a bearer-token REST API and a built-in Model
-  Context Protocol server, so you can drive it from a browser, from HTTP clients,
-  or directly from agents like Claude Code.'
+description: CompletionKit is a prompt testing platform that runs as a Rails engine
+  or a standalone app. Run prompts against real datasets, score every output with
+  an LLM judge against criteria you define, track prompt versions, and get AI-generated
+  improvement suggestions grounded in your actual results. Includes a web UI, REST
+  API, and a built-in MCP server with 36 tools.
 email:
 - damien@homemade.software
 executables: []
@@ -216,11 +215,10 @@ files:
 - app/assets/config/completion_kit_manifest.js
 - app/assets/config/manifest.js
 - app/assets/images/completion_kit/logo.svg
-- app/assets/javascripts/completion_kit/evaluation_steps_controller.js
 - app/assets/stylesheets/completion_kit/application.css
 - app/controllers/completion_kit/api/v1/base_controller.rb
-- app/controllers/completion_kit/api/v1/criteria_controller.rb
 - app/controllers/completion_kit/api/v1/datasets_controller.rb
+- app/controllers/completion_kit/api/v1/metric_groups_controller.rb
 - app/controllers/completion_kit/api/v1/metrics_controller.rb
 - app/controllers/completion_kit/api/v1/prompts_controller.rb
 - app/controllers/completion_kit/api/v1/provider_credentials_controller.rb
@@ -228,9 +226,9 @@ files:
 - app/controllers/completion_kit/api/v1/runs_controller.rb
 - app/controllers/completion_kit/api_reference_controller.rb
 - app/controllers/completion_kit/application_controller.rb
-- app/controllers/completion_kit/criteria_controller.rb
 - app/controllers/completion_kit/datasets_controller.rb
 - app/controllers/completion_kit/mcp_controller.rb
+- app/controllers/completion_kit/metric_groups_controller.rb
 - app/controllers/completion_kit/metrics_controller.rb
 - app/controllers/completion_kit/prompts_controller.rb
 - app/controllers/completion_kit/provider_credentials_controller.rb
@@ -243,10 +241,10 @@ files:
 - app/jobs/completion_kit/model_discovery_job.rb
 - app/mailers/completion_kit/application_mailer.rb
 - app/models/completion_kit/application_record.rb
-- app/models/completion_kit/criteria.rb
-- app/models/completion_kit/criteria_membership.rb
 - app/models/completion_kit/dataset.rb
 - app/models/completion_kit/metric.rb
+- app/models/completion_kit/metric_group.rb
+- app/models/completion_kit/metric_group_membership.rb
 - app/models/completion_kit/model.rb
 - app/models/completion_kit/prompt.rb
 - app/models/completion_kit/provider_credential.rb
@@ -261,8 +259,8 @@ files:
 - app/services/completion_kit/judge_service.rb
 - app/services/completion_kit/llm_client.rb
 - app/services/completion_kit/mcp_dispatcher.rb
-- app/services/completion_kit/mcp_tools/criteria.rb
 - app/services/completion_kit/mcp_tools/datasets.rb
+- app/services/completion_kit/mcp_tools/metric_groups.rb
 - app/services/completion_kit/mcp_tools/metrics.rb
 - app/services/completion_kit/mcp_tools/prompts.rb
 - app/services/completion_kit/mcp_tools/provider_credentials.rb
@@ -275,16 +273,16 @@ files:
 - app/services/completion_kit/prompt_improvement_service.rb
 - app/views/completion_kit/api_reference/_example.html.erb
 - app/views/completion_kit/api_reference/index.html.erb
-- app/views/completion_kit/criteria/_form.html.erb
-- app/views/completion_kit/criteria/edit.html.erb
-- app/views/completion_kit/criteria/index.html.erb
-- app/views/completion_kit/criteria/new.html.erb
-- app/views/completion_kit/criteria/show.html.erb
 - app/views/completion_kit/datasets/_form.html.erb
 - app/views/completion_kit/datasets/edit.html.erb
 - app/views/completion_kit/datasets/index.html.erb
 - app/views/completion_kit/datasets/new.html.erb
 - app/views/completion_kit/datasets/show.html.erb
+- app/views/completion_kit/metric_groups/_form.html.erb
+- app/views/completion_kit/metric_groups/edit.html.erb
+- app/views/completion_kit/metric_groups/index.html.erb
+- app/views/completion_kit/metric_groups/new.html.erb
+- app/views/completion_kit/metric_groups/show.html.erb
 - app/views/completion_kit/metrics/_form.html.erb
 - app/views/completion_kit/metrics/edit.html.erb
 - app/views/completion_kit/metrics/index.html.erb
@@ -324,6 +322,8 @@ files:
 - db/migrate/20260403000001_add_temperature_to_completion_kit_runs.rb
 - db/migrate/20260403000002_create_completion_kit_suggestions.rb
 - db/migrate/20260403000003_add_applied_at_to_completion_kit_suggestions.rb
+- db/migrate/20260416000001_remove_evaluation_steps_from_metrics.rb
+- db/migrate/20260417000001_rename_criteria_to_metric_groups.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/engine.rb
@@ -356,6 +356,6 @@ requirements: []
 rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
-summary: Rails engine for testing, scoring, and improving GenAI prompts against real
-  data — with a web UI, REST API, and built-in MCP server.
+summary: Your prompts need tests too. Run them against real data, score outputs with
+  an LLM judge, iterate until they work.
 test_files: []

data/app/assets/javascripts/completion_kit/evaluation_steps_controller.js

@@ -1,25 +0,0 @@
-document.addEventListener("DOMContentLoaded", function () {
-  document.addEventListener("click", function (event) {
-    var addBtn = event.target.closest("[data-action='evaluation-steps#add']");
-    if (addBtn) {
-      var container = addBtn.closest("[data-controller='evaluation-steps']");
-      var list = container.querySelector("[data-evaluation-steps-target='list']");
-      var row = document.createElement("div");
-      row.className = "ck-step-row";
-      row.setAttribute("data-evaluation-steps-target", "row");
-      row.innerHTML =
-        '<input type="text" name="metric[evaluation_steps][]" value="" class="ck-input" placeholder="Describe this evaluation step..." />' +
-        '<button type="button" class="ck-icon-btn" data-action="evaluation-steps#remove" aria-label="Remove step">' +
-        '<svg viewBox="0 0 24 24" width="16" height="16" fill="none" stroke="currentColor" stroke-width="1.75"><path d="M3 6h18"/><path d="M19 6v14c0 1-1 2-2 2H7c-1 0-2-1-2-2V6"/><path d="M8 6V4c0-1 1-2 2-2h4c1 0 2 1 2 2v2"/><line x1="10" y1="11" x2="10" y2="17"/><line x1="14" y1="11" x2="14" y2="17"/></svg>' +
-        "</button>";
-      list.appendChild(row);
-      row.querySelector("input").focus();
-    }
-
-    var removeBtn = event.target.closest("[data-action='evaluation-steps#remove']");
-    if (removeBtn) {
-      var stepRow = removeBtn.closest("[data-evaluation-steps-target='row']");
-      if (stepRow) stepRow.remove();
-    }
-  });
-});

data/app/controllers/completion_kit/api/v1/criteria_controller.rb

@@ -1,62 +0,0 @@
-module CompletionKit
-  module Api
-    module V1
-      class CriteriaController < BaseController
-        before_action :set_criteria, only: [:show, :update, :destroy]
-
-        def index
-          render json: Criteria.order(created_at: :desc)
-        end
-
-        def show
-          render json: @criteria
-        end
-
-        def create
-          criteria = Criteria.new(criteria_params.except(:metric_ids))
-          if criteria.save
-            replace_metric_memberships(criteria, params[:metric_ids]) if params.key?(:metric_ids)
-            render json: criteria.reload, status: :created
-          else
-            render json: {errors: criteria.errors}, status: :unprocessable_entity
-          end
-        end
-
-        def update
-          if @criteria.update(criteria_params.except(:metric_ids))
-            replace_metric_memberships(@criteria, params[:metric_ids]) if params.key?(:metric_ids)
-            render json: @criteria.reload
-          else
-            render json: {errors: @criteria.errors}, status: :unprocessable_entity
-          end
-        end
-
-        def destroy
-          @criteria.destroy!
-          head :no_content
-        end
-
-        private
-
-        def set_criteria
-          @criteria = Criteria.find(params[:id])
-        rescue ActiveRecord::RecordNotFound
-          not_found
-        end
-
-        def criteria_params
-          params.permit(:name, :description, metric_ids: [])
-        end
-
-        def replace_metric_memberships(criteria, metric_ids)
-          return unless metric_ids
-
-          criteria.criteria_memberships.delete_all
-          Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
-            criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
-          end
-        end
-      end
-    end
-  end
-end

data/app/controllers/completion_kit/criteria_controller.rb

@@ -1,67 +0,0 @@
-module CompletionKit
-  class CriteriaController < ApplicationController
-    before_action :set_criteria, only: [:show, :edit, :update, :destroy]
-
-    def index
-      @criterias = Criteria.includes(:metrics).order(:name)
-    end
-
-    def show
-    end
-
-    def new
-      @criteria = Criteria.new
-      @metrics = Metric.order(:name)
-    end
-
-    def edit
-      @metrics = Metric.order(:name)
-    end
-
-    def create
-      @criteria = Criteria.new(criteria_params.except(:metric_ids))
-      @metrics = Metric.order(:name)
-
-      if @criteria.save
-        replace_metric_memberships
-        redirect_to criterion_path(@criteria), notice: "Criteria was successfully created."
-      else
-        render :new, status: :unprocessable_entity
-      end
-    end
-
-    def update
-      @metrics = Metric.order(:name)
-
-      if @criteria.update(criteria_params.except(:metric_ids))
-        replace_metric_memberships
-        redirect_to criterion_path(@criteria), notice: "Criteria was successfully updated."
-      else
-        render :edit, status: :unprocessable_entity
-      end
-    end
-
-    def destroy
-      @criteria.destroy
-      redirect_to criteria_path, notice: "Criteria was successfully destroyed."
-    end
-
-    private
-
-    def set_criteria
-      @criteria = Criteria.find(params[:id])
-    end
-
-    def criteria_params
-      params.require(:criteria).permit(:name, :description, metric_ids: [])
-    end
-
-    def replace_metric_memberships
-      metric_ids = Array(criteria_params[:metric_ids]).reject(&:blank?)
-      @criteria.criteria_memberships.delete_all
-      metric_ids.each_with_index do |metric_id, index|
-        @criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
-      end
-    end
-  end
-end

data/app/models/completion_kit/criteria.rb

@@ -1,22 +0,0 @@
-module CompletionKit
-  class Criteria < ApplicationRecord
-    self.table_name = "completion_kit_criteria"
-
-    has_many :criteria_memberships, -> { order(:position, :id) }, dependent: :destroy
-    has_many :metrics, through: :criteria_memberships
-
-    validates :name, presence: true
-
-    def ordered_metrics
-      criteria_memberships.includes(:metric).map(&:metric).compact
-    end
-
-    def as_json(options = {})
-      {
-        id: id, name: name, description: description,
-        created_at: created_at, updated_at: updated_at,
-        metric_ids: metric_ids
-      }
-    end
-  end
-end

data/app/models/completion_kit/criteria_membership.rb

@@ -1,20 +0,0 @@
-module CompletionKit
-  class CriteriaMembership < ApplicationRecord
-    self.table_name = "completion_kit_criteria_memberships"
-
-    belongs_to :criteria, class_name: "CompletionKit::Criteria", foreign_key: "criteria_id"
-    belongs_to :metric
-
-    validates :metric_id, uniqueness: { scope: :criteria_id }
-
-    before_validation :set_default_position
-
-    private
-
-    def set_default_position
-      return if position.present? || criteria.blank?
-
-      self.position = criteria.criteria_memberships.maximum(:position).to_i + 1
-    end
-  end
-end

data/app/views/completion_kit/criteria/_form.html.erb

@@ -1,46 +0,0 @@
-<%= form_with(model: criteria, url: criteria.persisted? ? criterion_path(criteria) : criteria_path, local: true) do |form| %>
-  <% if criteria.errors.any? %>
-    <div class="ck-flash ck-flash--alert">
-      <p class="ck-flash__title"><%= pluralize(criteria.errors.count, "problem") %> prevented this criteria from being saved.</p>
-      <ul class="ck-error-list">
-        <% criteria.errors.full_messages.each do |message| %>
-          <li><%= message %></li>
-        <% end %>
-      </ul>
-    </div>
-  <% end %>
-
-  <div class="ck-card ck-form-card">
-    <div class="ck-field">
-      <%= form.label :name, "Criteria name", class: "ck-label" %>
-      <%= form.text_field :name, class: "ck-input", placeholder: "Support quality" %>
-    </div>
-
-    <div class="ck-field">
-      <%= form.label :description, class: "ck-label" %>
-      <%= form.text_area :description, rows: 3, class: "ck-input ck-input--area", placeholder: "When this criteria should be used." %>
-    </div>
-
-    <div class="ck-field">
-      <p class="ck-label">Metrics in this criteria</p>
-      <p class="ck-hint">A criteria groups metrics together for judging.</p>
-      <div class="ck-list ck-list--compact">
-        <% @metrics.each do |metric| %>
-          <label class="ck-item">
-            <span>
-              <strong><%= metric.name %></strong>
-              <span class="ck-meta-copy"><%= metric.instruction.presence || "No instruction set." %></span>
-            </span>
-            <%= check_box_tag "criteria[metric_ids][]", metric.id, criteria.metrics.exists?(metric.id), class: "ck-checkbox" %>
-          </label>
-        <% end %>
-      </div>
-      <%= hidden_field_tag "criteria[metric_ids][]", "" %>
-    </div>
-
-    <div class="ck-actions">
-      <%= link_to "Cancel", criteria_path, class: ck_button_classes(:light, variant: :outline) %>
-      <%= form.submit(criteria.persisted? ? "Save criteria" : "Create criteria", class: ck_button_classes(:dark)) %>
-    </div>
-  </div>
-<% end %>

data/app/views/completion_kit/criteria/edit.html.erb

@@ -1,14 +0,0 @@
-<ol class="ck-breadcrumb">
-  <li><%= link_to "Metrics", metrics_path %></li>
-  <li><%= link_to "Criteria", criteria_path %></li>
-  <li><%= link_to @criteria.name, criterion_path(@criteria) %></li>
-  <li>Edit</li>
-</ol>
-
-<section class="ck-page-header">
-  <div>
-    <h1 class="ck-title">Edit criteria</h1>
-  </div>
-</section>
-
-<%= render "form", criteria: @criteria %>

data/app/views/completion_kit/criteria/index.html.erb

@@ -1,37 +0,0 @@
-<ol class="ck-breadcrumb">
-  <li><%= link_to "Metrics", metrics_path %></li>
-  <li>Criteria</li>
-</ol>
-
-<section class="ck-page-header">
-  <div>
-    <h1 class="ck-title">Criteria</h1>
-    <p class="ck-lead">Named groups of metrics that can be applied to a run as a set. Use criteria to bundle related metrics together for reuse across runs.</p>
-  </div>
-  <div class="ck-actions">
-    <%= link_to "New criteria", new_criterion_path, class: ck_button_classes(:dark) %>
-  </div>
-</section>
-
-<% if @criterias.any? %>
-  <table class="ck-results-table">
-    <thead>
-      <tr>
-        <th>Name</th>
-        <th>Metrics</th>
-        <th></th>
-      </tr>
-    </thead>
-    <tbody>
-      <% @criterias.each do |criteria| %>
-        <tr onclick="window.location='<%= criterion_path(criteria) %>'" style="cursor: pointer;">
-          <td><strong><%= criteria.name %></strong></td>
-          <td><%= criteria.metrics.size %></td>
-          <td class="ck-results-table__arrow">&rarr;</td>
-        </tr>
-      <% end %>
-    </tbody>
-  </table>
-<% else %>
-  <div class="ck-empty">No criteria yet.</div>
-<% end %>

data/app/views/completion_kit/criteria/new.html.erb

@@ -1,13 +0,0 @@
-<ol class="ck-breadcrumb">
-  <li><%= link_to "Metrics", metrics_path %></li>
-  <li><%= link_to "Criteria", criteria_path %></li>
-  <li>New</li>
-</ol>
-
-<section class="ck-page-header">
-  <div>
-    <h1 class="ck-title">New criteria</h1>
-  </div>
-</section>
-
-<%= render "form", criteria: @criteria %>