completion-kit 0.1.0.rc1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +97 -86
- data/app/controllers/completion_kit/api/v1/metric_groups_controller.rb +53 -0
- data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
- data/app/controllers/completion_kit/api/v1/runs_controller.rb +2 -10
- data/app/controllers/completion_kit/metric_groups_controller.rb +59 -0
- data/app/controllers/completion_kit/metrics_controller.rb +2 -2
- data/app/controllers/completion_kit/runs_controller.rb +4 -11
- data/app/helpers/completion_kit/application_helper.rb +1 -8
- data/app/models/completion_kit/application_record.rb +7 -0
- data/app/models/completion_kit/metric.rb +4 -6
- data/app/models/completion_kit/metric_group.rb +30 -0
- data/app/models/completion_kit/metric_group_membership.rb +20 -0
- data/app/models/completion_kit/model.rb +1 -1
- data/app/models/completion_kit/provider_credential.rb +2 -1
- data/app/models/completion_kit/run.rb +11 -4
- data/app/services/completion_kit/anthropic_client.rb +4 -17
- data/app/services/completion_kit/judge_service.rb +3 -7
- data/app/services/completion_kit/llm_client.rb +15 -0
- data/app/services/completion_kit/mcp_dispatcher.rb +2 -2
- data/app/services/completion_kit/mcp_tools/base.rb +23 -0
- data/app/services/completion_kit/mcp_tools/datasets.rb +2 -18
- data/app/services/completion_kit/mcp_tools/metric_groups.rb +82 -0
- data/app/services/completion_kit/mcp_tools/metrics.rb +4 -22
- data/app/services/completion_kit/mcp_tools/prompts.rb +2 -18
- data/app/services/completion_kit/mcp_tools/provider_credentials.rb +2 -18
- data/app/services/completion_kit/mcp_tools/responses.rb +2 -13
- data/app/services/completion_kit/mcp_tools/runs.rb +4 -28
- data/app/services/completion_kit/ollama_client.rb +2 -15
- data/app/services/completion_kit/open_ai_client.rb +1 -10
- data/app/services/completion_kit/open_router_client.rb +1 -12
- data/app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb +15 -0
- data/app/views/completion_kit/api_reference/index.html.erb +11 -11
- data/app/views/completion_kit/metric_groups/_form.html.erb +46 -0
- data/app/views/completion_kit/metric_groups/edit.html.erb +13 -0
- data/app/views/completion_kit/metric_groups/index.html.erb +41 -0
- data/app/views/completion_kit/metric_groups/new.html.erb +12 -0
- data/app/views/completion_kit/{criteria → metric_groups}/show.html.erb +8 -9
- data/app/views/completion_kit/metrics/_form.html.erb +2 -23
- data/app/views/completion_kit/metrics/index.html.erb +13 -5
- data/app/views/completion_kit/metrics/show.html.erb +1 -12
- data/app/views/completion_kit/runs/_form.html.erb +5 -5
- data/app/views/layouts/completion_kit/application.html.erb +4 -1
- data/config/routes.rb +2 -2
- data/db/migrate/20260416000001_remove_evaluation_steps_from_metrics.rb +5 -0
- data/db/migrate/20260417000001_rename_criteria_to_metric_groups.rb +13 -0
- data/lib/completion_kit/engine.rb +1 -7
- data/lib/completion_kit/version.rb +1 -1
- data/lib/completion_kit.rb +5 -0
- metadata +23 -21
- data/app/assets/javascripts/completion_kit/evaluation_steps_controller.js +0 -25
- data/app/controllers/completion_kit/api/v1/criteria_controller.rb +0 -62
- data/app/controllers/completion_kit/criteria_controller.rb +0 -67
- data/app/models/completion_kit/criteria.rb +0 -22
- data/app/models/completion_kit/criteria_membership.rb +0 -20
- data/app/services/completion_kit/mcp_tools/criteria.rb +0 -106
- data/app/views/completion_kit/criteria/_form.html.erb +0 -46
- data/app/views/completion_kit/criteria/edit.html.erb +0 -14
- data/app/views/completion_kit/criteria/index.html.erb +0 -37
- data/app/views/completion_kit/criteria/new.html.erb +0 -13
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
module CompletionKit
|
|
2
|
-
module Api
|
|
3
|
-
module V1
|
|
4
|
-
class CriteriaController < BaseController
|
|
5
|
-
before_action :set_criteria, only: [:show, :update, :destroy]
|
|
6
|
-
|
|
7
|
-
def index
|
|
8
|
-
render json: Criteria.order(created_at: :desc)
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
def show
|
|
12
|
-
render json: @criteria
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
def create
|
|
16
|
-
criteria = Criteria.new(criteria_params.except(:metric_ids))
|
|
17
|
-
if criteria.save
|
|
18
|
-
replace_metric_memberships(criteria, params[:metric_ids]) if params.key?(:metric_ids)
|
|
19
|
-
render json: criteria.reload, status: :created
|
|
20
|
-
else
|
|
21
|
-
render json: {errors: criteria.errors}, status: :unprocessable_entity
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def update
|
|
26
|
-
if @criteria.update(criteria_params.except(:metric_ids))
|
|
27
|
-
replace_metric_memberships(@criteria, params[:metric_ids]) if params.key?(:metric_ids)
|
|
28
|
-
render json: @criteria.reload
|
|
29
|
-
else
|
|
30
|
-
render json: {errors: @criteria.errors}, status: :unprocessable_entity
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def destroy
|
|
35
|
-
@criteria.destroy!
|
|
36
|
-
head :no_content
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
private
|
|
40
|
-
|
|
41
|
-
def set_criteria
|
|
42
|
-
@criteria = Criteria.find(params[:id])
|
|
43
|
-
rescue ActiveRecord::RecordNotFound
|
|
44
|
-
not_found
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def criteria_params
|
|
48
|
-
params.permit(:name, :description, metric_ids: [])
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def replace_metric_memberships(criteria, metric_ids)
|
|
52
|
-
return unless metric_ids
|
|
53
|
-
|
|
54
|
-
criteria.criteria_memberships.delete_all
|
|
55
|
-
Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
|
|
56
|
-
criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
end
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
end
|
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
module CompletionKit
|
|
2
|
-
class CriteriaController < ApplicationController
|
|
3
|
-
before_action :set_criteria, only: [:show, :edit, :update, :destroy]
|
|
4
|
-
|
|
5
|
-
def index
|
|
6
|
-
@criterias = Criteria.includes(:metrics).order(:name)
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
def show
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def new
|
|
13
|
-
@criteria = Criteria.new
|
|
14
|
-
@metrics = Metric.order(:name)
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def edit
|
|
18
|
-
@metrics = Metric.order(:name)
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
def create
|
|
22
|
-
@criteria = Criteria.new(criteria_params.except(:metric_ids))
|
|
23
|
-
@metrics = Metric.order(:name)
|
|
24
|
-
|
|
25
|
-
if @criteria.save
|
|
26
|
-
replace_metric_memberships
|
|
27
|
-
redirect_to criterion_path(@criteria), notice: "Criteria was successfully created."
|
|
28
|
-
else
|
|
29
|
-
render :new, status: :unprocessable_entity
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
def update
|
|
34
|
-
@metrics = Metric.order(:name)
|
|
35
|
-
|
|
36
|
-
if @criteria.update(criteria_params.except(:metric_ids))
|
|
37
|
-
replace_metric_memberships
|
|
38
|
-
redirect_to criterion_path(@criteria), notice: "Criteria was successfully updated."
|
|
39
|
-
else
|
|
40
|
-
render :edit, status: :unprocessable_entity
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
def destroy
|
|
45
|
-
@criteria.destroy
|
|
46
|
-
redirect_to criteria_path, notice: "Criteria was successfully destroyed."
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
private
|
|
50
|
-
|
|
51
|
-
def set_criteria
|
|
52
|
-
@criteria = Criteria.find(params[:id])
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def criteria_params
|
|
56
|
-
params.require(:criteria).permit(:name, :description, metric_ids: [])
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
def replace_metric_memberships
|
|
60
|
-
metric_ids = Array(criteria_params[:metric_ids]).reject(&:blank?)
|
|
61
|
-
@criteria.criteria_memberships.delete_all
|
|
62
|
-
metric_ids.each_with_index do |metric_id, index|
|
|
63
|
-
@criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
|
|
64
|
-
end
|
|
65
|
-
end
|
|
66
|
-
end
|
|
67
|
-
end
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
module CompletionKit
|
|
2
|
-
class Criteria < ApplicationRecord
|
|
3
|
-
self.table_name = "completion_kit_criteria"
|
|
4
|
-
|
|
5
|
-
has_many :criteria_memberships, -> { order(:position, :id) }, dependent: :destroy
|
|
6
|
-
has_many :metrics, through: :criteria_memberships
|
|
7
|
-
|
|
8
|
-
validates :name, presence: true
|
|
9
|
-
|
|
10
|
-
def ordered_metrics
|
|
11
|
-
criteria_memberships.includes(:metric).map(&:metric).compact
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def as_json(options = {})
|
|
15
|
-
{
|
|
16
|
-
id: id, name: name, description: description,
|
|
17
|
-
created_at: created_at, updated_at: updated_at,
|
|
18
|
-
metric_ids: metric_ids
|
|
19
|
-
}
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
end
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
module CompletionKit
|
|
2
|
-
class CriteriaMembership < ApplicationRecord
|
|
3
|
-
self.table_name = "completion_kit_criteria_memberships"
|
|
4
|
-
|
|
5
|
-
belongs_to :criteria, class_name: "CompletionKit::Criteria", foreign_key: "criteria_id"
|
|
6
|
-
belongs_to :metric
|
|
7
|
-
|
|
8
|
-
validates :metric_id, uniqueness: { scope: :criteria_id }
|
|
9
|
-
|
|
10
|
-
before_validation :set_default_position
|
|
11
|
-
|
|
12
|
-
private
|
|
13
|
-
|
|
14
|
-
def set_default_position
|
|
15
|
-
return if position.present? || criteria.blank?
|
|
16
|
-
|
|
17
|
-
self.position = criteria.criteria_memberships.maximum(:position).to_i + 1
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
end
|
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
module CompletionKit
|
|
2
|
-
module McpTools
|
|
3
|
-
module Criteria
|
|
4
|
-
TOOLS = {
|
|
5
|
-
"criteria_list" => {
|
|
6
|
-
description: "List all criteria",
|
|
7
|
-
inputSchema: {type: "object", properties: {}, required: []},
|
|
8
|
-
handler: :list
|
|
9
|
-
},
|
|
10
|
-
"criteria_get" => {
|
|
11
|
-
description: "Get a criteria by ID",
|
|
12
|
-
inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
|
|
13
|
-
handler: :get
|
|
14
|
-
},
|
|
15
|
-
"criteria_create" => {
|
|
16
|
-
description: "Create a criteria grouping metrics",
|
|
17
|
-
inputSchema: {
|
|
18
|
-
type: "object",
|
|
19
|
-
properties: {
|
|
20
|
-
name: {type: "string"}, description: {type: "string"},
|
|
21
|
-
metric_ids: {type: "array", items: {type: "integer"}}
|
|
22
|
-
},
|
|
23
|
-
required: ["name"]
|
|
24
|
-
},
|
|
25
|
-
handler: :create
|
|
26
|
-
},
|
|
27
|
-
"criteria_update" => {
|
|
28
|
-
description: "Update a criteria",
|
|
29
|
-
inputSchema: {
|
|
30
|
-
type: "object",
|
|
31
|
-
properties: {
|
|
32
|
-
id: {type: "integer"}, name: {type: "string"}, description: {type: "string"},
|
|
33
|
-
metric_ids: {type: "array", items: {type: "integer"}}
|
|
34
|
-
},
|
|
35
|
-
required: ["id"]
|
|
36
|
-
},
|
|
37
|
-
handler: :update
|
|
38
|
-
},
|
|
39
|
-
"criteria_delete" => {
|
|
40
|
-
description: "Delete a criteria",
|
|
41
|
-
inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
|
|
42
|
-
handler: :delete
|
|
43
|
-
}
|
|
44
|
-
}.freeze
|
|
45
|
-
|
|
46
|
-
def self.definitions
|
|
47
|
-
TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def self.call(name, arguments)
|
|
51
|
-
tool = TOOLS.fetch(name)
|
|
52
|
-
send(tool[:handler], arguments)
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def self.list(_args)
|
|
56
|
-
text_result(CompletionKit::Criteria.order(created_at: :desc).map(&:as_json))
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
def self.get(args)
|
|
60
|
-
text_result(CompletionKit::Criteria.find(args["id"]).as_json)
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
def self.create(args)
|
|
64
|
-
criteria = CompletionKit::Criteria.new(args.slice("name", "description"))
|
|
65
|
-
if criteria.save
|
|
66
|
-
replace_metric_memberships(criteria, args["metric_ids"])
|
|
67
|
-
text_result(criteria.reload.as_json)
|
|
68
|
-
else
|
|
69
|
-
error_result(criteria.errors.full_messages.join(", "))
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
def self.update(args)
|
|
74
|
-
criteria = CompletionKit::Criteria.find(args["id"])
|
|
75
|
-
if criteria.update(args.except("id", "metric_ids").slice("name", "description"))
|
|
76
|
-
replace_metric_memberships(criteria, args["metric_ids"]) if args.key?("metric_ids")
|
|
77
|
-
text_result(criteria.reload.as_json)
|
|
78
|
-
else
|
|
79
|
-
error_result(criteria.errors.full_messages.join(", "))
|
|
80
|
-
end
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
def self.delete(args)
|
|
84
|
-
CompletionKit::Criteria.find(args["id"]).destroy!
|
|
85
|
-
text_result("Criteria #{args["id"]} deleted")
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
def self.text_result(data)
|
|
89
|
-
text = data.is_a?(String) ? data : data.to_json
|
|
90
|
-
{content: [{type: "text", text: text}]}
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
def self.error_result(message)
|
|
94
|
-
{content: [{type: "text", text: message}], isError: true}
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
def self.replace_metric_memberships(criteria, metric_ids)
|
|
98
|
-
return unless metric_ids
|
|
99
|
-
criteria.criteria_memberships.delete_all
|
|
100
|
-
Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
|
|
101
|
-
criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
end
|
|
105
|
-
end
|
|
106
|
-
end
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
<%= form_with(model: criteria, url: criteria.persisted? ? criterion_path(criteria) : criteria_path, local: true) do |form| %>
|
|
2
|
-
<% if criteria.errors.any? %>
|
|
3
|
-
<div class="ck-flash ck-flash--alert">
|
|
4
|
-
<p class="ck-flash__title"><%= pluralize(criteria.errors.count, "problem") %> prevented this criteria from being saved.</p>
|
|
5
|
-
<ul class="ck-error-list">
|
|
6
|
-
<% criteria.errors.full_messages.each do |message| %>
|
|
7
|
-
<li><%= message %></li>
|
|
8
|
-
<% end %>
|
|
9
|
-
</ul>
|
|
10
|
-
</div>
|
|
11
|
-
<% end %>
|
|
12
|
-
|
|
13
|
-
<div class="ck-card ck-form-card">
|
|
14
|
-
<div class="ck-field">
|
|
15
|
-
<%= form.label :name, "Criteria name", class: "ck-label" %>
|
|
16
|
-
<%= form.text_field :name, class: "ck-input", placeholder: "Support quality" %>
|
|
17
|
-
</div>
|
|
18
|
-
|
|
19
|
-
<div class="ck-field">
|
|
20
|
-
<%= form.label :description, class: "ck-label" %>
|
|
21
|
-
<%= form.text_area :description, rows: 3, class: "ck-input ck-input--area", placeholder: "When this criteria should be used." %>
|
|
22
|
-
</div>
|
|
23
|
-
|
|
24
|
-
<div class="ck-field">
|
|
25
|
-
<p class="ck-label">Metrics in this criteria</p>
|
|
26
|
-
<p class="ck-hint">A criteria groups metrics together for judging.</p>
|
|
27
|
-
<div class="ck-list ck-list--compact">
|
|
28
|
-
<% @metrics.each do |metric| %>
|
|
29
|
-
<label class="ck-item">
|
|
30
|
-
<span>
|
|
31
|
-
<strong><%= metric.name %></strong>
|
|
32
|
-
<span class="ck-meta-copy"><%= metric.instruction.presence || "No instruction set." %></span>
|
|
33
|
-
</span>
|
|
34
|
-
<%= check_box_tag "criteria[metric_ids][]", metric.id, criteria.metrics.exists?(metric.id), class: "ck-checkbox" %>
|
|
35
|
-
</label>
|
|
36
|
-
<% end %>
|
|
37
|
-
</div>
|
|
38
|
-
<%= hidden_field_tag "criteria[metric_ids][]", "" %>
|
|
39
|
-
</div>
|
|
40
|
-
|
|
41
|
-
<div class="ck-actions">
|
|
42
|
-
<%= link_to "Cancel", criteria_path, class: ck_button_classes(:light, variant: :outline) %>
|
|
43
|
-
<%= form.submit(criteria.persisted? ? "Save criteria" : "Create criteria", class: ck_button_classes(:dark)) %>
|
|
44
|
-
</div>
|
|
45
|
-
</div>
|
|
46
|
-
<% end %>
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
<ol class="ck-breadcrumb">
|
|
2
|
-
<li><%= link_to "Metrics", metrics_path %></li>
|
|
3
|
-
<li><%= link_to "Criteria", criteria_path %></li>
|
|
4
|
-
<li><%= link_to @criteria.name, criterion_path(@criteria) %></li>
|
|
5
|
-
<li>Edit</li>
|
|
6
|
-
</ol>
|
|
7
|
-
|
|
8
|
-
<section class="ck-page-header">
|
|
9
|
-
<div>
|
|
10
|
-
<h1 class="ck-title">Edit criteria</h1>
|
|
11
|
-
</div>
|
|
12
|
-
</section>
|
|
13
|
-
|
|
14
|
-
<%= render "form", criteria: @criteria %>
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
<ol class="ck-breadcrumb">
|
|
2
|
-
<li><%= link_to "Metrics", metrics_path %></li>
|
|
3
|
-
<li>Criteria</li>
|
|
4
|
-
</ol>
|
|
5
|
-
|
|
6
|
-
<section class="ck-page-header">
|
|
7
|
-
<div>
|
|
8
|
-
<h1 class="ck-title">Criteria</h1>
|
|
9
|
-
<p class="ck-lead">Named groups of metrics that can be applied to a run as a set. Use criteria to bundle related metrics together for reuse across runs.</p>
|
|
10
|
-
</div>
|
|
11
|
-
<div class="ck-actions">
|
|
12
|
-
<%= link_to "New criteria", new_criterion_path, class: ck_button_classes(:dark) %>
|
|
13
|
-
</div>
|
|
14
|
-
</section>
|
|
15
|
-
|
|
16
|
-
<% if @criterias.any? %>
|
|
17
|
-
<table class="ck-results-table">
|
|
18
|
-
<thead>
|
|
19
|
-
<tr>
|
|
20
|
-
<th>Name</th>
|
|
21
|
-
<th>Metrics</th>
|
|
22
|
-
<th></th>
|
|
23
|
-
</tr>
|
|
24
|
-
</thead>
|
|
25
|
-
<tbody>
|
|
26
|
-
<% @criterias.each do |criteria| %>
|
|
27
|
-
<tr onclick="window.location='<%= criterion_path(criteria) %>'" style="cursor: pointer;">
|
|
28
|
-
<td><strong><%= criteria.name %></strong></td>
|
|
29
|
-
<td><%= criteria.metrics.size %></td>
|
|
30
|
-
<td class="ck-results-table__arrow">→</td>
|
|
31
|
-
</tr>
|
|
32
|
-
<% end %>
|
|
33
|
-
</tbody>
|
|
34
|
-
</table>
|
|
35
|
-
<% else %>
|
|
36
|
-
<div class="ck-empty">No criteria yet.</div>
|
|
37
|
-
<% end %>
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
<ol class="ck-breadcrumb">
|
|
2
|
-
<li><%= link_to "Metrics", metrics_path %></li>
|
|
3
|
-
<li><%= link_to "Criteria", criteria_path %></li>
|
|
4
|
-
<li>New</li>
|
|
5
|
-
</ol>
|
|
6
|
-
|
|
7
|
-
<section class="ck-page-header">
|
|
8
|
-
<div>
|
|
9
|
-
<h1 class="ck-title">New criteria</h1>
|
|
10
|
-
</div>
|
|
11
|
-
</section>
|
|
12
|
-
|
|
13
|
-
<%= render "form", criteria: @criteria %>
|