completion-kit 0.1.0.rc1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +97 -86
  3. data/app/controllers/completion_kit/api/v1/metric_groups_controller.rb +53 -0
  4. data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
  5. data/app/controllers/completion_kit/api/v1/runs_controller.rb +2 -10
  6. data/app/controllers/completion_kit/metric_groups_controller.rb +59 -0
  7. data/app/controllers/completion_kit/metrics_controller.rb +2 -2
  8. data/app/controllers/completion_kit/runs_controller.rb +4 -11
  9. data/app/helpers/completion_kit/application_helper.rb +1 -8
  10. data/app/models/completion_kit/application_record.rb +7 -0
  11. data/app/models/completion_kit/metric.rb +4 -6
  12. data/app/models/completion_kit/metric_group.rb +30 -0
  13. data/app/models/completion_kit/metric_group_membership.rb +20 -0
  14. data/app/models/completion_kit/model.rb +1 -1
  15. data/app/models/completion_kit/provider_credential.rb +2 -1
  16. data/app/models/completion_kit/run.rb +11 -4
  17. data/app/services/completion_kit/anthropic_client.rb +4 -17
  18. data/app/services/completion_kit/judge_service.rb +3 -7
  19. data/app/services/completion_kit/llm_client.rb +15 -0
  20. data/app/services/completion_kit/mcp_dispatcher.rb +2 -2
  21. data/app/services/completion_kit/mcp_tools/base.rb +23 -0
  22. data/app/services/completion_kit/mcp_tools/datasets.rb +2 -18
  23. data/app/services/completion_kit/mcp_tools/metric_groups.rb +82 -0
  24. data/app/services/completion_kit/mcp_tools/metrics.rb +4 -22
  25. data/app/services/completion_kit/mcp_tools/prompts.rb +2 -18
  26. data/app/services/completion_kit/mcp_tools/provider_credentials.rb +2 -18
  27. data/app/services/completion_kit/mcp_tools/responses.rb +2 -13
  28. data/app/services/completion_kit/mcp_tools/runs.rb +4 -28
  29. data/app/services/completion_kit/ollama_client.rb +2 -15
  30. data/app/services/completion_kit/open_ai_client.rb +1 -10
  31. data/app/services/completion_kit/open_router_client.rb +1 -12
  32. data/app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb +15 -0
  33. data/app/views/completion_kit/api_reference/index.html.erb +11 -11
  34. data/app/views/completion_kit/metric_groups/_form.html.erb +46 -0
  35. data/app/views/completion_kit/metric_groups/edit.html.erb +13 -0
  36. data/app/views/completion_kit/metric_groups/index.html.erb +41 -0
  37. data/app/views/completion_kit/metric_groups/new.html.erb +12 -0
  38. data/app/views/completion_kit/{criteria → metric_groups}/show.html.erb +8 -9
  39. data/app/views/completion_kit/metrics/_form.html.erb +2 -23
  40. data/app/views/completion_kit/metrics/index.html.erb +13 -5
  41. data/app/views/completion_kit/metrics/show.html.erb +1 -12
  42. data/app/views/completion_kit/runs/_form.html.erb +5 -5
  43. data/app/views/layouts/completion_kit/application.html.erb +4 -1
  44. data/config/routes.rb +2 -2
  45. data/db/migrate/20260416000001_remove_evaluation_steps_from_metrics.rb +5 -0
  46. data/db/migrate/20260417000001_rename_criteria_to_metric_groups.rb +13 -0
  47. data/lib/completion_kit/engine.rb +1 -7
  48. data/lib/completion_kit/version.rb +1 -1
  49. data/lib/completion_kit.rb +5 -0
  50. metadata +23 -21
  51. data/app/assets/javascripts/completion_kit/evaluation_steps_controller.js +0 -25
  52. data/app/controllers/completion_kit/api/v1/criteria_controller.rb +0 -62
  53. data/app/controllers/completion_kit/criteria_controller.rb +0 -67
  54. data/app/models/completion_kit/criteria.rb +0 -22
  55. data/app/models/completion_kit/criteria_membership.rb +0 -20
  56. data/app/services/completion_kit/mcp_tools/criteria.rb +0 -106
  57. data/app/views/completion_kit/criteria/_form.html.erb +0 -46
  58. data/app/views/completion_kit/criteria/edit.html.erb +0 -14
  59. data/app/views/completion_kit/criteria/index.html.erb +0 -37
  60. data/app/views/completion_kit/criteria/new.html.erb +0 -13
@@ -1,62 +0,0 @@
1
- module CompletionKit
2
- module Api
3
- module V1
4
- class CriteriaController < BaseController
5
- before_action :set_criteria, only: [:show, :update, :destroy]
6
-
7
- def index
8
- render json: Criteria.order(created_at: :desc)
9
- end
10
-
11
- def show
12
- render json: @criteria
13
- end
14
-
15
- def create
16
- criteria = Criteria.new(criteria_params.except(:metric_ids))
17
- if criteria.save
18
- replace_metric_memberships(criteria, params[:metric_ids]) if params.key?(:metric_ids)
19
- render json: criteria.reload, status: :created
20
- else
21
- render json: {errors: criteria.errors}, status: :unprocessable_entity
22
- end
23
- end
24
-
25
- def update
26
- if @criteria.update(criteria_params.except(:metric_ids))
27
- replace_metric_memberships(@criteria, params[:metric_ids]) if params.key?(:metric_ids)
28
- render json: @criteria.reload
29
- else
30
- render json: {errors: @criteria.errors}, status: :unprocessable_entity
31
- end
32
- end
33
-
34
- def destroy
35
- @criteria.destroy!
36
- head :no_content
37
- end
38
-
39
- private
40
-
41
- def set_criteria
42
- @criteria = Criteria.find(params[:id])
43
- rescue ActiveRecord::RecordNotFound
44
- not_found
45
- end
46
-
47
- def criteria_params
48
- params.permit(:name, :description, metric_ids: [])
49
- end
50
-
51
- def replace_metric_memberships(criteria, metric_ids)
52
- return unless metric_ids
53
-
54
- criteria.criteria_memberships.delete_all
55
- Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
56
- criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
57
- end
58
- end
59
- end
60
- end
61
- end
62
- end
@@ -1,67 +0,0 @@
1
- module CompletionKit
2
- class CriteriaController < ApplicationController
3
- before_action :set_criteria, only: [:show, :edit, :update, :destroy]
4
-
5
- def index
6
- @criterias = Criteria.includes(:metrics).order(:name)
7
- end
8
-
9
- def show
10
- end
11
-
12
- def new
13
- @criteria = Criteria.new
14
- @metrics = Metric.order(:name)
15
- end
16
-
17
- def edit
18
- @metrics = Metric.order(:name)
19
- end
20
-
21
- def create
22
- @criteria = Criteria.new(criteria_params.except(:metric_ids))
23
- @metrics = Metric.order(:name)
24
-
25
- if @criteria.save
26
- replace_metric_memberships
27
- redirect_to criterion_path(@criteria), notice: "Criteria was successfully created."
28
- else
29
- render :new, status: :unprocessable_entity
30
- end
31
- end
32
-
33
- def update
34
- @metrics = Metric.order(:name)
35
-
36
- if @criteria.update(criteria_params.except(:metric_ids))
37
- replace_metric_memberships
38
- redirect_to criterion_path(@criteria), notice: "Criteria was successfully updated."
39
- else
40
- render :edit, status: :unprocessable_entity
41
- end
42
- end
43
-
44
- def destroy
45
- @criteria.destroy
46
- redirect_to criteria_path, notice: "Criteria was successfully destroyed."
47
- end
48
-
49
- private
50
-
51
- def set_criteria
52
- @criteria = Criteria.find(params[:id])
53
- end
54
-
55
- def criteria_params
56
- params.require(:criteria).permit(:name, :description, metric_ids: [])
57
- end
58
-
59
- def replace_metric_memberships
60
- metric_ids = Array(criteria_params[:metric_ids]).reject(&:blank?)
61
- @criteria.criteria_memberships.delete_all
62
- metric_ids.each_with_index do |metric_id, index|
63
- @criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
64
- end
65
- end
66
- end
67
- end
@@ -1,22 +0,0 @@
1
- module CompletionKit
2
- class Criteria < ApplicationRecord
3
- self.table_name = "completion_kit_criteria"
4
-
5
- has_many :criteria_memberships, -> { order(:position, :id) }, dependent: :destroy
6
- has_many :metrics, through: :criteria_memberships
7
-
8
- validates :name, presence: true
9
-
10
- def ordered_metrics
11
- criteria_memberships.includes(:metric).map(&:metric).compact
12
- end
13
-
14
- def as_json(options = {})
15
- {
16
- id: id, name: name, description: description,
17
- created_at: created_at, updated_at: updated_at,
18
- metric_ids: metric_ids
19
- }
20
- end
21
- end
22
- end
@@ -1,20 +0,0 @@
1
- module CompletionKit
2
- class CriteriaMembership < ApplicationRecord
3
- self.table_name = "completion_kit_criteria_memberships"
4
-
5
- belongs_to :criteria, class_name: "CompletionKit::Criteria", foreign_key: "criteria_id"
6
- belongs_to :metric
7
-
8
- validates :metric_id, uniqueness: { scope: :criteria_id }
9
-
10
- before_validation :set_default_position
11
-
12
- private
13
-
14
- def set_default_position
15
- return if position.present? || criteria.blank?
16
-
17
- self.position = criteria.criteria_memberships.maximum(:position).to_i + 1
18
- end
19
- end
20
- end
@@ -1,106 +0,0 @@
1
- module CompletionKit
2
- module McpTools
3
- module Criteria
4
- TOOLS = {
5
- "criteria_list" => {
6
- description: "List all criteria",
7
- inputSchema: {type: "object", properties: {}, required: []},
8
- handler: :list
9
- },
10
- "criteria_get" => {
11
- description: "Get a criteria by ID",
12
- inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
13
- handler: :get
14
- },
15
- "criteria_create" => {
16
- description: "Create a criteria grouping metrics",
17
- inputSchema: {
18
- type: "object",
19
- properties: {
20
- name: {type: "string"}, description: {type: "string"},
21
- metric_ids: {type: "array", items: {type: "integer"}}
22
- },
23
- required: ["name"]
24
- },
25
- handler: :create
26
- },
27
- "criteria_update" => {
28
- description: "Update a criteria",
29
- inputSchema: {
30
- type: "object",
31
- properties: {
32
- id: {type: "integer"}, name: {type: "string"}, description: {type: "string"},
33
- metric_ids: {type: "array", items: {type: "integer"}}
34
- },
35
- required: ["id"]
36
- },
37
- handler: :update
38
- },
39
- "criteria_delete" => {
40
- description: "Delete a criteria",
41
- inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
42
- handler: :delete
43
- }
44
- }.freeze
45
-
46
- def self.definitions
47
- TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
48
- end
49
-
50
- def self.call(name, arguments)
51
- tool = TOOLS.fetch(name)
52
- send(tool[:handler], arguments)
53
- end
54
-
55
- def self.list(_args)
56
- text_result(CompletionKit::Criteria.order(created_at: :desc).map(&:as_json))
57
- end
58
-
59
- def self.get(args)
60
- text_result(CompletionKit::Criteria.find(args["id"]).as_json)
61
- end
62
-
63
- def self.create(args)
64
- criteria = CompletionKit::Criteria.new(args.slice("name", "description"))
65
- if criteria.save
66
- replace_metric_memberships(criteria, args["metric_ids"])
67
- text_result(criteria.reload.as_json)
68
- else
69
- error_result(criteria.errors.full_messages.join(", "))
70
- end
71
- end
72
-
73
- def self.update(args)
74
- criteria = CompletionKit::Criteria.find(args["id"])
75
- if criteria.update(args.except("id", "metric_ids").slice("name", "description"))
76
- replace_metric_memberships(criteria, args["metric_ids"]) if args.key?("metric_ids")
77
- text_result(criteria.reload.as_json)
78
- else
79
- error_result(criteria.errors.full_messages.join(", "))
80
- end
81
- end
82
-
83
- def self.delete(args)
84
- CompletionKit::Criteria.find(args["id"]).destroy!
85
- text_result("Criteria #{args["id"]} deleted")
86
- end
87
-
88
- def self.text_result(data)
89
- text = data.is_a?(String) ? data : data.to_json
90
- {content: [{type: "text", text: text}]}
91
- end
92
-
93
- def self.error_result(message)
94
- {content: [{type: "text", text: message}], isError: true}
95
- end
96
-
97
- def self.replace_metric_memberships(criteria, metric_ids)
98
- return unless metric_ids
99
- criteria.criteria_memberships.delete_all
100
- Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
101
- criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
102
- end
103
- end
104
- end
105
- end
106
- end
@@ -1,46 +0,0 @@
1
- <%= form_with(model: criteria, url: criteria.persisted? ? criterion_path(criteria) : criteria_path, local: true) do |form| %>
2
- <% if criteria.errors.any? %>
3
- <div class="ck-flash ck-flash--alert">
4
- <p class="ck-flash__title"><%= pluralize(criteria.errors.count, "problem") %> prevented this criteria from being saved.</p>
5
- <ul class="ck-error-list">
6
- <% criteria.errors.full_messages.each do |message| %>
7
- <li><%= message %></li>
8
- <% end %>
9
- </ul>
10
- </div>
11
- <% end %>
12
-
13
- <div class="ck-card ck-form-card">
14
- <div class="ck-field">
15
- <%= form.label :name, "Criteria name", class: "ck-label" %>
16
- <%= form.text_field :name, class: "ck-input", placeholder: "Support quality" %>
17
- </div>
18
-
19
- <div class="ck-field">
20
- <%= form.label :description, class: "ck-label" %>
21
- <%= form.text_area :description, rows: 3, class: "ck-input ck-input--area", placeholder: "When this criteria should be used." %>
22
- </div>
23
-
24
- <div class="ck-field">
25
- <p class="ck-label">Metrics in this criteria</p>
26
- <p class="ck-hint">A criteria groups metrics together for judging.</p>
27
- <div class="ck-list ck-list--compact">
28
- <% @metrics.each do |metric| %>
29
- <label class="ck-item">
30
- <span>
31
- <strong><%= metric.name %></strong>
32
- <span class="ck-meta-copy"><%= metric.instruction.presence || "No instruction set." %></span>
33
- </span>
34
- <%= check_box_tag "criteria[metric_ids][]", metric.id, criteria.metrics.exists?(metric.id), class: "ck-checkbox" %>
35
- </label>
36
- <% end %>
37
- </div>
38
- <%= hidden_field_tag "criteria[metric_ids][]", "" %>
39
- </div>
40
-
41
- <div class="ck-actions">
42
- <%= link_to "Cancel", criteria_path, class: ck_button_classes(:light, variant: :outline) %>
43
- <%= form.submit(criteria.persisted? ? "Save criteria" : "Create criteria", class: ck_button_classes(:dark)) %>
44
- </div>
45
- </div>
46
- <% end %>
@@ -1,14 +0,0 @@
1
- <ol class="ck-breadcrumb">
2
- <li><%= link_to "Metrics", metrics_path %></li>
3
- <li><%= link_to "Criteria", criteria_path %></li>
4
- <li><%= link_to @criteria.name, criterion_path(@criteria) %></li>
5
- <li>Edit</li>
6
- </ol>
7
-
8
- <section class="ck-page-header">
9
- <div>
10
- <h1 class="ck-title">Edit criteria</h1>
11
- </div>
12
- </section>
13
-
14
- <%= render "form", criteria: @criteria %>
@@ -1,37 +0,0 @@
1
- <ol class="ck-breadcrumb">
2
- <li><%= link_to "Metrics", metrics_path %></li>
3
- <li>Criteria</li>
4
- </ol>
5
-
6
- <section class="ck-page-header">
7
- <div>
8
- <h1 class="ck-title">Criteria</h1>
9
- <p class="ck-lead">Named groups of metrics that can be applied to a run as a set. Use criteria to bundle related metrics together for reuse across runs.</p>
10
- </div>
11
- <div class="ck-actions">
12
- <%= link_to "New criteria", new_criterion_path, class: ck_button_classes(:dark) %>
13
- </div>
14
- </section>
15
-
16
- <% if @criterias.any? %>
17
- <table class="ck-results-table">
18
- <thead>
19
- <tr>
20
- <th>Name</th>
21
- <th>Metrics</th>
22
- <th></th>
23
- </tr>
24
- </thead>
25
- <tbody>
26
- <% @criterias.each do |criteria| %>
27
- <tr onclick="window.location='<%= criterion_path(criteria) %>'" style="cursor: pointer;">
28
- <td><strong><%= criteria.name %></strong></td>
29
- <td><%= criteria.metrics.size %></td>
30
- <td class="ck-results-table__arrow">&rarr;</td>
31
- </tr>
32
- <% end %>
33
- </tbody>
34
- </table>
35
- <% else %>
36
- <div class="ck-empty">No criteria yet.</div>
37
- <% end %>
@@ -1,13 +0,0 @@
1
- <ol class="ck-breadcrumb">
2
- <li><%= link_to "Metrics", metrics_path %></li>
3
- <li><%= link_to "Criteria", criteria_path %></li>
4
- <li>New</li>
5
- </ol>
6
-
7
- <section class="ck-page-header">
8
- <div>
9
- <h1 class="ck-title">New criteria</h1>
10
- </div>
11
- </section>
12
-
13
- <%= render "form", criteria: @criteria %>