completion-kit 0.29.2 → 0.29.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/stylesheets/completion_kit/application.css +29 -0
- data/app/controllers/completion_kit/api/v1/prompts_controller.rb +14 -2
- data/app/controllers/completion_kit/prompts_controller.rb +1 -6
- data/app/controllers/completion_kit/runs_controller.rb +8 -1
- data/app/models/completion_kit/prompt.rb +7 -0
- data/app/services/completion_kit/mcp_tools/prompts.rb +5 -3
- data/app/views/completion_kit/api_reference/_body.html.erb +1 -1
- data/app/views/completion_kit/runs/compare_picker.html.erb +5 -3
- data/lib/completion_kit/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 12931f9b21433398c80f8676655a8dcaf075638dfd2015fe30e94f4f9b318bb5
|
|
4
|
+
data.tar.gz: 15869d3565bef4b44d618f0bb3e04649201cb488505160f0fc7a1170eee348df
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2aa2cf27bdde25b06fc7b30294623ffafee6f0f1a7cd2b4a9f4990407743638c07e5f79459f813516c452fbd6d5f2aa12d0c638aa702865db76df69f7435aa93
|
|
7
|
+
data.tar.gz: 8ea6c48e88708cc574d8d43c2099ef332aba04eb94ba75fc0c3b50a283d17d0ac3a40e3e5b292c8a3dc4bd9be7c2b18526133dc2b000f3330880b9483026621b
|
|
@@ -4165,6 +4165,35 @@ a.ck-metric-group-pill {
|
|
|
4165
4165
|
.ck-responses-table .ck-metric-bar--compact { flex-wrap: wrap; row-gap: 3px; }
|
|
4166
4166
|
|
|
4167
4167
|
|
|
4168
|
+
.ck-compare-picker td:first-child {
|
|
4169
|
+
overflow: hidden;
|
|
4170
|
+
text-overflow: ellipsis;
|
|
4171
|
+
}
|
|
4172
|
+
|
|
4173
|
+
.ck-compare-picker th:nth-child(2),
|
|
4174
|
+
.ck-compare-picker td:nth-child(2) {
|
|
4175
|
+
width: 9rem;
|
|
4176
|
+
white-space: nowrap;
|
|
4177
|
+
}
|
|
4178
|
+
|
|
4179
|
+
.ck-compare-picker th:nth-child(3),
|
|
4180
|
+
.ck-compare-picker td:nth-child(3) {
|
|
4181
|
+
width: 13rem;
|
|
4182
|
+
overflow: hidden;
|
|
4183
|
+
text-overflow: ellipsis;
|
|
4184
|
+
white-space: nowrap;
|
|
4185
|
+
}
|
|
4186
|
+
|
|
4187
|
+
.ck-compare-picker th:nth-child(4),
|
|
4188
|
+
.ck-compare-picker td:nth-child(4) {
|
|
4189
|
+
width: 9rem;
|
|
4190
|
+
}
|
|
4191
|
+
|
|
4192
|
+
.ck-compare-picker th:last-child,
|
|
4193
|
+
.ck-compare-picker td:last-child {
|
|
4194
|
+
width: 8rem;
|
|
4195
|
+
}
|
|
4196
|
+
|
|
4168
4197
|
table.ck-runs-table {
|
|
4169
4198
|
table-layout: fixed;
|
|
4170
4199
|
width: 100%;
|
|
@@ -25,12 +25,15 @@ module CompletionKit
|
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
def update
|
|
28
|
-
|
|
28
|
+
publish = publish_flag
|
|
29
|
+
return render_error("publish must be true or false", status: :unprocessable_entity) if publish.nil?
|
|
30
|
+
|
|
31
|
+
if @prompt.runs.exists? || (!publish && @prompt.current? && @prompt.behavioral_change?(prompt_params))
|
|
29
32
|
new_prompt = @prompt.build_next_version(prompt_params.to_h)
|
|
30
33
|
if new_prompt.valid?
|
|
31
34
|
CompletionKit::ApplicationRecord.transaction do
|
|
32
35
|
new_prompt.save!
|
|
33
|
-
new_prompt.publish!
|
|
36
|
+
new_prompt.publish! if publish
|
|
34
37
|
end
|
|
35
38
|
render json: new_prompt.reload
|
|
36
39
|
else
|
|
@@ -68,6 +71,15 @@ module CompletionKit
|
|
|
68
71
|
def prompt_params
|
|
69
72
|
params.permit(:name, :description, :template, :llm_model, tag_names: [])
|
|
70
73
|
end
|
|
74
|
+
|
|
75
|
+
PUBLISH_VALUES = {"true" => true, "1" => true, "false" => false, "0" => false}.freeze
|
|
76
|
+
|
|
77
|
+
def publish_flag
|
|
78
|
+
value = params[:publish]
|
|
79
|
+
return true if value.blank? && value != false
|
|
80
|
+
|
|
81
|
+
PUBLISH_VALUES[value.to_s]
|
|
82
|
+
end
|
|
71
83
|
end
|
|
72
84
|
end
|
|
73
85
|
end
|
|
@@ -65,13 +65,8 @@ module CompletionKit
|
|
|
65
65
|
|
|
66
66
|
private
|
|
67
67
|
|
|
68
|
-
BEHAVIORAL_ATTRS = %w[template llm_model].freeze
|
|
69
|
-
|
|
70
68
|
def behavioral_change?
|
|
71
|
-
|
|
72
|
-
BEHAVIORAL_ATTRS.any? do |attr|
|
|
73
|
-
permitted.key?(attr) && permitted[attr].to_s != @prompt.public_send(attr).to_s
|
|
74
|
-
end
|
|
69
|
+
@prompt.behavioral_change?(prompt_params)
|
|
75
70
|
end
|
|
76
71
|
|
|
77
72
|
def set_prompt
|
|
@@ -88,9 +88,10 @@ module CompletionKit
|
|
|
88
88
|
def compare
|
|
89
89
|
other_id = params[:with]
|
|
90
90
|
if other_id.blank?
|
|
91
|
-
@other_runs = Run.where(dataset_id: @run.dataset_id, prompt_id:
|
|
91
|
+
@other_runs = Run.where(dataset_id: @run.dataset_id, prompt_id: comparable_prompt_ids)
|
|
92
92
|
.where.not(id: @run.id)
|
|
93
93
|
.display_scoped
|
|
94
|
+
.includes(:prompt)
|
|
94
95
|
.order(created_at: :desc)
|
|
95
96
|
.limit(50)
|
|
96
97
|
return render(:compare_picker)
|
|
@@ -171,6 +172,12 @@ module CompletionKit
|
|
|
171
172
|
|
|
172
173
|
private
|
|
173
174
|
|
|
175
|
+
def comparable_prompt_ids
|
|
176
|
+
return nil unless @run.prompt
|
|
177
|
+
|
|
178
|
+
@run.prompt.family_versions.select(:id)
|
|
179
|
+
end
|
|
180
|
+
|
|
174
181
|
def set_run
|
|
175
182
|
@run = Run.find(params[:id])
|
|
176
183
|
end
|
|
@@ -56,6 +56,13 @@ module CompletionKit
|
|
|
56
56
|
self.class.where(family_key: family_key).order(version_number: :desc, created_at: :desc)
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
+
BEHAVIORAL_ATTRS = %w[template llm_model].freeze
|
|
60
|
+
|
|
61
|
+
def behavioral_change?(attrs)
|
|
62
|
+
attrs = attrs.to_h.stringify_keys
|
|
63
|
+
BEHAVIORAL_ATTRS.any? { |attr| attrs.key?(attr) && attrs[attr].to_s != public_send(attr).to_s }
|
|
64
|
+
end
|
|
65
|
+
|
|
59
66
|
def build_next_version(overrides = {})
|
|
60
67
|
version = self.class.new(
|
|
61
68
|
{
|
|
@@ -28,7 +28,7 @@ module CompletionKit
|
|
|
28
28
|
handler: :create
|
|
29
29
|
},
|
|
30
30
|
"prompts_update" => {
|
|
31
|
-
description: "Update a prompt.
|
|
31
|
+
description: "Update a prompt. Changing the template or model of the published version, or editing any version that already has runs, creates a new DRAFT version (current=false) instead of changing it, so an agent's edits don't go live without a gate. Run and compare the draft, then promote it with prompts_publish. An unpublished draft with no runs, or a name, description or tag change on a published version with no runs, is updated in place.",
|
|
32
32
|
inputSchema: {
|
|
33
33
|
type: "object",
|
|
34
34
|
properties: {
|
|
@@ -82,8 +82,10 @@ module CompletionKit
|
|
|
82
82
|
def self.update(args)
|
|
83
83
|
prompt = Prompt.find(args["id"])
|
|
84
84
|
attrs = args.except("id").slice("name", "description", "template", "llm_model")
|
|
85
|
-
if prompt.runs.exists?
|
|
86
|
-
new_prompt = prompt.
|
|
85
|
+
if prompt.runs.exists? || (prompt.current? && prompt.behavioral_change?(attrs))
|
|
86
|
+
new_prompt = prompt.build_next_version(attrs)
|
|
87
|
+
return error_result(new_prompt.errors.full_messages.join(", ")) unless new_prompt.save
|
|
88
|
+
|
|
87
89
|
new_prompt.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
|
|
88
90
|
text_result(new_prompt.reload.as_json)
|
|
89
91
|
elsif prompt.update(attrs)
|
|
@@ -103,7 +103,7 @@
|
|
|
103
103
|
} %>
|
|
104
104
|
<div class="ck-api-endpoint">
|
|
105
105
|
<p class="ck-api-method"><span class="ck-chip ck-chip--soft">PATCH</span> /api/v1/prompts/:id</p>
|
|
106
|
-
<p class="ck-meta-copy">Update a prompt. Accepts same params as create. If the prompt has no runs it is updated in place; if it already has runs, a new
|
|
106
|
+
<p class="ck-meta-copy">Update a prompt. Accepts same params as create. If the prompt has no runs it is updated in place; if it already has runs, a new version is created and returned (with a new id), leaving the original unchanged. The new version is published unless you send <code>"publish": false</code>, which saves it as an unpublished draft you can run and compare first, then publish with <code>POST /api/v1/prompts/:id/publish</code>. With <code>"publish": false</code>, a template or model change to the published version is saved as a draft even when it has no runs, so what your app fetches doesn't change. <code>publish</code> must be <code>true</code> or <code>false</code>; any other value returns 422.</p>
|
|
107
107
|
<%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X PATCH #{base_url}/api/v1/prompts/1 \\\n -H \"Authorization: Bearer #{token}\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"template\": \"Summarize concisely: {{text}}\"}'" %>
|
|
108
108
|
</div>
|
|
109
109
|
<div class="ck-api-endpoint">
|
|
@@ -8,15 +8,16 @@
|
|
|
8
8
|
<section class="ck-page-header">
|
|
9
9
|
<div>
|
|
10
10
|
<h1 class="ck-title">Compare with another run</h1>
|
|
11
|
-
<p class="ck-lead">Pick a run on the same dataset and prompt to see per-case score deltas side by side.</p>
|
|
11
|
+
<p class="ck-lead">Pick a run on the same dataset and prompt, from any version of the prompt, to see per-case score deltas side by side.</p>
|
|
12
12
|
</div>
|
|
13
13
|
</section>
|
|
14
14
|
|
|
15
15
|
<% if @other_runs.any? %>
|
|
16
|
-
<table class="ck-results-table">
|
|
16
|
+
<table class="ck-results-table ck-compare-picker">
|
|
17
17
|
<thead>
|
|
18
18
|
<tr>
|
|
19
19
|
<th scope="col">Run</th>
|
|
20
|
+
<th scope="col">Prompt version</th>
|
|
20
21
|
<th scope="col">Judge</th>
|
|
21
22
|
<th scope="col">Created</th>
|
|
22
23
|
<th scope="col"></th>
|
|
@@ -25,7 +26,8 @@
|
|
|
25
26
|
<tbody>
|
|
26
27
|
<% @other_runs.each do |other| %>
|
|
27
28
|
<tr>
|
|
28
|
-
<td><%= link_to other.name, run_path(other), class: "ck-link" %></td>
|
|
29
|
+
<td><%= link_to other.name, run_path(other), class: "ck-link", title: other.name %></td>
|
|
30
|
+
<td class="ck-meta-copy"><%= other.prompt&.version_label %></td>
|
|
29
31
|
<td class="ck-meta-copy"><%= other.judge_model %></td>
|
|
30
32
|
<td class="ck-meta-copy"><time datetime="<%= other.created_at.utc.iso8601 %>"><%= time_ago_in_words(other.created_at) %> ago</time></td>
|
|
31
33
|
<td class="ck-results-table__arrow"><%= link_to "Compare →", compare_run_path(@run, with: other.id), class: "ck-link" %></td>
|