completion-kit 0.29.1 → 0.29.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f314f2989362808e829e3f1bbbfc5718eca193aa4602ae4b4b3e72bc6c9251c1
4
- data.tar.gz: b9322db963ff8bb00660e09309940f692cab0fff289c8069b3be68ecf81db429
3
+ metadata.gz: 12931f9b21433398c80f8676655a8dcaf075638dfd2015fe30e94f4f9b318bb5
4
+ data.tar.gz: 15869d3565bef4b44d618f0bb3e04649201cb488505160f0fc7a1170eee348df
5
5
  SHA512:
6
- metadata.gz: bff1f44883f8a652ffde54da844bd626db029e432e91f5f79648b926b34ada993599b0a04176bfc842bd1c7da786dc9a05af5744779ba6e3fdca677e213639d1
7
- data.tar.gz: bff0d10f9a3f7ccc857d52b5038ac06e4ccbc6e15e73195ff18c95a2f0223c4d223bdd17749f36d6933d7bb5925e3dd9e522cf4e012b26ac76a9c9b78c40a0c6
6
+ metadata.gz: 2aa2cf27bdde25b06fc7b30294623ffafee6f0f1a7cd2b4a9f4990407743638c07e5f79459f813516c452fbd6d5f2aa12d0c638aa702865db76df69f7435aa93
7
+ data.tar.gz: 8ea6c48e88708cc574d8d43c2099ef332aba04eb94ba75fc0c3b50a283d17d0ac3a40e3e5b292c8a3dc4bd9be7c2b18526133dc2b000f3330880b9483026621b
@@ -4165,6 +4165,35 @@ a.ck-metric-group-pill {
4165
4165
  .ck-responses-table .ck-metric-bar--compact { flex-wrap: wrap; row-gap: 3px; }
4166
4166
 
4167
4167
 
4168
+ .ck-compare-picker td:first-child {
4169
+ overflow: hidden;
4170
+ text-overflow: ellipsis;
4171
+ }
4172
+
4173
+ .ck-compare-picker th:nth-child(2),
4174
+ .ck-compare-picker td:nth-child(2) {
4175
+ width: 9rem;
4176
+ white-space: nowrap;
4177
+ }
4178
+
4179
+ .ck-compare-picker th:nth-child(3),
4180
+ .ck-compare-picker td:nth-child(3) {
4181
+ width: 13rem;
4182
+ overflow: hidden;
4183
+ text-overflow: ellipsis;
4184
+ white-space: nowrap;
4185
+ }
4186
+
4187
+ .ck-compare-picker th:nth-child(4),
4188
+ .ck-compare-picker td:nth-child(4) {
4189
+ width: 9rem;
4190
+ }
4191
+
4192
+ .ck-compare-picker th:last-child,
4193
+ .ck-compare-picker td:last-child {
4194
+ width: 8rem;
4195
+ }
4196
+
4168
4197
  table.ck-runs-table {
4169
4198
  table-layout: fixed;
4170
4199
  width: 100%;
@@ -45,10 +45,10 @@ module CompletionKit
45
45
 
46
46
  def paginate(scope)
47
47
  total = scope.count
48
- limit = (params[:limit].presence || PAGINATION_DEFAULT_LIMIT).to_i
48
+ limit = params[:limit].to_s.to_i
49
49
  limit = PAGINATION_DEFAULT_LIMIT if limit <= 0
50
50
  limit = PAGINATION_MAX_LIMIT if limit > PAGINATION_MAX_LIMIT
51
- offset = params[:offset].to_i
51
+ offset = params[:offset].to_s.to_i
52
52
  offset = 0 if offset < 0
53
53
  response.set_header("X-Total-Count", total.to_s)
54
54
  response.set_header("X-Limit", limit.to_s)
@@ -48,7 +48,7 @@ module CompletionKit
48
48
  return
49
49
  end
50
50
 
51
- generator = MetricVariantGenerator.new(@metric, count: params[:count].to_i, model: params[:model])
51
+ generator = MetricVariantGenerator.new(@metric, count: params[:count].to_s.to_i, model: params[:model])
52
52
  variants = generator.call
53
53
  MetricVersion.drafts.where(metric_id: @metric.id, source: "suggestion").destroy_all
54
54
  if variants.empty?
@@ -25,12 +25,15 @@ module CompletionKit
25
25
  end
26
26
 
27
27
  def update
28
- if @prompt.runs.exists?
28
+ publish = publish_flag
29
+ return render_error("publish must be true or false", status: :unprocessable_entity) if publish.nil?
30
+
31
+ if @prompt.runs.exists? || (!publish && @prompt.current? && @prompt.behavioral_change?(prompt_params))
29
32
  new_prompt = @prompt.build_next_version(prompt_params.to_h)
30
33
  if new_prompt.valid?
31
34
  CompletionKit::ApplicationRecord.transaction do
32
35
  new_prompt.save!
33
- new_prompt.publish!
36
+ new_prompt.publish! if publish
34
37
  end
35
38
  render json: new_prompt.reload
36
39
  else
@@ -68,6 +71,15 @@ module CompletionKit
68
71
  def prompt_params
69
72
  params.permit(:name, :description, :template, :llm_model, tag_names: [])
70
73
  end
74
+
75
+ PUBLISH_VALUES = {"true" => true, "1" => true, "false" => false, "0" => false}.freeze
76
+
77
+ def publish_flag
78
+ value = params[:publish]
79
+ return true if value.blank? && value != false
80
+
81
+ PUBLISH_VALUES[value.to_s]
82
+ end
71
83
  end
72
84
  end
73
85
  end
@@ -65,13 +65,8 @@ module CompletionKit
65
65
 
66
66
  private
67
67
 
68
- BEHAVIORAL_ATTRS = %w[template llm_model].freeze
69
-
70
68
  def behavioral_change?
71
- permitted = prompt_params
72
- BEHAVIORAL_ATTRS.any? do |attr|
73
- permitted.key?(attr) && permitted[attr].to_s != @prompt.public_send(attr).to_s
74
- end
69
+ @prompt.behavioral_change?(prompt_params)
75
70
  end
76
71
 
77
72
  def set_prompt
@@ -88,9 +88,10 @@ module CompletionKit
88
88
  def compare
89
89
  other_id = params[:with]
90
90
  if other_id.blank?
91
- @other_runs = Run.where(dataset_id: @run.dataset_id, prompt_id: @run.prompt_id)
91
+ @other_runs = Run.where(dataset_id: @run.dataset_id, prompt_id: comparable_prompt_ids)
92
92
  .where.not(id: @run.id)
93
93
  .display_scoped
94
+ .includes(:prompt)
94
95
  .order(created_at: :desc)
95
96
  .limit(50)
96
97
  return render(:compare_picker)
@@ -171,6 +172,12 @@ module CompletionKit
171
172
 
172
173
  private
173
174
 
175
+ def comparable_prompt_ids
176
+ return nil unless @run.prompt
177
+
178
+ @run.prompt.family_versions.select(:id)
179
+ end
180
+
174
181
  def set_run
175
182
  @run = Run.find(params[:id])
176
183
  end
@@ -56,6 +56,13 @@ module CompletionKit
56
56
  self.class.where(family_key: family_key).order(version_number: :desc, created_at: :desc)
57
57
  end
58
58
 
59
+ BEHAVIORAL_ATTRS = %w[template llm_model].freeze
60
+
61
+ def behavioral_change?(attrs)
62
+ attrs = attrs.to_h.stringify_keys
63
+ BEHAVIORAL_ATTRS.any? { |attr| attrs.key?(attr) && attrs[attr].to_s != public_send(attr).to_s }
64
+ end
65
+
59
66
  def build_next_version(overrides = {})
60
67
  version = self.class.new(
61
68
  {
@@ -28,7 +28,7 @@ module CompletionKit
28
28
  handler: :create
29
29
  },
30
30
  "prompts_update" => {
31
- description: "Update a prompt. If the prompt already has runs, this creates a new DRAFT version (current=false) rather than editing in place or publishing. Promote it with prompts_publish, so an agent's edits don't go live without a gate. If it has no runs, it is updated in place.",
31
+ description: "Update a prompt. Changing the template or model of the published version, or editing any version that already has runs, creates a new DRAFT version (current=false) instead of changing it, so an agent's edits don't go live without a gate. Run and compare the draft, then promote it with prompts_publish. An unpublished draft with no runs, or a name, description or tag change on a published version with no runs, is updated in place.",
32
32
  inputSchema: {
33
33
  type: "object",
34
34
  properties: {
@@ -82,8 +82,10 @@ module CompletionKit
82
82
  def self.update(args)
83
83
  prompt = Prompt.find(args["id"])
84
84
  attrs = args.except("id").slice("name", "description", "template", "llm_model")
85
- if prompt.runs.exists?
86
- new_prompt = prompt.clone_as_new_version(attrs)
85
+ if prompt.runs.exists? || (prompt.current? && prompt.behavioral_change?(attrs))
86
+ new_prompt = prompt.build_next_version(attrs)
87
+ return error_result(new_prompt.errors.full_messages.join(", ")) unless new_prompt.save
88
+
87
89
  new_prompt.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
88
90
  text_result(new_prompt.reload.as_json)
89
91
  elsif prompt.update(attrs)
@@ -103,7 +103,7 @@
103
103
  } %>
104
104
  <div class="ck-api-endpoint">
105
105
  <p class="ck-api-method"><span class="ck-chip ck-chip--soft">PATCH</span> /api/v1/prompts/:id</p>
106
- <p class="ck-meta-copy">Update a prompt. Accepts same params as create. If the prompt has no runs it is updated in place; if it already has runs, a new published version is created and returned (with a new id), leaving the original unchanged.</p>
106
+ <p class="ck-meta-copy">Update a prompt. Accepts same params as create. If the prompt has no runs it is updated in place; if it already has runs, a new version is created and returned (with a new id), leaving the original unchanged. The new version is published unless you send <code>"publish": false</code>, which saves it as an unpublished draft you can run and compare first, then publish with <code>POST /api/v1/prompts/:id/publish</code>. With <code>"publish": false</code>, a template or model change to the published version is saved as a draft even when it has no runs, so what your app fetches doesn't change. <code>publish</code> must be <code>true</code> or <code>false</code>; any other value returns 422.</p>
107
107
  <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X PATCH #{base_url}/api/v1/prompts/1 \\\n -H \"Authorization: Bearer #{token}\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"template\": \"Summarize concisely: {{text}}\"}'" %>
108
108
  </div>
109
109
  <div class="ck-api-endpoint">
@@ -126,7 +126,7 @@
126
126
  <% pred = predecessor_of[v] %>
127
127
  <% next unless v.change_summary_against(pred) %>
128
128
  <% vs = v.validation_summary %>
129
- <dialog id="ck-mvdiff-<%= v.id %>" class="ck-modal" aria-labelledby="ck-mvdiff-title-<%= v.id %>" data-ck-dialog<%= " data-ck-dialog-autoopen" if params[:show_change].to_i == v.id %>>
129
+ <dialog id="ck-mvdiff-<%= v.id %>" class="ck-modal" aria-labelledby="ck-mvdiff-title-<%= v.id %>" data-ck-dialog<%= " data-ck-dialog-autoopen" if params[:show_change].to_s.to_i == v.id %>>
130
130
  <article class="ck-modal__panel" tabindex="-1">
131
131
  <header class="ck-modal__header">
132
132
  <div class="ck-modal__heading">
@@ -8,15 +8,16 @@
8
8
  <section class="ck-page-header">
9
9
  <div>
10
10
  <h1 class="ck-title">Compare with another run</h1>
11
- <p class="ck-lead">Pick a run on the same dataset and prompt to see per-case score deltas side by side.</p>
11
+ <p class="ck-lead">Pick a run on the same dataset and prompt, from any version of the prompt, to see per-case score deltas side by side.</p>
12
12
  </div>
13
13
  </section>
14
14
 
15
15
  <% if @other_runs.any? %>
16
- <table class="ck-results-table">
16
+ <table class="ck-results-table ck-compare-picker">
17
17
  <thead>
18
18
  <tr>
19
19
  <th scope="col">Run</th>
20
+ <th scope="col">Prompt version</th>
20
21
  <th scope="col">Judge</th>
21
22
  <th scope="col">Created</th>
22
23
  <th scope="col"></th>
@@ -25,7 +26,8 @@
25
26
  <tbody>
26
27
  <% @other_runs.each do |other| %>
27
28
  <tr>
28
- <td><%= link_to other.name, run_path(other), class: "ck-link" %></td>
29
+ <td><%= link_to other.name, run_path(other), class: "ck-link", title: other.name %></td>
30
+ <td class="ck-meta-copy"><%= other.prompt&.version_label %></td>
29
31
  <td class="ck-meta-copy"><%= other.judge_model %></td>
30
32
  <td class="ck-meta-copy"><time datetime="<%= other.created_at.utc.iso8601 %>"><%= time_ago_in_words(other.created_at) %> ago</time></td>
31
33
  <td class="ck-results-table__arrow"><%= link_to "Compare →", compare_run_path(@run, with: other.id), class: "ck-link" %></td>
@@ -1,3 +1,3 @@
1
1
  module CompletionKit
2
- VERSION = "0.29.1"
2
+ VERSION = "0.29.3"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: completion-kit
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.29.1
4
+ version: 0.29.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Damien Bastin