completion-kit 0.5.42 → 0.5.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/completion_kit/application.js +17 -0
- data/app/assets/stylesheets/completion_kit/application.css +505 -39
- data/app/controllers/completion_kit/metrics_controller.rb +35 -24
- data/app/jobs/completion_kit/judge_review_job.rb +11 -0
- data/app/models/completion_kit/judge_version.rb +32 -1
- data/app/services/completion_kit/judge_variant_generator.rb +8 -6
- data/app/services/completion_kit/metric_calibration_stats.rb +16 -4
- data/app/views/completion_kit/api_reference/_body.html.erb +1 -1
- data/app/views/completion_kit/calibrations/_buttons.html.erb +43 -6
- data/app/views/completion_kit/calibrations/_trust_panel.html.erb +27 -28
- data/app/views/completion_kit/metrics/_form.html.erb +90 -4
- data/app/views/completion_kit/metrics/_rubric_diff.html.erb +25 -0
- data/app/views/completion_kit/metrics/_rubric_hint.html.erb +4 -0
- data/app/views/completion_kit/metrics/_starter_card.html.erb +13 -9
- data/app/views/completion_kit/metrics/edit.html.erb +5 -1
- data/app/views/completion_kit/metrics/index.html.erb +5 -3
- data/app/views/completion_kit/metrics/show.html.erb +132 -126
- data/app/views/completion_kit/metrics/starter_preview.html.erb +6 -6
- data/app/views/completion_kit/responses/show.html.erb +1 -1
- data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
- data/config/routes.rb +2 -1
- data/db/migrate/20260525000001_add_version_number_and_published_at_to_judge_versions.rb +24 -0
- data/lib/completion_kit/version.rb +1 -1
- metadata +4 -1
|
@@ -30,15 +30,17 @@
|
|
|
30
30
|
<%= link_to metric_path(metric), class: "ck-record-name" do %><strong><%= metric.name %></strong><% end %>
|
|
31
31
|
<% if CompletionKit.config.judge_calibration_enabled %>
|
|
32
32
|
<% s = CompletionKit::MetricCalibrationStats.for(metric) %>
|
|
33
|
-
<p class="ck-metrics-table__trust">
|
|
33
|
+
<p class="ck-metrics-table__trust" title="Calibration: how often this metric's scores match the humans who reviewed them.">
|
|
34
|
+
<%= heroicon_tag "adjustments-horizontal", variant: :outline, "aria-hidden": "true", class: "ck-trust-icon" %>
|
|
35
|
+
<span class="ck-metrics-table__trust-label">Calibration</span>
|
|
34
36
|
<% if s.counter_only? %>
|
|
35
37
|
<% if s.sample_size.zero? %>
|
|
36
|
-
|
|
38
|
+
<span class="ck-metrics-table__trust-state">Not measured yet</span>
|
|
37
39
|
<% else %>
|
|
38
40
|
<%= s.sample_size %> / <%= CompletionKit::MetricCalibrationStats::PROVISIONAL_MIN %> verdicts
|
|
39
41
|
<% end %>
|
|
40
42
|
<% else %>
|
|
41
|
-
<span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read
|
|
43
|
+
<span class="ck-metrics-table__trust-rate" title="<%= s.firm? ? 'Settled read.' : 'Early read. Keep giving verdicts.' %>">~<%= (s.agreement_point * 100).round %>%</span>
|
|
42
44
|
±<%= (s.margin * 100).round %> pt · <%= s.firm? ? "settled" : "early" %>
|
|
43
45
|
<% end %>
|
|
44
46
|
</p>
|
|
@@ -8,70 +8,40 @@
|
|
|
8
8
|
<h1 class="ck-title"><%= @metric.name %></h1>
|
|
9
9
|
<% if CompletionKit.config.judge_calibration_enabled %>
|
|
10
10
|
<%= render "completion_kit/calibrations/trust_panel",
|
|
11
|
-
stats: CompletionKit::MetricCalibrationStats.for(@metric)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
<div class="ck-prompt-preview__header">
|
|
19
|
-
<p class="ck-kicker">Draft pending</p>
|
|
20
|
-
<%= button_to "Publish this version", publish_draft_metric_path(@metric, draft_id: @edit_draft.id),
|
|
21
|
-
method: :post, form_class: "inline-block",
|
|
22
|
-
class: ck_button_classes(:dark) %>
|
|
23
|
-
</div>
|
|
24
|
-
<p class="ck-meta-copy">A draft of this metric is saved. Publishing it replaces the live instruction<%= ", rubric," if rubric_changed %> for future runs. Here's what changes.</p>
|
|
25
|
-
|
|
26
|
-
<% if instruction_changed %>
|
|
27
|
-
<div class="ck-suggest-diff">
|
|
28
|
-
<div class="ck-suggest-diff__pane">
|
|
29
|
-
<div class="ck-suggest-diff__header">
|
|
30
|
-
<span class="ck-suggest-diff__label ck-suggest-diff__label--before">Currently live</span>
|
|
31
|
-
</div>
|
|
32
|
-
<pre class="ck-suggest-diff__code"><%= ck_word_diff_old(pub_instr, draft_instr) %></pre>
|
|
33
|
-
</div>
|
|
34
|
-
<div class="ck-suggest-diff__pane">
|
|
35
|
-
<div class="ck-suggest-diff__header">
|
|
36
|
-
<span class="ck-suggest-diff__label ck-suggest-diff__label--after">Draft</span>
|
|
37
|
-
</div>
|
|
38
|
-
<pre class="ck-suggest-diff__code"><%= ck_word_diff_new(pub_instr, draft_instr) %></pre>
|
|
39
|
-
</div>
|
|
40
|
-
</div>
|
|
41
|
-
<% else %>
|
|
42
|
-
<p class="ck-meta-copy">The instruction is unchanged.</p>
|
|
43
|
-
<% end %>
|
|
44
|
-
|
|
45
|
-
<% if rubric_changed %>
|
|
46
|
-
<p class="ck-meta-copy"><strong>Rubric also changed.</strong> Edit the metric to inspect each band, or publish to apply the new wording.</p>
|
|
47
|
-
<% end %>
|
|
48
|
-
</section>
|
|
49
|
-
<% end %>
|
|
11
|
+
stats: CompletionKit::MetricCalibrationStats.for(@metric),
|
|
12
|
+
metric: @metric %>
|
|
13
|
+
<% end %>
|
|
14
|
+
<% if @metric.tags.any? %>
|
|
15
|
+
<div class="tag-marks-row tag-marks-row--header">
|
|
16
|
+
<%= render "completion_kit/tags/marks", tags: @metric.tags %>
|
|
17
|
+
</div>
|
|
50
18
|
<% end %>
|
|
51
19
|
</div>
|
|
52
20
|
<div class="ck-actions">
|
|
53
21
|
<% if CompletionKit.config.judge_calibration_enabled %>
|
|
54
|
-
<% if @
|
|
22
|
+
<% if @suggestion_draft %>
|
|
23
|
+
<%= link_to "Review improvements →", edit_metric_path(@metric),
|
|
24
|
+
class: ck_button_classes(:dark),
|
|
25
|
+
title: "The model proposed improvements based on your disagreements. Review and apply what you want." %>
|
|
26
|
+
<% elsif @edit_draft %>
|
|
27
|
+
<%= link_to "Review draft →", edit_metric_path(@metric),
|
|
28
|
+
class: ck_button_classes(:dark),
|
|
29
|
+
title: "An unpublished draft of this metric is saved. Review and publish, or keep editing." %>
|
|
30
|
+
<% elsif @improve_disagreement_count.positive? %>
|
|
55
31
|
<%= button_to "Improve the metric", suggest_variants_metric_path(@metric),
|
|
56
32
|
method: :post, form_class: "inline-block",
|
|
57
33
|
class: ck_button_classes(:light, variant: :outline),
|
|
58
|
-
title: "
|
|
34
|
+
title: "Have the model rewrite this metric's instruction and rubric based on the disagreements collected so far.",
|
|
59
35
|
data: { turbo_confirm: "Rewrite this metric based on the disagreements collected so far?" } %>
|
|
60
36
|
<% else %>
|
|
61
37
|
<button type="button" class="<%= ck_button_classes(:light, variant: :outline) %>" disabled
|
|
62
|
-
title="Mark at least one
|
|
38
|
+
title="Mark at least one case as Disagree before the model can suggest a change.">Improve the metric</button>
|
|
63
39
|
<% end %>
|
|
64
40
|
<% end %>
|
|
65
41
|
<%= link_to "Edit", edit_metric_path(@metric), class: ck_button_classes(:light, variant: :outline) %>
|
|
66
42
|
</div>
|
|
67
43
|
</section>
|
|
68
44
|
|
|
69
|
-
<% if @metric.tags.any? %>
|
|
70
|
-
<div class="tag-marks-row tag-marks-row--header">
|
|
71
|
-
<%= render "completion_kit/tags/marks", tags: @metric.tags %>
|
|
72
|
-
</div>
|
|
73
|
-
<% end %>
|
|
74
|
-
|
|
75
45
|
<% if @metric.instruction.present? %>
|
|
76
46
|
<section class="ck-card">
|
|
77
47
|
<p class="ck-kicker">Instruction</p>
|
|
@@ -80,7 +50,7 @@
|
|
|
80
50
|
<% end %>
|
|
81
51
|
|
|
82
52
|
<section class="ck-card ck-card--spaced">
|
|
83
|
-
<p class="ck-kicker">Rubric
|
|
53
|
+
<p class="ck-kicker">Rubric<%= render "completion_kit/metrics/rubric_hint" %></p>
|
|
84
54
|
<div class="ck-rubric-display">
|
|
85
55
|
<% @metric.rubric_bands_for_form.each do |band| %>
|
|
86
56
|
<div class="ck-rubric-row ck-rubric-row--display">
|
|
@@ -97,64 +67,121 @@
|
|
|
97
67
|
</div>
|
|
98
68
|
</section>
|
|
99
69
|
|
|
100
|
-
<% if CompletionKit.config.judge_calibration_enabled && @
|
|
101
|
-
<%
|
|
102
|
-
<%
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
70
|
+
<% if CompletionKit.config.judge_calibration_enabled && @versions.size > 1 %>
|
|
71
|
+
<% predecessor_of = @versions.index_with { |v| @versions.detect { |o| o.version_number < v.version_number } } %>
|
|
72
|
+
<% version_changed = ->(v, pred) { pred && (pred.instruction.to_s != v.instruction.to_s || pred.rubric_bands != v.rubric_bands) } %>
|
|
73
|
+
<section class="ck-card ck-card--spaced">
|
|
74
|
+
<p class="ck-kicker">Versions</p>
|
|
75
|
+
<table class="ck-results-table ck-metric-versions-table">
|
|
76
|
+
<thead>
|
|
77
|
+
<tr>
|
|
78
|
+
<th scope="col">Version</th>
|
|
79
|
+
<th scope="col">Source</th>
|
|
80
|
+
<th scope="col">Created</th>
|
|
81
|
+
</tr>
|
|
82
|
+
</thead>
|
|
83
|
+
<tbody>
|
|
84
|
+
<% @versions.each do |v| %>
|
|
85
|
+
<% pred = predecessor_of[v] %>
|
|
86
|
+
<tr>
|
|
87
|
+
<td>
|
|
88
|
+
<div class="ck-version-cell">
|
|
89
|
+
<div class="ck-version-cell__label">
|
|
90
|
+
<strong><%= v.version_label %></strong>
|
|
91
|
+
<% if v.current? %>
|
|
92
|
+
<span class="ck-chip">Published</span>
|
|
93
|
+
<% elsif v.draft? %>
|
|
94
|
+
<%= button_to "Publish", publish_draft_metric_path(@metric, draft_id: v.id),
|
|
95
|
+
method: :post, form_class: "inline-block",
|
|
96
|
+
class: "ck-chip ck-chip--publish" %>
|
|
97
|
+
<% else %>
|
|
98
|
+
<%= button_to "Make current", publish_draft_metric_path(@metric, draft_id: v.id),
|
|
99
|
+
method: :post, form_class: "inline-block",
|
|
100
|
+
class: "ck-chip ck-chip--publish",
|
|
101
|
+
data: { turbo_confirm: "Roll the live judge back to #{v.version_label}? Calibration verdicts collected against the current version stay tied to it." } %>
|
|
102
|
+
<% end %>
|
|
103
|
+
</div>
|
|
104
|
+
<% if version_changed.call(v, pred) %>
|
|
105
|
+
<button type="button" class="ck-cell-link ck-cell-link--delta"
|
|
106
|
+
title="What changed from #{pred.version_label}"
|
|
107
|
+
onclick="document.getElementById('ck-mvdiff-<%= v.id %>').showModal()">Δ</button>
|
|
108
|
+
<% end %>
|
|
109
|
+
</div>
|
|
110
|
+
</td>
|
|
111
|
+
<td>
|
|
112
|
+
<% source_label, source_class = case v.source
|
|
113
|
+
when "suggestion" then ["AI suggestion", "ck-source-chip ck-source-chip--ai"]
|
|
114
|
+
when "edit" then ["Manual edit", "ck-source-chip ck-source-chip--manual"]
|
|
115
|
+
else ["Original", "ck-source-chip ck-source-chip--initial"]
|
|
116
|
+
end %>
|
|
117
|
+
<span class="<%= source_class %>"><%= source_label %></span>
|
|
118
|
+
</td>
|
|
119
|
+
<td class="ck-meta-copy">
|
|
120
|
+
<time datetime="<%= v.created_at.utc.iso8601 %>" data-relative-time><%= time_ago_in_words(v.created_at) %> ago</time>
|
|
121
|
+
</td>
|
|
122
|
+
</tr>
|
|
123
|
+
<% end %>
|
|
124
|
+
</tbody>
|
|
125
|
+
</table>
|
|
126
|
+
</section>
|
|
111
127
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
<
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
128
|
+
<% @versions.each do |v| %>
|
|
129
|
+
<% pred = predecessor_of[v] %>
|
|
130
|
+
<% next unless version_changed.call(v, pred) %>
|
|
131
|
+
<dialog id="ck-mvdiff-<%= v.id %>" class="ck-modal" onclick="if(event.target===this)this.close()">
|
|
132
|
+
<article class="ck-modal__panel" tabindex="-1" onclick="event.stopPropagation()">
|
|
133
|
+
<header class="ck-modal__header">
|
|
134
|
+
<div class="ck-modal__heading">
|
|
135
|
+
<h2 class="ck-modal__title"><%= pred.version_label %> → <%= v.version_label %></h2>
|
|
136
|
+
<span class="ck-modal__meta">What changed in <%= v.version_label %><% if v.current? %> (live)<% elsif v.draft? %> (draft)<% end %></span>
|
|
137
|
+
</div>
|
|
138
|
+
<button type="button" class="ck-modal__close" aria-label="Close" onclick="this.closest('dialog').close()">×</button>
|
|
139
|
+
</header>
|
|
140
|
+
<div class="ck-modal__body">
|
|
141
|
+
<% if pred.instruction.to_s != v.instruction.to_s %>
|
|
142
|
+
<div class="ck-suggest-diff">
|
|
143
|
+
<div class="ck-suggest-diff__pane">
|
|
144
|
+
<div class="ck-suggest-diff__header"><span class="ck-suggest-diff__label ck-suggest-diff__label--before"><%= pred.version_label %> instruction</span></div>
|
|
145
|
+
<pre class="ck-suggest-diff__code"><%= ck_word_diff_old(pred.instruction.to_s, v.instruction.to_s) %></pre>
|
|
146
|
+
</div>
|
|
147
|
+
<div class="ck-suggest-diff__pane">
|
|
148
|
+
<div class="ck-suggest-diff__header"><span class="ck-suggest-diff__label ck-suggest-diff__label--after"><%= v.version_label %> instruction</span></div>
|
|
149
|
+
<pre class="ck-suggest-diff__code"><%= ck_word_diff_new(pred.instruction.to_s, v.instruction.to_s) %></pre>
|
|
150
|
+
</div>
|
|
151
|
+
</div>
|
|
152
|
+
<% end %>
|
|
153
|
+
<% if pred.rubric_bands != v.rubric_bands %>
|
|
154
|
+
<p class="ck-kicker ck-kicker--inset">Rubric changes</p>
|
|
155
|
+
<%= render "completion_kit/metrics/rubric_diff",
|
|
156
|
+
current_bands: pred.rubric_bands || [],
|
|
157
|
+
draft_bands: v.rubric_bands || [] %>
|
|
158
|
+
<% end %>
|
|
122
159
|
</div>
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
<% if sd_rubric_changed %>
|
|
128
|
-
<p class="ck-meta-copy"><strong>Rubric also changed.</strong> Publishing applies the new rubric too.</p>
|
|
129
|
-
<% end %>
|
|
130
|
-
|
|
131
|
-
<div class="ck-actions">
|
|
132
|
-
<%= button_to "Discard", dismiss_suggestion_metric_path(@metric, draft_id: @suggestion_draft.id),
|
|
133
|
-
method: :delete, form_class: "inline-block",
|
|
134
|
-
class: ck_button_classes(:light, variant: :outline),
|
|
135
|
-
data: { turbo_confirm: "Drop this suggestion?" } %>
|
|
136
|
-
<%= button_to "Use this version", publish_draft_metric_path(@metric, draft_id: @suggestion_draft.id),
|
|
137
|
-
method: :post, form_class: "inline-block",
|
|
138
|
-
class: ck_button_classes(:dark) %>
|
|
139
|
-
</div>
|
|
140
|
-
</section>
|
|
160
|
+
</article>
|
|
161
|
+
</dialog>
|
|
162
|
+
<% end %>
|
|
141
163
|
<% end %>
|
|
142
164
|
|
|
143
165
|
<% if CompletionKit.config.judge_calibration_enabled && @disagreements.any? %>
|
|
144
166
|
<section class="ck-card ck-card--spaced">
|
|
145
167
|
<div class="ck-prompt-preview__header">
|
|
146
168
|
<p class="ck-kicker">Cases to learn from</p>
|
|
147
|
-
<span class="ck-chip"><%= pluralize(@disagreements.size, "
|
|
169
|
+
<span class="ck-chip"><%= pluralize(@disagreements.size, "case") %></span>
|
|
148
170
|
</div>
|
|
149
|
-
<p class="ck-meta-copy">
|
|
171
|
+
<p class="ck-meta-copy">Cases where a reviewer's score didn't match the judge's. Pin useful ones with <strong>Remember this</strong> so the judge sees them next time it grades.</p>
|
|
150
172
|
<% existing_ids = Array(@metric.few_shot_examples).map { |fs| fs["calibration_id"] } %>
|
|
151
173
|
<ul class="ck-disagreement-list">
|
|
152
174
|
<% @disagreements.each do |cal| %>
|
|
153
175
|
<% review = cal.response.reviews.find { |r| r.metric_id == @metric.id } %>
|
|
154
176
|
<% already = existing_ids.include?(cal.id) %>
|
|
155
|
-
|
|
177
|
+
<% cal_version = cal.judge_version %>
|
|
178
|
+
<% on_current = cal_version&.id == @published_judge_version.id %>
|
|
179
|
+
<li class="ck-disagreement<%= " ck-disagreement--remembered" if already %><%= " ck-disagreement--stale" unless on_current %>">
|
|
156
180
|
<div class="ck-disagreement__head">
|
|
157
181
|
<div class="ck-disagreement__scores">
|
|
182
|
+
<% if cal_version %>
|
|
183
|
+
<span class="ck-source-chip <%= on_current ? "ck-source-chip--current" : "ck-source-chip--past" %>" title="<%= on_current ? "Verdict on the live judge version." : "Verdict on a superseded judge version." %>"><%= cal_version.version_label %></span>
|
|
184
|
+
<% end %>
|
|
158
185
|
<span class="ck-disagreement__scores-label">Judge</span>
|
|
159
186
|
<% if review&.ai_score %>
|
|
160
187
|
<span class="<%= ck_badge_classes(ck_score_kind(review.ai_score.to_f)) %>"><%= review.ai_score %></span>
|
|
@@ -171,14 +198,21 @@
|
|
|
171
198
|
</div>
|
|
172
199
|
<div class="ck-disagreement__action">
|
|
173
200
|
<% if already %>
|
|
174
|
-
|
|
201
|
+
<%= button_to "Forget",
|
|
202
|
+
remove_few_shot_metric_path(@metric, calibration_id: cal.id),
|
|
203
|
+
method: :delete,
|
|
204
|
+
form_class: "inline-block",
|
|
205
|
+
class: ck_button_classes(:light, variant: :outline),
|
|
206
|
+
title: "Stop showing this case to the judge.",
|
|
207
|
+
data: { turbo_confirm: "Stop showing this case to the judge?" } %>
|
|
208
|
+
<span class="ck-chip ck-chip--done" title="The judge sees this row when it grades for this metric.">Remembered</span>
|
|
175
209
|
<% else %>
|
|
176
210
|
<%= button_to "Remember this",
|
|
177
211
|
add_few_shot_metric_path(@metric, calibration_id: cal.id),
|
|
178
212
|
method: :post,
|
|
179
213
|
form_class: "inline-block",
|
|
180
214
|
class: ck_button_classes(:light, variant: :outline),
|
|
181
|
-
title: "Pin this
|
|
215
|
+
title: "Pin this case so the judge sees it next time it grades for this metric." %>
|
|
182
216
|
<% end %>
|
|
183
217
|
</div>
|
|
184
218
|
</div>
|
|
@@ -186,42 +220,14 @@
|
|
|
186
220
|
<p class="ck-disagreement__note"><%= cal.note %></p>
|
|
187
221
|
<% end %>
|
|
188
222
|
<p class="ck-disagreement__source ck-meta-copy">
|
|
189
|
-
<%= link_to cal.response.run
|
|
190
|
-
|
|
191
|
-
|
|
223
|
+
<%= link_to run_response_path(cal.response.run, cal.response, anchor: @metric.name.parameterize),
|
|
224
|
+
class: "ck-disagreement__source-link" do %>
|
|
225
|
+
<% case_display = cal.response.row_index.nil? ? "##{cal.response.id}" : (cal.response.row_index + 1).to_s %>
|
|
226
|
+
View case <%= case_display %> in <%= cal.response.run.name.to_s.truncate(50) %> →
|
|
227
|
+
<% end %>
|
|
192
228
|
</p>
|
|
193
229
|
</li>
|
|
194
230
|
<% end %>
|
|
195
231
|
</ul>
|
|
196
232
|
</section>
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
<% if Array(@metric.few_shot_examples).any? %>
|
|
200
|
-
<section class="ck-card ck-card--spaced">
|
|
201
|
-
<div class="ck-prompt-preview__header">
|
|
202
|
-
<p class="ck-kicker">What the judge remembers</p>
|
|
203
|
-
<span class="ck-chip"><%= pluralize(Array(@metric.few_shot_examples).size, "case") %></span>
|
|
204
|
-
</div>
|
|
205
|
-
<p class="ck-meta-copy">Rows you've pinned so the judge sees them next time it grades. Each one shows what the judge gave and what a human said it should have been.</p>
|
|
206
|
-
<ol class="ck-few-shot-list">
|
|
207
|
-
<% Array(@metric.few_shot_examples).each do |fs| %>
|
|
208
|
-
<li class="ck-few-shot-item">
|
|
209
|
-
<div class="ck-few-shot-item__scores">
|
|
210
|
-
<span class="ck-meta-copy">judge said</span>
|
|
211
|
-
<% if fs["judge_score"] %>
|
|
212
|
-
<span class="<%= ck_badge_classes(ck_score_kind(fs["judge_score"].to_f)) %>"><%= fs["judge_score"] %></span>
|
|
213
|
-
<% end %>
|
|
214
|
-
<span class="ck-meta-copy">human said</span>
|
|
215
|
-
<% if fs["human_score"] %>
|
|
216
|
-
<span class="<%= ck_badge_classes(ck_score_kind(fs["human_score"].to_f)) %>"><%= fs["human_score"] %></span>
|
|
217
|
-
<% end %>
|
|
218
|
-
</div>
|
|
219
|
-
<% if fs["human_note"].to_s.present? %>
|
|
220
|
-
<p class="ck-copy"><%= fs["human_note"] %></p>
|
|
221
|
-
<% end %>
|
|
222
|
-
</li>
|
|
223
|
-
<% end %>
|
|
224
|
-
</ol>
|
|
225
|
-
</section>
|
|
226
|
-
<% end %>
|
|
227
233
|
<% end %>
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
<div>
|
|
3
3
|
<p class="ck-kicker">Starter metric</p>
|
|
4
4
|
<h1 class="ck-title"><%= @starter.name %></h1>
|
|
5
|
-
<p class="ck-lead"><%= @starter.description %></p>
|
|
6
5
|
</div>
|
|
7
6
|
<div class="ck-actions">
|
|
8
7
|
<%= link_to "← Back to metrics", metrics_path, class: ck_button_classes(:light, variant: :outline) %>
|
|
@@ -10,17 +9,18 @@
|
|
|
10
9
|
</section>
|
|
11
10
|
|
|
12
11
|
<section class="ck-card ck-card--spaced">
|
|
13
|
-
<p class="ck-kicker">
|
|
12
|
+
<p class="ck-kicker">Why use this</p>
|
|
13
|
+
<p class="ck-copy"><strong><%= @starter.description %></strong></p>
|
|
14
14
|
<p class="ck-copy"><%= @starter.catches %></p>
|
|
15
15
|
</section>
|
|
16
16
|
|
|
17
17
|
<section class="ck-card ck-card--spaced">
|
|
18
|
-
<p class="ck-kicker">
|
|
18
|
+
<p class="ck-kicker">Judge instruction</p>
|
|
19
19
|
<p class="ck-copy"><%= @starter.instruction %></p>
|
|
20
20
|
</section>
|
|
21
21
|
|
|
22
22
|
<section class="ck-card ck-card--spaced">
|
|
23
|
-
<p class="ck-kicker">Rubric
|
|
23
|
+
<p class="ck-kicker">Rubric<%= render "completion_kit/metrics/rubric_hint" %></p>
|
|
24
24
|
<div class="ck-rubric-display">
|
|
25
25
|
<% @starter.rubric_bands.sort_by { |b| -b["stars"] }.each do |band| %>
|
|
26
26
|
<div class="ck-rubric-row ck-rubric-row--display">
|
|
@@ -37,9 +37,9 @@
|
|
|
37
37
|
</div>
|
|
38
38
|
</section>
|
|
39
39
|
|
|
40
|
-
<div class="ck-
|
|
40
|
+
<div class="ck-starter-actions">
|
|
41
41
|
<%= link_to "Cancel", metrics_path, class: ck_button_classes(:light, variant: :outline) %>
|
|
42
|
-
<%= button_to "Add
|
|
42
|
+
<%= button_to "Add #{@starter.name} to my metrics", adopt_starter_metrics_path(key: @starter.key),
|
|
43
43
|
method: :post, form_class: "inline-block",
|
|
44
44
|
class: ck_button_classes(:dark) %>
|
|
45
45
|
</div>
|
|
@@ -98,7 +98,7 @@
|
|
|
98
98
|
|
|
99
99
|
<div class="ck-review-list">
|
|
100
100
|
<% @reviews.each do |review| %>
|
|
101
|
-
<div class="ck-review-card">
|
|
101
|
+
<div class="ck-review-card" id="<%= review.metric&.name&.parameterize || "review-#{review.id}" %>">
|
|
102
102
|
<div class="ck-review-card__header">
|
|
103
103
|
<span class="ck-review-card__metric"><% if review.metric %><%= link_to review.metric_name, metric_path(review.metric), class: "ck-link" %><% else %><%= review.metric_name %><% end %></span>
|
|
104
104
|
<div class="ck-inline">
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
<% else %>
|
|
47
47
|
<span class="ck-metric-pip ck-metric-pip--pending">
|
|
48
48
|
<span class="ck-metric-pip__bar"></span>
|
|
49
|
-
<span class="ck-metric-pip__label"><%= metric.name %> <
|
|
49
|
+
<span class="ck-metric-pip__label"><%= metric.name %> <span class="ck-metric-pip__sub">pending</span></span>
|
|
50
50
|
</span>
|
|
51
51
|
<% end %>
|
|
52
52
|
<% end %>
|
|
@@ -72,7 +72,7 @@
|
|
|
72
72
|
|
|
73
73
|
<% if failed_count > 0 %>
|
|
74
74
|
<%= button_to retry_failures_run_path(run), method: :post, class: "ck-run-status__retry", form_class: "ck-run-status__action" do %>
|
|
75
|
-
Retry <%= failed_count %> failed <%= "
|
|
75
|
+
Retry <%= failed_count %> failed <%= "case".pluralize(failed_count) %>
|
|
76
76
|
<% end %>
|
|
77
77
|
<% end %>
|
|
78
78
|
</section>
|
data/config/routes.rb
CHANGED
|
@@ -19,7 +19,8 @@ CompletionKit::Engine.routes.draw do
|
|
|
19
19
|
post "starters/:key/dismiss", to: "metrics#dismiss_starter", as: :dismiss_starter
|
|
20
20
|
end
|
|
21
21
|
member do
|
|
22
|
-
post
|
|
22
|
+
post :add_few_shot
|
|
23
|
+
delete :remove_few_shot
|
|
23
24
|
post :publish_draft
|
|
24
25
|
post :suggest_variants
|
|
25
26
|
delete :dismiss_suggestion
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
class AddVersionNumberAndPublishedAtToJudgeVersions < ActiveRecord::Migration[8.1]
|
|
2
|
+
def change
|
|
3
|
+
add_column :completion_kit_judge_versions, :version_number, :integer
|
|
4
|
+
add_column :completion_kit_judge_versions, :published_at, :datetime
|
|
5
|
+
|
|
6
|
+
reversible do |dir|
|
|
7
|
+
dir.up do
|
|
8
|
+
jv = Class.new(ActiveRecord::Base) { self.table_name = "completion_kit_judge_versions" }
|
|
9
|
+
jv.distinct.pluck(:metric_id).each do |metric_id|
|
|
10
|
+
jv.where(metric_id: metric_id).order(:created_at, :id).each_with_index do |row, i|
|
|
11
|
+
updates = { version_number: i + 1 }
|
|
12
|
+
updates[:published_at] = row.created_at if row[:state] == "published"
|
|
13
|
+
jv.where(id: row.id).update_all(updates)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
change_column_null :completion_kit_judge_versions, :version_number, false
|
|
20
|
+
add_index :completion_kit_judge_versions,
|
|
21
|
+
[:metric_id, :version_number],
|
|
22
|
+
name: "index_ck_judge_versions_on_metric_version"
|
|
23
|
+
end
|
|
24
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: completion-kit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.43
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Damien Bastin
|
|
@@ -347,6 +347,8 @@ files:
|
|
|
347
347
|
- app/views/completion_kit/metric_groups/new.html.erb
|
|
348
348
|
- app/views/completion_kit/metric_groups/show.html.erb
|
|
349
349
|
- app/views/completion_kit/metrics/_form.html.erb
|
|
350
|
+
- app/views/completion_kit/metrics/_rubric_diff.html.erb
|
|
351
|
+
- app/views/completion_kit/metrics/_rubric_hint.html.erb
|
|
350
352
|
- app/views/completion_kit/metrics/_starter_card.html.erb
|
|
351
353
|
- app/views/completion_kit/metrics/edit.html.erb
|
|
352
354
|
- app/views/completion_kit/metrics/index.html.erb
|
|
@@ -419,6 +421,7 @@ files:
|
|
|
419
421
|
- db/migrate/20260523000001_add_few_shot_examples_to_completion_kit_metrics.rb
|
|
420
422
|
- db/migrate/20260523000002_add_state_to_completion_kit_judge_versions.rb
|
|
421
423
|
- db/migrate/20260524000001_create_completion_kit_starter_metric_dismissals.rb
|
|
424
|
+
- db/migrate/20260525000001_add_version_number_and_published_at_to_judge_versions.rb
|
|
422
425
|
- lib/completion-kit.rb
|
|
423
426
|
- lib/completion_kit.rb
|
|
424
427
|
- lib/completion_kit/concurrency_check.rb
|