completion-kit 0.28.35 → 0.28.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/completion_kit/runs_controller.rb +1 -0
- data/app/services/completion_kit/llm_client.rb +13 -6
- data/app/services/completion_kit/mcp_tools/runs.rb +3 -1
- data/app/views/completion_kit/runs/_run_config.html.erb +53 -0
- data/app/views/completion_kit/runs/show.html.erb +1 -53
- data/lib/completion_kit/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9c91a74066fe93a6df18e5e02a8fc8277bb93f4f15f3d4aed392ab911e88febc
|
|
4
|
+
data.tar.gz: 4d193ac02c79e75763aa97fcc528ab21521ce2ddbf290350818c539746601633
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8cbeba6b6a76455b23c4c9f45c88e77c73995e1dfa29bbc4be3acce814633977cd4a00af594df92f77e4740640907cdff520373582d19688aa2aec897bcf84b5
|
|
7
|
+
data.tar.gz: bd564daac0efe50d01a1b0850af2ca452f427ab6496b22a1af88865f4dc01917bb32f50d9b2371be88dbfe56e75acff0adfcd0e5b42f34c32d1ffd3b5f035f81
|
|
@@ -130,6 +130,7 @@ module CompletionKit
|
|
|
130
130
|
format.turbo_stream do
|
|
131
131
|
render turbo_stream: [
|
|
132
132
|
turbo_stream.replace("run_status_header", partial: "completion_kit/runs/status_header", locals: { run: @run }),
|
|
133
|
+
turbo_stream.replace("run_config", partial: "completion_kit/runs/run_config", locals: { run: @run }),
|
|
133
134
|
turbo_stream.replace("run_status_panel", partial: "completion_kit/runs/status_panel", locals: { run: @run }),
|
|
134
135
|
turbo_stream.replace("run_responses_region", partial: "completion_kit/runs/responses_region",
|
|
135
136
|
locals: { run: @run, responses: @responses, responses_offset: @responses_offset })
|
|
@@ -14,11 +14,16 @@ module CompletionKit
|
|
|
14
14
|
# recoverable refusal into a failed row.
|
|
15
15
|
#
|
|
16
16
|
# The wording can sit on either side of the parameter name, so both orders
|
|
17
|
-
# carry the same phrase set
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
#
|
|
17
|
+
# carry the same phrase set, and a provider that names the offending field
|
|
18
|
+
# outright gets a branch of its own.
|
|
19
|
+
#
|
|
20
|
+
# Neither gap may cross a brace, which stops a refusal aimed at some other
|
|
21
|
+
# parameter from matching a temperature echoed back in a neighbouring
|
|
22
|
+
# object of the same body. The phrase-first gap additionally bars commas,
|
|
23
|
+
# because "unsupported parameter: logprobs. Supported: model, prompt,
|
|
24
|
+
# temperature" is a list of what IS allowed and must not read as a refusal.
|
|
25
|
+
# The temperature-first gap has to allow commas, since "temperature, top_p
|
|
26
|
+
# and top_k are not supported" is a genuine refusal.
|
|
22
27
|
REFUSAL_PHRASE = /
|
|
23
28
|
deprecated | not\s+supported | does\s+not\s+support |
|
|
24
29
|
only\s+the\s+default | unsupported\s+(?:parameter|value)
|
|
@@ -26,7 +31,9 @@ module CompletionKit
|
|
|
26
31
|
TEMPERATURE_REFUSAL = /
|
|
27
32
|
temperature [^{}]{0,80}? #{REFUSAL_PHRASE}
|
|
28
33
|
|
|
|
29
|
-
#{REFUSAL_PHRASE} [^{}]{0,80}? temperature
|
|
34
|
+
#{REFUSAL_PHRASE} [^{},]{0,80}? temperature
|
|
35
|
+
|
|
|
36
|
+
"param" \s*:\s* "temperature"
|
|
30
37
|
/xi
|
|
31
38
|
|
|
32
39
|
def initialize(config = {})
|
|
@@ -4,7 +4,9 @@ module CompletionKit
|
|
|
4
4
|
extend Base
|
|
5
5
|
|
|
6
6
|
TEMPERATURE_DESCRIPTION = "Sampling temperature for generation, 0 to 1. Defaults to the column default. " \
|
|
7
|
-
"Reasoning models ignore it and the run is flagged temperature_ignored."
|
|
7
|
+
"Reasoning models ignore it and the run is flagged temperature_ignored. " \
|
|
8
|
+
"Pass null to send no temperature at all, which is what models that refuse " \
|
|
9
|
+
"the parameter require.".freeze
|
|
8
10
|
|
|
9
11
|
MAX_TOKENS_DESCRIPTION = "Cap on generated tokens per row. Leave unset to use the provider client's default, " \
|
|
10
12
|
"which is what silently truncates long outputs and makes the judge score malformed " \
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
<div class="ck-run-config" id="run_config">
|
|
2
|
+
<div class="ck-run-config__row">
|
|
3
|
+
<span class="ck-run-config__key">Created</span>
|
|
4
|
+
<time datetime="<%= run.created_at.iso8601 %>" data-local-time><%= run.created_at.utc.strftime("%Y-%m-%d %H:%M UTC") %></time>
|
|
5
|
+
</div>
|
|
6
|
+
<div class="ck-run-config__row">
|
|
7
|
+
<span class="ck-run-config__key">Dataset</span>
|
|
8
|
+
<% if run.dataset %>
|
|
9
|
+
<span class="ck-run-config__dataset">
|
|
10
|
+
<%= link_to run.dataset.name, dataset_path(run.dataset), class: "ck-link" %>
|
|
11
|
+
<span class="ck-run-config__dataset-meta"><%= run.dataset.row_count %> rows</span>
|
|
12
|
+
<button type="button" class="ck-run-config__dataset-preview" onclick="document.getElementById('dataset-preview-<%= run.id %>').showModal()">Preview</button>
|
|
13
|
+
</span>
|
|
14
|
+
<% else %>
|
|
15
|
+
<span class="ck-run-config__none">None</span>
|
|
16
|
+
<% end %>
|
|
17
|
+
</div>
|
|
18
|
+
<% if run.judge_model.present? %>
|
|
19
|
+
<div class="ck-run-config__row">
|
|
20
|
+
<span class="ck-run-config__key">Judge</span>
|
|
21
|
+
<span style="text-transform: none;"><%= run.judge_model %></span>
|
|
22
|
+
<% unless run.judge_configured? %>
|
|
23
|
+
<span class="ck-run-config__warn">provider not configured</span>
|
|
24
|
+
<% end %>
|
|
25
|
+
</div>
|
|
26
|
+
<% end %>
|
|
27
|
+
<% if run.metrics.any? %>
|
|
28
|
+
<div class="ck-run-config__row">
|
|
29
|
+
<span class="ck-run-config__key">Metrics</span>
|
|
30
|
+
<span><%= run.metrics.map { |m| link_to(m.name, metric_path(m), class: "ck-link") }.join(", ").html_safe %></span>
|
|
31
|
+
</div>
|
|
32
|
+
<% end %>
|
|
33
|
+
<div class="ck-run-config__row">
|
|
34
|
+
<span class="ck-run-config__key">Temperature</span>
|
|
35
|
+
<span><%= run.temperature.nil? ? "Not sent, provider default" : run.temperature %></span>
|
|
36
|
+
<% if run.temperature_ignored? %>
|
|
37
|
+
<span class="ck-run-config__warn" style="color: var(--ck-dim);" title="The model rejected the temperature parameter, so CompletionKit re-sent the request without it.">ignored by model</span>
|
|
38
|
+
<% end %>
|
|
39
|
+
</div>
|
|
40
|
+
<div class="ck-run-config__row">
|
|
41
|
+
<span class="ck-run-config__key">Max tokens</span>
|
|
42
|
+
<span><%= run.max_tokens || "Provider default" %></span>
|
|
43
|
+
</div>
|
|
44
|
+
<div class="ck-run-config__row">
|
|
45
|
+
<span class="ck-run-config__key">Judge temperature</span>
|
|
46
|
+
<span><%= run.judge_temperature %></span>
|
|
47
|
+
<% if run.judge_temperature_ignored? %>
|
|
48
|
+
<span class="ck-run-config__warn" style="color: var(--ck-warning);" title="The judge model refused the temperature parameter, so CompletionKit re-sent the request without it and the provider applied its own default. The scores are not reproducible despite the setting above.">refused by judge, not reproducible</span>
|
|
49
|
+
<% elsif run.nondeterministic_judge? %>
|
|
50
|
+
<span class="ck-run-config__warn" style="color: var(--ck-warning);" title="Judging above temperature 0 makes scores irreproducible. The same output can get a different score on a re-judge.">scores not reproducible</span>
|
|
51
|
+
<% end %>
|
|
52
|
+
</div>
|
|
53
|
+
</div>
|
|
@@ -50,59 +50,7 @@
|
|
|
50
50
|
<% end %>
|
|
51
51
|
<% end %>
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
<div class="ck-run-config__row">
|
|
55
|
-
<span class="ck-run-config__key">Created</span>
|
|
56
|
-
<time datetime="<%= @run.created_at.iso8601 %>" data-local-time><%= @run.created_at.utc.strftime("%Y-%m-%d %H:%M UTC") %></time>
|
|
57
|
-
</div>
|
|
58
|
-
<div class="ck-run-config__row">
|
|
59
|
-
<span class="ck-run-config__key">Dataset</span>
|
|
60
|
-
<% if @run.dataset %>
|
|
61
|
-
<span class="ck-run-config__dataset">
|
|
62
|
-
<%= link_to @run.dataset.name, dataset_path(@run.dataset), class: "ck-link" %>
|
|
63
|
-
<span class="ck-run-config__dataset-meta"><%= @run.dataset.row_count %> rows</span>
|
|
64
|
-
<button type="button" class="ck-run-config__dataset-preview" onclick="document.getElementById('dataset-preview-<%= @run.id %>').showModal()">Preview</button>
|
|
65
|
-
</span>
|
|
66
|
-
<% else %>
|
|
67
|
-
<span class="ck-run-config__none">None</span>
|
|
68
|
-
<% end %>
|
|
69
|
-
</div>
|
|
70
|
-
<% if @run.judge_model.present? %>
|
|
71
|
-
<div class="ck-run-config__row">
|
|
72
|
-
<span class="ck-run-config__key">Judge</span>
|
|
73
|
-
<span style="text-transform: none;"><%= @run.judge_model %></span>
|
|
74
|
-
<% unless @run.judge_configured? %>
|
|
75
|
-
<span class="ck-run-config__warn">provider not configured</span>
|
|
76
|
-
<% end %>
|
|
77
|
-
</div>
|
|
78
|
-
<% end %>
|
|
79
|
-
<% if @run.metrics.any? %>
|
|
80
|
-
<div class="ck-run-config__row">
|
|
81
|
-
<span class="ck-run-config__key">Metrics</span>
|
|
82
|
-
<span><%= @run.metrics.map { |m| link_to(m.name, metric_path(m), class: "ck-link") }.join(", ").html_safe %></span>
|
|
83
|
-
</div>
|
|
84
|
-
<% end %>
|
|
85
|
-
<div class="ck-run-config__row">
|
|
86
|
-
<span class="ck-run-config__key">Temperature</span>
|
|
87
|
-
<span><%= @run.temperature %></span>
|
|
88
|
-
<% if @run.temperature_ignored? %>
|
|
89
|
-
<span class="ck-run-config__warn" style="color: var(--ck-dim);" title="The model rejected the temperature parameter, so CompletionKit re-sent the request without it.">ignored by model</span>
|
|
90
|
-
<% end %>
|
|
91
|
-
</div>
|
|
92
|
-
<div class="ck-run-config__row">
|
|
93
|
-
<span class="ck-run-config__key">Max tokens</span>
|
|
94
|
-
<span><%= @run.max_tokens || "Provider default" %></span>
|
|
95
|
-
</div>
|
|
96
|
-
<div class="ck-run-config__row">
|
|
97
|
-
<span class="ck-run-config__key">Judge temperature</span>
|
|
98
|
-
<span><%= @run.judge_temperature %></span>
|
|
99
|
-
<% if @run.judge_temperature_ignored? %>
|
|
100
|
-
<span class="ck-run-config__warn" style="color: var(--ck-warning);" title="The judge model refused the temperature parameter, so CompletionKit re-sent the request without it and the provider applied its own default. The scores are not reproducible despite the setting above.">refused by judge, not reproducible</span>
|
|
101
|
-
<% elsif @run.nondeterministic_judge? %>
|
|
102
|
-
<span class="ck-run-config__warn" style="color: var(--ck-warning);" title="Judging above temperature 0 makes scores irreproducible. The same output can get a different score on a re-judge.">scores not reproducible</span>
|
|
103
|
-
<% end %>
|
|
104
|
-
</div>
|
|
105
|
-
</div>
|
|
53
|
+
<%= render "run_config", run: @run %>
|
|
106
54
|
|
|
107
55
|
<% if @run.prompt %>
|
|
108
56
|
<div class="ck-prompt-preview">
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: completion-kit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.28.
|
|
4
|
+
version: 0.28.36
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Damien Bastin
|
|
@@ -373,6 +373,7 @@ files:
|
|
|
373
373
|
- app/views/completion_kit/runs/_response_row.html.erb
|
|
374
374
|
- app/views/completion_kit/runs/_responses_region.html.erb
|
|
375
375
|
- app/views/completion_kit/runs/_row.html.erb
|
|
376
|
+
- app/views/completion_kit/runs/_run_config.html.erb
|
|
376
377
|
- app/views/completion_kit/runs/_sort_toolbar.html.erb
|
|
377
378
|
- app/views/completion_kit/runs/_status_header.html.erb
|
|
378
379
|
- app/views/completion_kit/runs/_status_panel.html.erb
|