completion-kit 0.5.30 → 0.5.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/stylesheets/completion_kit/application.css +7 -0
- data/app/controllers/completion_kit/application_controller.rb +7 -0
- data/app/controllers/completion_kit/dashboard_controller.rb +22 -0
- data/app/controllers/completion_kit/onboarding_controller.rb +4 -6
- data/app/services/completion_kit/onboarding/concepts.rb +32 -0
- data/app/views/completion_kit/dashboard/show.html.erb +91 -0
- data/app/views/completion_kit/onboarding/_concept.html.erb +3 -0
- data/app/views/completion_kit/onboarding/show.html.erb +1 -0
- data/app/views/layouts/completion_kit/application.html.erb +1 -1
- data/config/routes.rb +1 -0
- data/lib/completion_kit/version.rb +1 -1
- metadata +5 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3edbdacce1ca9737b2e7c3c93f2f7cfd27fbea96b2d55d470c8f5dfa7299d247
|
|
4
|
+
data.tar.gz: 8dc9c09b93800fbb7262015a9cf340030bb8e90dad8bda5ca1a97579a1987173
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 21a65c71afb3769bdbe8434ea71bf9ee6101ac0f47377080f4bfda4b8eb2acb7e4d2631d41202493eec619e3db44a13827fc46a568a1e134fcd836d2182ae5a6
|
|
7
|
+
data.tar.gz: 77c8534e4842727910628c18672918983d21c655ebdcb3998a70a3bfe0578257c3d41aeee200dc4841f505d74d2b097aa0ce1b493456e015f47c9438e5e8d622
|
|
@@ -118,6 +118,13 @@ form.button_to {
|
|
|
118
118
|
padding: 0.75rem 0;
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
+
@media (min-width: 641px) {
|
|
122
|
+
.ck-topbar__inner .ck-nav {
|
|
123
|
+
position: relative;
|
|
124
|
+
top: 8px;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
121
128
|
/* Mobile navigation — a <details> hamburger. On desktop the trigger is
|
|
122
129
|
hidden and the nav shows inline; at the mobile breakpoint the nav
|
|
123
130
|
collapses behind the trigger and opens as a dropdown panel. The two
|
|
@@ -3,12 +3,19 @@ module CompletionKit
|
|
|
3
3
|
helper Heroicons::IconsHelper
|
|
4
4
|
layout "completion_kit/application"
|
|
5
5
|
|
|
6
|
+
ONBOARDING_DISMISS_COOKIE = :ck_onboarding_dismissed
|
|
7
|
+
|
|
6
8
|
rate_limit to: CompletionKit.config.web_rate_limit, within: 1.minute,
|
|
7
9
|
with: -> { render plain: "Rate limit exceeded. Please slow down.", status: :too_many_requests }
|
|
8
10
|
before_action :authenticate_completion_kit!
|
|
9
11
|
|
|
10
12
|
private
|
|
11
13
|
|
|
14
|
+
def workspace_ready?
|
|
15
|
+
CompletionKit::Onboarding::Checklist.new.complete? ||
|
|
16
|
+
cookies[ONBOARDING_DISMISS_COOKIE].present?
|
|
17
|
+
end
|
|
18
|
+
|
|
12
19
|
def authenticate_completion_kit!
|
|
13
20
|
cfg = CompletionKit.config
|
|
14
21
|
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
class DashboardController < ApplicationController
|
|
3
|
+
def show
|
|
4
|
+
return redirect_to(onboarding_path) unless workspace_ready?
|
|
5
|
+
|
|
6
|
+
@prompt_count = Prompt.current_versions.count
|
|
7
|
+
@run_count = Run.count
|
|
8
|
+
@dataset_count = Dataset.count
|
|
9
|
+
@metric_count = Metric.count
|
|
10
|
+
@recent_runs = Run.order(created_at: :desc).limit(5)
|
|
11
|
+
|
|
12
|
+
return unless @run_count > 5
|
|
13
|
+
|
|
14
|
+
@activity = DashboardStats.activity
|
|
15
|
+
@worst_metric = DashboardStats.worst_metric(since: 7.days.ago)
|
|
16
|
+
@failures = DashboardStats.failures(since: 7.days.ago)
|
|
17
|
+
@ignored_metrics = DashboardDismissal.metrics
|
|
18
|
+
@ignored_failures = DashboardDismissal.failures
|
|
19
|
+
@prompt_changes = DashboardStats.prompt_changes
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -1,18 +1,16 @@
|
|
|
1
1
|
module CompletionKit
|
|
2
2
|
class OnboardingController < ApplicationController
|
|
3
|
-
DISMISS_COOKIE = :ck_onboarding_dismissed
|
|
4
|
-
|
|
5
3
|
def show
|
|
6
|
-
cookies.delete(
|
|
4
|
+
cookies.delete(ONBOARDING_DISMISS_COOKIE) if params[:reset]
|
|
7
5
|
@checklist = Onboarding::Checklist.new
|
|
8
6
|
return if params[:reset]
|
|
9
7
|
|
|
10
|
-
redirect_to
|
|
8
|
+
redirect_to dashboard_path if workspace_ready?
|
|
11
9
|
end
|
|
12
10
|
|
|
13
11
|
def dismiss
|
|
14
|
-
cookies[
|
|
15
|
-
redirect_to
|
|
12
|
+
cookies[ONBOARDING_DISMISS_COOKIE] = { value: "1", expires: 1.year.from_now, httponly: true }
|
|
13
|
+
redirect_to dashboard_path, notice: "Setup skipped. Pick it back up from Settings → Getting started any time."
|
|
16
14
|
end
|
|
17
15
|
|
|
18
16
|
def sample_data
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Onboarding
|
|
3
|
+
module Concepts
|
|
4
|
+
DEFINITIONS = {
|
|
5
|
+
credential: {
|
|
6
|
+
name: "Provider Credential",
|
|
7
|
+
definition: "An API key for a model provider such as OpenAI or Anthropic. Encrypted at rest and never returned through the API."
|
|
8
|
+
},
|
|
9
|
+
dataset: {
|
|
10
|
+
name: "Dataset",
|
|
11
|
+
definition: "A CSV of real inputs. Each row becomes one test case."
|
|
12
|
+
},
|
|
13
|
+
prompt: {
|
|
14
|
+
name: "Prompt",
|
|
15
|
+
definition: "A versioned template with {{variable}} placeholders. Editing a prompt that has already been run creates a new version, so past results stay reproducible."
|
|
16
|
+
},
|
|
17
|
+
run: {
|
|
18
|
+
name: "Run",
|
|
19
|
+
definition: "One execution of a prompt against a dataset. Stores every output and the judge's scores."
|
|
20
|
+
},
|
|
21
|
+
response: {
|
|
22
|
+
name: "Response",
|
|
23
|
+
definition: "The model's output for a single dataset row, with the judge's reviews attached."
|
|
24
|
+
},
|
|
25
|
+
metric: {
|
|
26
|
+
name: "Metric",
|
|
27
|
+
definition: "An evaluation dimension with its own 1-5 rubric. The LLM judge scores every response against it."
|
|
28
|
+
}
|
|
29
|
+
}.freeze
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
<section class="ck-page-header">
|
|
2
|
+
<div>
|
|
3
|
+
<p class="ck-kicker">Dashboard</p>
|
|
4
|
+
<h1 class="ck-title">Prompt Testing Lab</h1>
|
|
5
|
+
</div>
|
|
6
|
+
<div class="ck-actions">
|
|
7
|
+
<%= link_to "New run →", new_run_path, class: "ck-button ck-button--secondary" %>
|
|
8
|
+
</div>
|
|
9
|
+
</section>
|
|
10
|
+
|
|
11
|
+
<nav class="ck-statbar ck-rise" aria-label="Workspace totals">
|
|
12
|
+
<%= link_to prompts_path, class: "ck-statbar__item" do %>
|
|
13
|
+
<span class="ck-statbar__label">Prompts</span>
|
|
14
|
+
<span class="ck-statbar__value"><%= @prompt_count %></span>
|
|
15
|
+
<% end %>
|
|
16
|
+
<%= link_to metrics_path, class: "ck-statbar__item" do %>
|
|
17
|
+
<span class="ck-statbar__label">Metrics</span>
|
|
18
|
+
<span class="ck-statbar__value"><%= @metric_count %></span>
|
|
19
|
+
<% end %>
|
|
20
|
+
<%= link_to datasets_path, class: "ck-statbar__item" do %>
|
|
21
|
+
<span class="ck-statbar__label">Datasets</span>
|
|
22
|
+
<span class="ck-statbar__value"><%= @dataset_count %></span>
|
|
23
|
+
<% end %>
|
|
24
|
+
<%= link_to runs_path, class: "ck-statbar__item" do %>
|
|
25
|
+
<span class="ck-statbar__label">Runs</span>
|
|
26
|
+
<span class="ck-statbar__value"><%= @run_count %></span>
|
|
27
|
+
<% end %>
|
|
28
|
+
</nav>
|
|
29
|
+
|
|
30
|
+
<% if @activity %>
|
|
31
|
+
<div class="ck-grid ck-grid--cards ck-grid--cards-3 ck-pulse-grid">
|
|
32
|
+
<div class="ck-card ck-stat-card ck-rise" style="--rise-delay: 60ms;">
|
|
33
|
+
<p class="ck-kicker">Activity · last 14 days</p>
|
|
34
|
+
<% activity_max = @activity.map { |d| d[:count] }.max %>
|
|
35
|
+
<div class="ck-sparkline" role="img" aria-label="<%= @activity.sum { |d| d[:count] } %> runs over the last 14 days">
|
|
36
|
+
<% @activity.each do |day| %>
|
|
37
|
+
<span class="ck-sparkline__bar<%= ' is-peak' if activity_max.to_i.positive? && day[:count] == activity_max %>"
|
|
38
|
+
style="height: <%= activity_max.to_i.zero? ? 0 : (day[:count] * 100.0 / activity_max).round %>%"
|
|
39
|
+
title="<%= day[:date].strftime('%b %-d') %>: <%= day[:count] %> run<%= 's' unless day[:count] == 1 %>"></span>
|
|
40
|
+
<% end %>
|
|
41
|
+
</div>
|
|
42
|
+
<p class="ck-stat-card__foot">
|
|
43
|
+
<span class="ck-stat-card__figure"><%= @activity.sum { |d| d[:count] } %></span> runs in the window
|
|
44
|
+
</p>
|
|
45
|
+
</div>
|
|
46
|
+
|
|
47
|
+
<%= render "completion_kit/dashboard/worst_metric_card",
|
|
48
|
+
worst_metric: @worst_metric, ignored_metrics: @ignored_metrics %>
|
|
49
|
+
|
|
50
|
+
<%= render "completion_kit/dashboard/failures_card",
|
|
51
|
+
failures: @failures, ignored_failures: @ignored_failures %>
|
|
52
|
+
</div>
|
|
53
|
+
|
|
54
|
+
<div class="ck-card ck-card--spaced ck-rise" style="--rise-delay: 240ms;">
|
|
55
|
+
<p class="ck-kicker">Prompt changes · version over version</p>
|
|
56
|
+
<% if @prompt_changes.any? %>
|
|
57
|
+
<ul class="ck-improvements">
|
|
58
|
+
<% @prompt_changes.each do |row| %>
|
|
59
|
+
<% gained = row[:delta].positive? %>
|
|
60
|
+
<li class="ck-improvement">
|
|
61
|
+
<%= link_to row[:prompt].name, prompt_path(row[:prompt]), class: "ck-improvement__name ck-link" %>
|
|
62
|
+
<span class="ck-improvement__versions">v<%= row[:from_version] %> → v<%= row[:to_version] %></span>
|
|
63
|
+
<span class="ck-improvement__scores">
|
|
64
|
+
<span class="ck-improvement__from"><%= row[:from_score] %></span>
|
|
65
|
+
<span class="ck-improvement__arrow">→</span>
|
|
66
|
+
<span class="ck-improvement__to"><%= row[:to_score] %></span>
|
|
67
|
+
</span>
|
|
68
|
+
<span class="ck-improvement__delta <%= gained ? 'is-gain' : 'is-loss' %>"><%= gained ? '▲' : '▼' %> <%= '%+.2f' % row[:delta] %></span>
|
|
69
|
+
</li>
|
|
70
|
+
<% end %>
|
|
71
|
+
</ul>
|
|
72
|
+
<% else %>
|
|
73
|
+
<p class="ck-improvements__empty">
|
|
74
|
+
No measured changes yet. Edit a prompt to create a new version, re-run it against the same dataset and metrics, and the score change, up or down, shows up here.
|
|
75
|
+
</p>
|
|
76
|
+
<% end %>
|
|
77
|
+
</div>
|
|
78
|
+
<% end %>
|
|
79
|
+
|
|
80
|
+
<section class="ck-card--spaced">
|
|
81
|
+
<div class="ck-split">
|
|
82
|
+
<h2 class="ck-section-title">Recent runs</h2>
|
|
83
|
+
<%= link_to "View all", runs_path, class: "ck-link" %>
|
|
84
|
+
</div>
|
|
85
|
+
|
|
86
|
+
<% if @recent_runs.any? %>
|
|
87
|
+
<%= render "completion_kit/runs/table", runs: @recent_runs %>
|
|
88
|
+
<% else %>
|
|
89
|
+
<div class="ck-empty">No runs yet. <%= link_to "Create your first run →", new_run_path, class: "ck-link" %></div>
|
|
90
|
+
<% end %>
|
|
91
|
+
</section>
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
<% concept = CompletionKit::Onboarding::Concepts::DEFINITIONS.fetch(key) -%>
|
|
2
|
+
<% term = local_assigns.fetch(:term, "") -%>
|
|
3
|
+
<span class="ck-concept"><%= term %><button type="button" class="ck-concept__toggle" aria-label="What is a <%= concept[:name] %>?" aria-describedby="concept-<%= key %>-pop"><%= heroicon_tag "information-circle", variant: :outline, size: 14, class: "ck-concept__icon", "aria-hidden": "true" %></button><span class="ck-concept__pop" role="tooltip" id="concept-<%= key %>-pop"><span class="ck-concept__name"><%= concept[:name] %></span><span class="ck-concept__body"><%= concept[:definition] %></span></span></span>
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
<body class="ck-app">
|
|
16
16
|
<header class="ck-topbar">
|
|
17
17
|
<div class="ck-wrap ck-topbar__inner">
|
|
18
|
-
<%= link_to
|
|
18
|
+
<%= link_to dashboard_path, class: "ck-brand" do %><%= image_tag "completion_kit/logo.png", alt: "CompletionKit", style: "height: 64px; width: auto;" %><span class="ck-brand__name">Completion<span class="ck-brand__kit">Kit</span></span><% end %>
|
|
19
19
|
|
|
20
20
|
<details class="ck-nav-menu">
|
|
21
21
|
<summary class="ck-nav-menu__trigger" aria-label="Menu">
|
data/config/routes.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: completion-kit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.31
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Damien Bastin
|
|
@@ -242,6 +242,7 @@ files:
|
|
|
242
242
|
- app/controllers/completion_kit/api/v1/tags_controller.rb
|
|
243
243
|
- app/controllers/completion_kit/api_reference_controller.rb
|
|
244
244
|
- app/controllers/completion_kit/application_controller.rb
|
|
245
|
+
- app/controllers/completion_kit/dashboard_controller.rb
|
|
245
246
|
- app/controllers/completion_kit/dashboard_dismissals_controller.rb
|
|
246
247
|
- app/controllers/completion_kit/datasets_controller.rb
|
|
247
248
|
- app/controllers/completion_kit/mcp_controller.rb
|
|
@@ -299,6 +300,7 @@ files:
|
|
|
299
300
|
- app/services/completion_kit/model_discovery_service.rb
|
|
300
301
|
- app/services/completion_kit/ollama_client.rb
|
|
301
302
|
- app/services/completion_kit/onboarding/checklist.rb
|
|
303
|
+
- app/services/completion_kit/onboarding/concepts.rb
|
|
302
304
|
- app/services/completion_kit/onboarding/sample_data.rb
|
|
303
305
|
- app/services/completion_kit/open_ai_client.rb
|
|
304
306
|
- app/services/completion_kit/open_router_client.rb
|
|
@@ -315,6 +317,7 @@ files:
|
|
|
315
317
|
- app/views/completion_kit/dashboard/_eye_off_icon.html.erb
|
|
316
318
|
- app/views/completion_kit/dashboard/_failures_card.html.erb
|
|
317
319
|
- app/views/completion_kit/dashboard/_worst_metric_card.html.erb
|
|
320
|
+
- app/views/completion_kit/dashboard/show.html.erb
|
|
318
321
|
- app/views/completion_kit/dashboard_dismissals/refresh.turbo_stream.erb
|
|
319
322
|
- app/views/completion_kit/datasets/_form.html.erb
|
|
320
323
|
- app/views/completion_kit/datasets/edit.html.erb
|
|
@@ -331,6 +334,7 @@ files:
|
|
|
331
334
|
- app/views/completion_kit/metrics/index.html.erb
|
|
332
335
|
- app/views/completion_kit/metrics/new.html.erb
|
|
333
336
|
- app/views/completion_kit/metrics/show.html.erb
|
|
337
|
+
- app/views/completion_kit/onboarding/_concept.html.erb
|
|
334
338
|
- app/views/completion_kit/onboarding/show.html.erb
|
|
335
339
|
- app/views/completion_kit/prompts/_form.html.erb
|
|
336
340
|
- app/views/completion_kit/prompts/edit.html.erb
|