completion-kit 0.5.3 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f64415a327bca168f071390f5b44188e5fb3b7f66a9b92f5a0d6e03da1977454
-  data.tar.gz: a64a0446f6b6ee0d10e076e10405be75cb298377e8d4a45f921fe3ff8d2946d6
+  metadata.gz: 89a2dd49e9edb75dada51386769cfd7e16f596c3b30c434588864487ab801d5b
+  data.tar.gz: df8d9acfd4ef8aeab73508e7cf9adb2c083c7b9ef3d1c4304e1a3e01723b17e1
 SHA512:
-  metadata.gz: 02c394c4216d8fcefa8a585e4c0fdab80a64d63cd865664c24c99047065bb371dc9f4c490340c3da825331549fcbbee873b93547afc5b655e4138d938a39731c
-  data.tar.gz: d04afd98a0b25bfdc500b1826ec71dd3f055a106eec7528f4deff615c95c0ba3d76517320869e023739e7e43ff59f92ebaa754a3fc10446e42985dcb21124ae5
+  metadata.gz: 9b03af38565cb43b7a57747896136817d954b06a89ba4551afb314ca93cd56a70e9d3068271d4f522741efa7d830b106a3a39e05997ea38fc3fb65996741ad08
+  data.tar.gz: be0dad1d98c850854770351edbfdcd7867fbda7b6590634bae2997707a136a94cee066c79fd2ae8a259ea9c01db06a89b7088012a072655c7c627f7c475a801a

data/app/assets/stylesheets/completion_kit/application.css

@@ -494,10 +494,18 @@ tr:hover .ck-chip--publish {
   letter-spacing: 0.04em;
   text-transform: uppercase;
   line-height: 1;
+  white-space: nowrap;
   cursor: pointer;
   transition: all 0.15s ease;
 }
 
+.ck-magic-icon {
+  width: 1.1em;
+  height: 1.1em;
+  flex-shrink: 0;
+  color: var(--ck-warning);
+}
+
 .ck-button:hover {
   transform: translateY(-1px);
 }
@@ -1140,6 +1148,22 @@ tr:hover .ck-chip--publish {
   font-size: 0.85rem;
 }
 
+.ck-model-table__unknown {
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  width: 1.25rem;
+  height: 1.25rem;
+  border-radius: 50%;
+  background: rgba(148, 163, 184, 0.12);
+  color: var(--ck-muted);
+  font-family: var(--ck-mono);
+  font-size: 0.78rem;
+  font-weight: 700;
+  line-height: 1;
+  cursor: help;
+}
+
 .ck-model-list__summary {
   display: flex;
   align-items: center;
@@ -3779,3 +3803,354 @@ a.tag-mark {
 .ck-tag-filter__clear:hover { color: var(--ck-accent); }
 
 
+/* --- Onboarding / launch checklist --- */
+@keyframes ck-launch-rise {
+  from { opacity: 0; transform: translateY(14px); }
+  to   { opacity: 1; transform: translateY(0); }
+}
+
+.ck-launch {
+  position: relative;
+  max-width: 720px;
+  margin: 1rem auto 4rem;
+  padding-top: 1rem;
+}
+
+.ck-launch__field {
+  position: absolute;
+  inset: -2rem -3rem auto -3rem;
+  height: 300px;
+  pointer-events: none;
+  z-index: 0;
+  background:
+    radial-gradient(ellipse 70% 80% at 50% 0%, var(--ck-accent-soft) 0%, transparent 65%),
+    repeating-linear-gradient(90deg, transparent 0 23px, rgba(148, 163, 184, 0.035) 23px 24px),
+    repeating-linear-gradient(0deg,  transparent 0 23px, rgba(148, 163, 184, 0.035) 23px 24px);
+  -webkit-mask-image: linear-gradient(to bottom, #000 0%, transparent 100%);
+          mask-image: linear-gradient(to bottom, #000 0%, transparent 100%);
+}
+
+.ck-launch__header {
+  position: relative;
+  z-index: 1;
+  text-align: center;
+  margin-bottom: 2.25rem;
+  animation: ck-launch-rise 0.5s ease both;
+}
+
+.ck-launch__overline {
+  margin: 0 0 0.6rem;
+  font-family: var(--ck-mono);
+  font-size: 0.72rem;
+  letter-spacing: 0.32em;
+  text-transform: uppercase;
+  color: var(--ck-accent);
+}
+
+.ck-launch__org {
+  margin: 0;
+  font-family: var(--ck-mono);
+  font-size: clamp(1.8rem, 4vw, 2.6rem);
+  font-weight: 600;
+  letter-spacing: -0.02em;
+  color: var(--ck-text);
+}
+
+.ck-launch__handle {
+  margin: 0.5rem 0 0;
+  font-family: var(--ck-mono);
+  font-size: 0.8rem;
+  letter-spacing: 0.04em;
+  color: var(--ck-muted);
+}
+
+.ck-launch__panel {
+  position: relative;
+  z-index: 1;
+  background: var(--ck-surface);
+  border: 1px solid var(--ck-line);
+  border-radius: var(--ck-radius-lg);
+  padding: 1.75rem 2rem 2rem;
+  box-shadow: 0 24px 64px -28px rgba(0, 0, 0, 0.5);
+  animation: ck-launch-rise 0.5s ease both;
+  animation-delay: 0.08s;
+}
+
+.ck-launch__progress {
+  padding-bottom: 1.5rem;
+  margin-bottom: 0.5rem;
+  border-bottom: 1px solid var(--ck-line);
+}
+.ck-launch__progress-head {
+  display: flex;
+  align-items: baseline;
+  justify-content: space-between;
+  gap: 1rem;
+  margin-bottom: 0.85rem;
+}
+.ck-launch__progress-title {
+  margin: 0;
+  font-family: var(--ck-mono);
+  font-size: 1.05rem;
+  font-weight: 600;
+  letter-spacing: -0.02em;
+  color: var(--ck-text);
+}
+.ck-launch__progress-count {
+  margin: 0;
+  font-family: var(--ck-mono);
+  font-size: 0.78rem;
+  letter-spacing: 0.02em;
+  color: var(--ck-muted);
+  white-space: nowrap;
+}
+.ck-launch__progress-bar {
+  height: 5px;
+  border-radius: 999px;
+  background: var(--ck-line);
+  overflow: hidden;
+}
+.ck-launch__progress-fill {
+  display: block;
+  height: 100%;
+  border-radius: 999px;
+  background: var(--ck-accent);
+  box-shadow: 0 0 10px var(--ck-accent-soft);
+  transition: width 0.4s ease;
+}
+
+.ck-launch__steps {
+  list-style: none;
+  margin: 0;
+  padding: 0;
+  display: flex;
+  flex-direction: column;
+}
+
+.ck-launch__step {
+  display: grid;
+  grid-template-columns: 2.4rem 1fr;
+  gap: 1rem;
+  align-items: baseline;
+  padding: 1.25rem 0 1.25rem 1rem;
+  border-top: 1px solid var(--ck-line);
+  border-left: 2px solid transparent;
+  animation: ck-launch-rise 0.45s ease both;
+  animation-delay: calc(0.24s + var(--i) * 0.07s);
+}
+
+.ck-launch__step-num {
+  font-family: var(--ck-mono);
+  font-size: 0.8rem;
+  font-weight: 600;
+  letter-spacing: 0.04em;
+  color: var(--ck-dim);
+  user-select: none;
+  white-space: nowrap;
+}
+.ck-launch__step--done .ck-launch__step-num { color: var(--ck-success); }
+.ck-launch__step--next .ck-launch__step-num { color: var(--ck-accent); }
+.ck-launch__step-tick { margin-left: 0.3em; font-size: 0.78em; }
+
+.ck-launch__step--done { border-left-color: rgba(45, 212, 168, 0.4); }
+
+.ck-launch__step-body { min-width: 0; }
+
+.ck-launch__step-head {
+  display: flex;
+  align-items: baseline;
+  gap: 0.7rem;
+  flex-wrap: wrap;
+  margin-bottom: 0.3rem;
+}
+
+.ck-launch__step-title {
+  margin: 0;
+  font-size: 1rem;
+  font-weight: 500;
+  letter-spacing: -0.005em;
+}
+.ck-launch__step-title a { color: var(--ck-text); text-decoration: none; }
+.ck-launch__step-title a:hover { color: var(--ck-accent); }
+.ck-launch__step--done .ck-launch__step-title { color: var(--ck-muted); }
+
+.ck-launch__step-tag {
+  font-family: var(--ck-mono);
+  font-size: 0.6rem;
+  letter-spacing: 0.18em;
+  text-transform: uppercase;
+  color: var(--ck-accent);
+  padding: 0.12rem 0.45rem;
+  border: 1px solid var(--ck-accent);
+  border-radius: 999px;
+  line-height: 1.4;
+}
+
+.ck-launch__step-desc {
+  margin: 0;
+  font-size: 0.9rem;
+  line-height: 1.55;
+  color: var(--ck-muted);
+}
+
+.ck-launch__step--pending .ck-launch__step-title,
+.ck-launch__step--pending .ck-launch__step-desc { opacity: 0.65; }
+
+.ck-launch__step--next {
+  border-left-color: var(--ck-accent);
+  background: linear-gradient(90deg, var(--ck-accent-soft) 0%, transparent 30%);
+  margin-left: -1rem;
+  padding-left: 2rem;
+  padding-right: 1rem;
+  border-radius: 0 var(--ck-radius) var(--ck-radius) 0;
+}
+
+.ck-launch__step-cta {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.4rem;
+  margin-top: 0.85rem;
+  padding: 0.5rem 0.95rem;
+  background: var(--ck-accent);
+  color: #06121a;
+  border-radius: var(--ck-radius);
+  font-family: var(--ck-mono);
+  font-size: 0.75rem;
+  letter-spacing: 0.06em;
+  text-transform: uppercase;
+  font-weight: 600;
+  text-decoration: none;
+  transition: filter 0.15s, transform 0.15s;
+}
+.ck-launch__step-cta:hover { filter: brightness(1.1); transform: translateY(-1px); }
+
+/* Opt-in starter data — quiet, secondary (bright cyan is reserved for the next-step CTA) */
+.ck-launch__sample {
+  margin-top: 1.5rem;
+  padding: 1rem 1.1rem;
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  justify-content: space-between;
+  gap: 0.85rem 1.25rem;
+  border: 1px dashed var(--ck-line);
+  border-radius: var(--ck-radius);
+  background: var(--ck-bg-strong);
+}
+.ck-launch__sample-copy {
+  margin: 0;
+  flex: 1 1 18rem;
+  font-size: 0.85rem;
+  line-height: 1.55;
+  color: var(--ck-muted);
+}
+.ck-launch__sample-copy code { font-family: var(--ck-mono); font-size: 0.85em; color: var(--ck-text); }
+.ck-launch__sample-cta {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.4rem;
+  flex-shrink: 0;
+  margin: 0;
+  padding: 0.5rem 0.95rem;
+  background: transparent;
+  border: 1px solid var(--ck-line);
+  color: var(--ck-text);
+  border-radius: var(--ck-radius);
+  font-family: var(--ck-mono);
+  font-size: 0.72rem;
+  letter-spacing: 0.06em;
+  text-transform: uppercase;
+  font-weight: 600;
+  cursor: pointer;
+  transition: border-color 0.15s, color 0.15s, background 0.15s;
+}
+.ck-launch__sample-cta:hover { border-color: var(--ck-accent); color: var(--ck-accent); background: var(--ck-accent-soft); }
+
+.ck-launch__panel-footer {
+  margin-top: 1.75rem;
+  padding-top: 1.25rem;
+  border-top: 1px solid var(--ck-line);
+}
+.ck-launch__dismiss {
+  margin: 0;
+  padding: 0;
+  background: transparent;
+  border: none;
+  color: var(--ck-muted);
+  font-family: var(--ck-mono);
+  font-size: 0.78rem;
+  letter-spacing: 0.04em;
+  cursor: pointer;
+  text-decoration: underline;
+  text-underline-offset: 3px;
+  transition: color 0.15s;
+}
+.ck-launch__dismiss:hover { color: var(--ck-accent); }
+.ck-launch__dismiss-hint {
+  margin: 0.5rem 0 0;
+  font-size: 0.78rem;
+  color: var(--ck-dim);
+}
+
+.ck-launch__ready-panel {
+  position: relative;
+  z-index: 1;
+  background: var(--ck-surface);
+  border: 1px solid var(--ck-line);
+  border-radius: var(--ck-radius-lg);
+  padding: 2rem;
+  text-align: center;
+  animation: ck-launch-rise 0.5s ease both;
+  animation-delay: 0.08s;
+}
+
+.ck-launch__ready-flag {
+  margin: 0;
+  font-family: var(--ck-mono);
+  font-size: 0.85rem;
+  letter-spacing: 0.14em;
+  text-transform: uppercase;
+  color: var(--ck-success);
+  display: inline-flex;
+  align-items: center;
+  gap: 0.6rem;
+}
+.ck-launch__ready-dot {
+  width: 8px;
+  height: 8px;
+  border-radius: 50%;
+  background: var(--ck-success);
+  box-shadow: 0 0 12px var(--ck-success);
+}
+.ck-launch__ready-copy { margin: 0.6rem 0 1.5rem; color: var(--ck-muted); }
+
+.ck-launch__quicklinks {
+  display: flex;
+  gap: 0.6rem;
+  justify-content: center;
+  flex-wrap: wrap;
+}
+.ck-launch__quicklink {
+  padding: 0.5rem 0.95rem;
+  border: 1px solid var(--ck-line-strong);
+  border-radius: var(--ck-radius);
+  font-family: var(--ck-mono);
+  font-size: 0.75rem;
+  letter-spacing: 0.06em;
+  text-transform: uppercase;
+  color: var(--ck-text);
+  text-decoration: none;
+  transition: border-color 0.15s, background 0.15s;
+}
+.ck-launch__quicklink:hover { border-color: var(--ck-accent); background: var(--ck-surface-hover); }
+
+@media (max-width: 560px) {
+  .ck-launch__panel { padding: 1.5rem 1.25rem; }
+}
+
+@media (prefers-reduced-motion: reduce) {
+  .ck-launch__header,
+  .ck-launch__panel,
+  .ck-launch__ready-panel,
+  .ck-launch__step { animation: none; }
+}

data/app/controllers/completion_kit/onboarding_controller.rb

@@ -0,0 +1,23 @@
+module CompletionKit
+  class OnboardingController < ApplicationController
+    DISMISS_COOKIE = :ck_onboarding_dismissed
+
+    def show
+      cookies.delete(DISMISS_COOKIE) if params[:reset]
+      @checklist = Onboarding::Checklist.new
+      return if params[:reset]
+
+      redirect_to prompts_path if @checklist.complete? || cookies[DISMISS_COOKIE]
+    end
+
+    def dismiss
+      cookies[DISMISS_COOKIE] = { value: "1", expires: 1.year.from_now, httponly: true }
+      redirect_to prompts_path, notice: "Setup skipped. Pick it back up from Settings → Getting started any time."
+    end
+
+    def sample_data
+      Onboarding::SampleData.install!
+      redirect_to onboarding_path, notice: "Loaded a sample dataset and prompt — edit or delete them whenever."
+    end
+  end
+end

data/app/helpers/completion_kit/application_helper.rb

@@ -72,6 +72,11 @@ module CompletionKit
       CompletionKit::ProviderCredential::PROVIDER_LABELS[provider.to_s] || provider.to_s.titleize
     end
 
+    def ck_model_option_label(model)
+      return "#{model[:name]} (?)" if model.key?(:judging_confirmed) && !model[:judging_confirmed]
+      model[:name]
+    end
+
     def ck_grouped_models(models, selected = nil)
       if selected.present? && models.none? { |m| m[:id] == selected }
         retired = CompletionKit::Model.find_by(model_id: selected)
@@ -90,7 +95,7 @@ module CompletionKit
       end
 
       ordered_keys = groups.keys.sort_by { |label| [label.start_with?("OpenRouter") ? 1 : 0, label] }
-      grouped = ordered_keys.map { |label| [label, groups[label].map { |m| [m[:name], m[:id]] }] }
+      grouped = ordered_keys.map { |label| [label, groups[label].map { |m| [ck_model_option_label(m), m[:id]] }] }
       grouped_options_for_select(grouped, selected)
     end
 

data/app/jobs/completion_kit/judge_review_job.rb

@@ -71,6 +71,7 @@ module CompletionKit
       )
       review.save!
 
+      confirm_judging_capability(run.judge_model)
       run.send(:broadcast_response_update, response)
       run.send(:broadcast_progress)
       enqueue_completion_check
@@ -78,6 +79,15 @@ module CompletionKit
 
     private
 
+    # A model with supports_judging == nil ("untested") just produced a valid
+    # review — promote it to confirmed. No-op once confirmed (so repeated runs
+    # don't churn the row), and a model already flagged as a bad judge stays so.
+    def confirm_judging_capability(judge_model_id)
+      model = Model.find_by(provider: ApiConfig.provider_for_model(judge_model_id), model_id: judge_model_id)
+      return unless model && model.supports_judging.nil?
+      model.update_columns(supports_judging: true, judging_error: nil)
+    end
+
     def record_terminal_failure!(error)
       response_id = @response_id || arguments.first
       metric_id = @metric_id || arguments.last

data/app/models/completion_kit/model.rb

@@ -8,6 +8,9 @@ module CompletionKit
 
     scope :active, -> { where(status: "active") }
     scope :for_generation, -> { active.where(supports_generation: true) }
-    scope :for_judging, -> { active.where(supports_judging: true) }
+    # Includes models not yet confirmed as judges (supports_judging: nil) — worth
+    # a try, and a successful run flips them to confirmed. Only models known to be
+    # bad judges (false) are excluded.
+    scope :for_judging, -> { active.where(supports_judging: [true, nil]) }
   end
 end

data/app/services/completion_kit/api_config.rb

@@ -61,7 +61,9 @@ module CompletionKit
               end
       query = query.where(provider: provider) if provider.present?
       models = query.order(:provider, :display_name).map do |m|
-        { id: m.model_id, name: m.display_name || m.model_id, provider: m.provider }
+        entry = { id: m.model_id, name: m.display_name || m.model_id, provider: m.provider }
+        entry[:judging_confirmed] = !m.supports_judging.nil? if scope == :judging
+        entry
       end
 
       return models if models.any?

data/app/services/completion_kit/model_discovery_service.rb

@@ -13,8 +13,13 @@ module CompletionKit
     end
 
     def refresh!(&on_progress)
-      models_with_names = fetch_models
-      reconcile(models_with_names)
+      discovered = fetch_models
+      reconcile(discovered)
+      # OpenRouter publishes capability metadata (output modalities, etc.), so we
+      # derive everything from the model list and skip live probing entirely.
+      # Judging stays unknown ("?") until a real run proves it.
+      return if @provider == "openrouter"
+
       probe_new_models(&on_progress)
     end
 
@@ -86,10 +91,19 @@ module CompletionKit
         next nil if entry["deprecated"] == true
         context_length = entry["context_length"].to_i
         next nil if context_length < 8192
-        { id: entry["id"], display_name: entry["name"] }
+        { id: entry["id"], display_name: entry["name"], supports_generation: openrouter_text_output?(entry) }
       end
     end
 
+    # OpenRouter exposes architecture.output_modalities (e.g. ["text"], ["image"],
+    # ["text", "image"]). A model can be used for generation/judging only if it
+    # outputs text. When the field is missing we keep the historical default of
+    # treating the model as text-capable.
+    def openrouter_text_output?(entry)
+      modalities = Array(entry.dig("architecture", "output_modalities")).map(&:to_s)
+      modalities.empty? || modalities.include?("text")
+    end
+
     def fetch_ollama_models
       raise DiscoveryError, "Ollama endpoint URL is required" if @api_endpoint.blank?
       base_url = @api_endpoint.to_s.delete_suffix("/")
@@ -100,35 +114,67 @@ module CompletionKit
       JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: e["id"] } }
     end
 
-    def reconcile(models_with_names)
-      api_model_ids = models_with_names.map { |m| m[:id] }
-      names_by_id = models_with_names.each_with_object({}) { |m, h| h[m[:id]] = m[:display_name] }
+    def reconcile(discovered)
+      api_model_ids = discovered.map { |m| m[:id] }
+      meta_by_id = discovered.index_by { |m| m[:id] }
       existing = Model.where(provider: @provider).index_by(&:model_id)
 
       api_model_ids.each do |model_id|
-        if existing[model_id]
-          attrs = { status: "active", retired_at: nil }
-          attrs[:display_name] = names_by_id[model_id] if names_by_id[model_id].present?
-          existing[model_id].update!(attrs) if existing[model_id].status == "retired" || names_by_id[model_id].present?
+        meta = meta_by_id[model_id]
+        if (model = existing[model_id])
+          reconcile_existing_model(model, meta)
         else
-          attrs = {
-            provider: @provider,
-            model_id: model_id,
-            display_name: names_by_id[model_id],
-            status: "active",
-            discovered_at: Time.current
-          }
-          if %w[openrouter ollama].include?(@provider)
-            attrs[:supports_generation] = true
-            attrs[:probed_at] = nil
-          end
-          Model.create!(attrs)
+          Model.create!(new_model_attrs(model_id, meta))
         end
       end
 
-      active_not_in_api = Model.where(provider: @provider, status: "active")
-                               .where.not(model_id: api_model_ids)
-      active_not_in_api.update_all(status: "retired", retired_at: Time.current)
+      Model.where(provider: @provider, status: "active")
+           .where.not(model_id: api_model_ids)
+           .update_all(status: "retired", retired_at: Time.current)
+    end
+
+    def new_model_attrs(model_id, meta)
+      attrs = {
+        provider: @provider,
+        model_id: model_id,
+        display_name: meta[:display_name],
+        status: "active",
+        discovered_at: Time.current
+      }
+      if @provider == "openrouter"
+        supports_generation = meta[:supports_generation] != false
+        attrs.merge!(
+          supports_generation: supports_generation,
+          supports_judging: nil,
+          probed_at: Time.current,
+          status: supports_generation ? "active" : "failed"
+        )
+      elsif @provider == "ollama"
+        attrs[:supports_generation] = true
+        attrs[:probed_at] = nil
+      end
+      attrs
+    end
+
+    def reconcile_existing_model(model, meta)
+      if @provider == "openrouter"
+        # Re-derive generation capability from the published metadata every refresh
+        # (fixes models discovered before capability metadata was used). Leave
+        # supports_judging alone — it's "learned" from successful runs.
+        supports_generation = meta[:supports_generation] != false
+        model.update!(
+          display_name: meta[:display_name].presence || model.display_name,
+          supports_generation: supports_generation,
+          generation_error: nil,
+          probed_at: Time.current,
+          status: supports_generation ? "active" : "failed",
+          retired_at: nil
+        )
+      else
+        attrs = { status: "active", retired_at: nil }
+        attrs[:display_name] = meta[:display_name] if meta[:display_name].present?
+        model.update!(attrs) if model.status == "retired" || meta[:display_name].present?
+      end
     end
 
     def probe_new_models(&on_progress)
@@ -223,7 +269,6 @@ module CompletionKit
       case @provider
       when "openai" then openai_probe(model_id, input, max_tokens)
       when "anthropic" then anthropic_probe(model_id, input, max_tokens)
-      when "openrouter" then openrouter_probe(model_id, input, max_tokens)
       when "ollama" then ollama_probe(model_id, input, max_tokens)
       else raise ArgumentError, "Unsupported probe provider: #{@provider}"
       end
@@ -290,23 +335,6 @@ module CompletionKit
       end
     end
 
-    def openrouter_probe(model_id, input, max_tokens)
-      conn = Faraday.new(url: "https://openrouter.ai") do |f|
-        f.options.timeout = 30
-        f.options.open_timeout = 5
-        f.request :retry, max: 1, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-      conn.post do |req|
-        req.url "/api/v1/chat/completions"
-        req.headers["Content-Type"] = "application/json"
-        req.headers["Authorization"] = "Bearer #{@api_key}"
-        req.headers["HTTP-Referer"] = "https://completionkit.com"
-        req.headers["X-Title"] = "CompletionKit"
-        req.body = { model: model_id, messages: [{ role: "user", content: input }], max_tokens: max_tokens }.to_json
-      end
-    end
-
     def ollama_probe(model_id, input, max_tokens)
       base_url = @api_endpoint.to_s.delete_suffix("/")
       conn = Faraday.new(url: base_url) do |f|

data/app/services/completion_kit/onboarding/checklist.rb

@@ -0,0 +1,66 @@
+module CompletionKit
+  module Onboarding
+    Step = Data.define(:key, :title, :description, :path_name, :done?)
+
+    class Checklist
+      STEP_DEFS = [
+        {
+          key: :credential,
+          title: "Connect a provider",
+          description: "Add an API key for OpenAI, Anthropic, or another model provider so you can run prompts and score outputs.",
+          path_name: :new_provider_credential_path,
+          done: -> { CompletionKit::ProviderCredential.exists? }
+        },
+        {
+          key: :dataset,
+          title: "Upload a dataset",
+          description: "Drop in a CSV of test inputs. Column headers become the {{variables}} your prompts get evaluated against.",
+          path_name: :new_dataset_path,
+          done: -> { CompletionKit::Dataset.exists? }
+        },
+        {
+          key: :prompt,
+          title: "Write your first prompt",
+          description: "Create a versioned prompt template with {{variable}} placeholders. Publishing freezes it; editing makes a new version.",
+          path_name: :new_prompt_path,
+          done: -> { CompletionKit::Prompt.exists? }
+        },
+        {
+          key: :run,
+          title: "Run it",
+          description: "Kick off a run against the dataset and watch the responses come in, each scored by the LLM judge against your metrics.",
+          path_name: :runs_path,
+          done: -> { CompletionKit::Run.exists? }
+        }
+      ].freeze
+
+      def steps
+        @steps ||= STEP_DEFS.map do |d|
+          Step.new(
+            key: d[:key],
+            title: d[:title],
+            description: d[:description],
+            path_name: d[:path_name],
+            done?: d[:done].call
+          )
+        end
+      end
+
+      def complete?
+        steps.all?(&:done?)
+      end
+
+      # Whether the "Load sample data" button should show — only while neither
+      # the dataset nor the prompt step is done (SampleData.install! no-ops
+      # otherwise, so the button would do nothing).
+      def sample_loadable?
+        steps.none? { |s| %i[dataset prompt].include?(s.key) && s.done? }
+      end
+
+      def progress
+        done = steps.count(&:done?)
+        { done: done, total: steps.size, percent: ((done.to_f / steps.size) * 100).round }
+      end
+    end
+  end
+end

data/app/services/completion_kit/onboarding/sample_data.rb

@@ -0,0 +1,37 @@
+module CompletionKit
+  module Onboarding
+    # Opt-in starter data for the onboarding page: one dataset + one prompt so a
+    # brand-new install has something to poke at. Idempotent — a no-op once the
+    # workspace already has any prompt or dataset. Deliberately does NOT create a
+    # provider credential (needs a real API key) or a run (user-initiated).
+    module SampleData
+      SAMPLE_CSV = <<~CSV.freeze
+        ticket
+        "My order #4827 arrived with a dented panel. I emailed photos 11 days ago and heard nothing. Today I was told the return window 'closed'. I paid $749. I want a refund or replacement, not store credit."
+        "Tracking says delivered to my porch Tuesday 3:47pm. I was home all day, nothing arrived, neighbours' cameras show no van. Order #5102 — a $315 mixer, wedding gift, wedding is Saturday. Can someone look today?"
+        "WELCOME20 says 'invalid' at checkout but the promo email says it's good through May 31. Same email I'm signed in with. Tried Chrome and Safari. Cart is $186 waiting on you."
+      CSV
+
+      SAMPLE_PROMPT = {
+        name: "Sample: Support reply",
+        description: "A starter prompt — drafts a warm, professional reply to a customer support ticket. Edit it or delete it; it's just here to get you going.",
+        template: "You are a senior customer-support specialist. Write a warm, professional reply to this ticket. Acknowledge the customer's situation, be specific about next steps, and don't be defensive.\n\nTicket:\n{{ticket}}",
+        llm_model: "gpt-4o-mini"
+      }.freeze
+
+      module_function
+
+      def install!
+        return if CompletionKit::Prompt.exists? || CompletionKit::Dataset.exists?
+
+        CompletionKit::Dataset.create!(name: "Sample: Customer tickets", csv_data: SAMPLE_CSV)
+        CompletionKit::Prompt.create!(
+          name: SAMPLE_PROMPT[:name],
+          description: SAMPLE_PROMPT[:description],
+          template: SAMPLE_PROMPT[:template],
+          llm_model: SAMPLE_PROMPT[:llm_model]
+        )
+      end
+    end
+  end
+end

data/app/views/completion_kit/onboarding/show.html.erb

@@ -0,0 +1,89 @@
+<div class="ck-launch <%= "ck-launch--ready" if @checklist.complete? %>">
+  <div class="ck-launch__field" aria-hidden="true"></div>
+
+  <header class="ck-launch__header">
+    <p class="ck-launch__overline">Welcome to CompletionKit</p>
+    <h1 class="ck-launch__org">Your prompts need tests too</h1>
+    <p class="ck-launch__handle">Four steps from a fresh install to a scored test run.</p>
+  </header>
+
+  <% if @checklist.complete? %>
+    <section class="ck-launch__ready-panel">
+      <p class="ck-launch__ready-flag">
+        <span class="ck-launch__ready-dot" aria-hidden="true"></span>
+        You're all set up
+      </p>
+      <p class="ck-launch__ready-copy">Jump in wherever you like.</p>
+      <div class="ck-launch__quicklinks">
+        <%= link_to "Prompts",  prompts_path,  class: "ck-launch__quicklink" %>
+        <%= link_to "Datasets", datasets_path, class: "ck-launch__quicklink" %>
+        <%= link_to "Metrics",  metrics_path,  class: "ck-launch__quicklink" %>
+        <%= link_to "Runs",     runs_path,     class: "ck-launch__quicklink" %>
+      </div>
+    </section>
+  <% else %>
+    <% next_index = @checklist.steps.index { |s| !s.done? } %>
+    <% progress = @checklist.progress %>
+    <section class="ck-launch__panel">
+      <div class="ck-launch__progress">
+        <div class="ck-launch__progress-head">
+          <h2 class="ck-launch__progress-title">Finish setting up</h2>
+          <p class="ck-launch__progress-count"><%= progress[:done] %> of <%= progress[:total] %> done</p>
+        </div>
+        <div class="ck-launch__progress-bar" role="img" aria-label="<%= progress[:done] %> of <%= progress[:total] %> setup steps complete">
+          <span class="ck-launch__progress-fill" style="width: <%= progress[:percent] %>%"></span>
+        </div>
+      </div>
+
+      <ol class="ck-launch__steps">
+        <% @checklist.steps.each_with_index do |step, i| %>
+          <%
+            state = if step.done?
+                      "done"
+                    elsif i == next_index
+                      "next"
+                    else
+                      "pending"
+                    end
+            step_path = public_send(step.path_name)
+          %>
+          <li class="ck-launch__step ck-launch__step--<%= state %>" style="--i: <%= i %>">
+            <span class="ck-launch__step-num" aria-hidden="true"><%= format("%02d", i + 1) %><% if step.done? %><span class="ck-launch__step-tick">✓</span><% end %></span>
+            <div class="ck-launch__step-body">
+              <div class="ck-launch__step-head">
+                <h3 class="ck-launch__step-title">
+                  <% if step.done? %>
+                    <%= step.title %>
+                  <% else %>
+                    <%= link_to step.title, step_path %>
+                  <% end %>
+                </h3>
+                <% if state == "next" %><span class="ck-launch__step-tag">Next up</span><% end %>
+              </div>
+              <p class="ck-launch__step-desc"><%= step.description %></p>
+              <% if state == "next" %>
+                <%= link_to step_path, class: "ck-launch__step-cta" do %>Start <span aria-hidden="true">&rarr;</span><% end %>
+              <% end %>
+            </div>
+          </li>
+        <% end %>
+      </ol>
+
+      <% if @checklist.sample_loadable? %>
+        <div class="ck-launch__sample">
+          <p class="ck-launch__sample-copy">Just exploring? Drop in a sample dataset and prompt to poke around — they're labelled <code>Sample:</code>, and you can edit or delete them whenever.</p>
+          <%= button_to onboarding_sample_data_path, method: :post, class: "ck-launch__sample-cta", form: { style: "display:inline" } do %>
+            Load sample data <span aria-hidden="true">&rarr;</span>
+          <% end %>
+        </div>
+      <% end %>
+
+      <div class="ck-launch__panel-footer">
+        <%= button_to dismiss_onboarding_path, method: :post, class: "ck-launch__dismiss", form: { style: "display:inline" } do %>
+          Skip setup &mdash; go to the app <span aria-hidden="true">&rarr;</span>
+        <% end %>
+        <p class="ck-launch__dismiss-hint">You can come back to this from <strong>Settings &rarr; Getting started</strong> any time.</p>
+      </div>
+    </section>
+  <% end %>
+</div>

data/app/views/completion_kit/prompts/show.html.erb

@@ -37,7 +37,7 @@
     <% judged_run = @runs.select { |r| r.prompt_id == @prompt.id && r.status == "completed" }.find { |r| r.responses.joins(:reviews).exists? } %>
     <% if judged_run %>
       <%= button_to suggest_run_path(judged_run), method: :post, class: ck_button_classes(:light, variant: :outline) + " ck-button--sm", form_class: "inline-block" do %>
-        <%= heroicon_tag "sparkles", variant: :outline, size: 14, "aria-hidden": "true" %>
+        <%= heroicon_tag "sparkles", variant: :outline, class: "ck-magic-icon", "aria-hidden": "true" %>
         Suggest improvements
       <% end %>
     <% else %>

data/app/views/completion_kit/provider_credentials/_models_card.html.erb

@@ -36,7 +36,7 @@
                     Gen<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup ck-info-popup--right">Generation models produce the responses your prompts ask for. Pick one when creating a prompt.</span>
                   </th>
                   <th scope="col" class="ck-model-table__cap">
-                    Judge<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup ck-info-popup--right">Judge models score generated responses against your metrics. Pick one when configuring a run.</span>
+                    Judge<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup ck-info-popup--right">Judge models score generated responses against your metrics. Pick one when configuring a run. A <strong>?</strong> means we haven't confirmed this model works as a judge — it's still selectable, and a successful run promotes it to ✓.</span>
                   </th>
                 </tr>
               </thead>
@@ -54,8 +54,10 @@
                     <td class="ck-model-table__cap">
                       <% if m.supports_judging %>
                         <span class="ck-model-table__tick" aria-label="Supports judging">✓</span>
+                      <% elsif m.supports_judging.nil? %>
+                        <span class="ck-model-table__unknown" aria-label="Untested as judge" title="Untested as a judge — selectable; a successful run confirms it">?</span>
                       <% else %>
-                        <span class="ck-model-table__dash" aria-label="No judging support">—</span>
+                        <span class="ck-model-table__dash" aria-label="Not usable as judge">—</span>
                       <% end %>
                     </td>
                   </tr>

data/app/views/completion_kit/runs/show.html.erb

@@ -67,7 +67,7 @@
       <%= link_to "View suggestion", suggestion_path(latest_suggestion, from: "run"), class: ck_button_classes(:light, variant: :outline) + " ck-button--sm" %>
     <% elsif @run.status == "completed" && @run.responses.joins(:reviews).exists? %>
       <%= button_to suggest_run_path(@run), method: :post, class: ck_button_classes(:light, variant: :outline) + " ck-button--sm", form_class: "inline-block" do %>
-        <%= heroicon_tag "sparkles", variant: :outline, size: 14, "aria-hidden": "true" %>
+        <%= heroicon_tag "sparkles", variant: :outline, class: "ck-magic-icon", "aria-hidden": "true" %>
         Suggest improvements
       <% end %>
     <% end %>

data/app/views/layouts/completion_kit/application.html.erb

@@ -23,7 +23,7 @@
         <%= link_to "Metrics", metrics_path, class: request.path.start_with?(metrics_path) || request.path.start_with?(metric_groups_path) ? ck_button_classes(:dark) : ck_button_classes(:light, variant: :outline) %>
         <%= link_to "Datasets", datasets_path, class: active.(datasets_path) %>
         <%= link_to "Runs", runs_path, class: active.(runs_path) %>
-        <% settings_active = request.path.start_with?(provider_credentials_path) || request.path.start_with?(tags_path) %>
+        <% settings_active = request.path.start_with?(provider_credentials_path) || request.path.start_with?(tags_path) || request.path.start_with?(onboarding_path) %>
         <details class="ck-settings-menu">
           <summary class="<%= settings_active ? ck_button_classes(:dark) : ck_button_classes(:light, variant: :outline) %> ck-settings-menu__trigger" aria-label="Settings">
             Settings
@@ -31,6 +31,7 @@
           <div class="ck-settings-menu__panel" role="menu">
             <%= link_to "Providers", provider_credentials_path, class: "ck-settings-menu__item" %>
             <%= link_to "Tags", tags_path, class: "ck-settings-menu__item" %>
+            <%= link_to "Getting started", onboarding_path(reset: 1), class: "ck-settings-menu__item" %>
           </div>
         </details>
         <%= link_to "API", api_reference_path, class: active.(api_reference_path) %>

data/config/routes.rb

@@ -1,5 +1,9 @@
 CompletionKit::Engine.routes.draw do
-  root to: "prompts#index"
+  root to: "onboarding#show"
+
+  get "onboarding", to: "onboarding#show", as: :onboarding
+  post "onboarding/dismiss", to: "onboarding#dismiss", as: :dismiss_onboarding
+  post "onboarding/sample-data", to: "onboarding#sample_data", as: :onboarding_sample_data
 
   resources :prompts do
     member do

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.3"
+  VERSION = "0.5.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.5
 platform: ruby
 authors:
 - Damien Bastin
@@ -245,6 +245,7 @@ files:
 - app/controllers/completion_kit/mcp_controller.rb
 - app/controllers/completion_kit/metric_groups_controller.rb
 - app/controllers/completion_kit/metrics_controller.rb
+- app/controllers/completion_kit/onboarding_controller.rb
 - app/controllers/completion_kit/prompts_controller.rb
 - app/controllers/completion_kit/provider_credentials_controller.rb
 - app/controllers/completion_kit/responses_controller.rb
@@ -292,6 +293,8 @@ files:
 - app/services/completion_kit/mcp_tools/tags.rb
 - app/services/completion_kit/model_discovery_service.rb
 - app/services/completion_kit/ollama_client.rb
+- app/services/completion_kit/onboarding/checklist.rb
+- app/services/completion_kit/onboarding/sample_data.rb
 - app/services/completion_kit/open_ai_client.rb
 - app/services/completion_kit/open_router_client.rb
 - app/services/completion_kit/prompt_improvement_service.rb
@@ -314,6 +317,7 @@ files:
 - app/views/completion_kit/metrics/index.html.erb
 - app/views/completion_kit/metrics/new.html.erb
 - app/views/completion_kit/metrics/show.html.erb
+- app/views/completion_kit/onboarding/show.html.erb
 - app/views/completion_kit/prompts/_form.html.erb
 - app/views/completion_kit/prompts/edit.html.erb
 - app/views/completion_kit/prompts/index.html.erb