completion-kit 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb708ad1732ab18fac5f415ad90f53374832bd8e629ab2f1530e860a2eca6c03
-  data.tar.gz: 3e159d8c80601f791a8b5e9fcabea66c1755dfb1635d268efa104a0a17bd9eb1
+  metadata.gz: 89a2dd49e9edb75dada51386769cfd7e16f596c3b30c434588864487ab801d5b
+  data.tar.gz: df8d9acfd4ef8aeab73508e7cf9adb2c083c7b9ef3d1c4304e1a3e01723b17e1
 SHA512:
-  metadata.gz: 2d6bb436f79222b66d6f12bb0cb82642eb9e50c48704bbea76fb6b47c3c3b65f665363a905b4d8e57062445ad31bee44ba65c901de2a07d716d231fdef59e7df
-  data.tar.gz: ba8c62309efc2f19d343c6744b47e1db25c2a64e3597f19f957bdfd971dac6ee5118fd7f2017c573799f04a5423ac030627ff91ec9785d77822bc22a9ba7d27b
+  metadata.gz: 9b03af38565cb43b7a57747896136817d954b06a89ba4551afb314ca93cd56a70e9d3068271d4f522741efa7d830b106a3a39e05997ea38fc3fb65996741ad08
+  data.tar.gz: be0dad1d98c850854770351edbfdcd7867fbda7b6590634bae2997707a136a94cee066c79fd2ae8a259ea9c01db06a89b7088012a072655c7c627f7c475a801a

data/app/assets/stylesheets/completion_kit/application.css

@@ -1148,6 +1148,22 @@ tr:hover .ck-chip--publish {
   font-size: 0.85rem;
 }
 
+.ck-model-table__unknown {
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  width: 1.25rem;
+  height: 1.25rem;
+  border-radius: 50%;
+  background: rgba(148, 163, 184, 0.12);
+  color: var(--ck-muted);
+  font-family: var(--ck-mono);
+  font-size: 0.78rem;
+  font-weight: 700;
+  line-height: 1;
+  cursor: help;
+}
+
 .ck-model-list__summary {
   display: flex;
   align-items: center;
@@ -4008,6 +4024,48 @@ a.tag-mark {
 }
 .ck-launch__step-cta:hover { filter: brightness(1.1); transform: translateY(-1px); }
 
+/* Opt-in starter data — quiet, secondary (bright cyan is reserved for the next-step CTA) */
+.ck-launch__sample {
+  margin-top: 1.5rem;
+  padding: 1rem 1.1rem;
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  justify-content: space-between;
+  gap: 0.85rem 1.25rem;
+  border: 1px dashed var(--ck-line);
+  border-radius: var(--ck-radius);
+  background: var(--ck-bg-strong);
+}
+.ck-launch__sample-copy {
+  margin: 0;
+  flex: 1 1 18rem;
+  font-size: 0.85rem;
+  line-height: 1.55;
+  color: var(--ck-muted);
+}
+.ck-launch__sample-copy code { font-family: var(--ck-mono); font-size: 0.85em; color: var(--ck-text); }
+.ck-launch__sample-cta {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.4rem;
+  flex-shrink: 0;
+  margin: 0;
+  padding: 0.5rem 0.95rem;
+  background: transparent;
+  border: 1px solid var(--ck-line);
+  color: var(--ck-text);
+  border-radius: var(--ck-radius);
+  font-family: var(--ck-mono);
+  font-size: 0.72rem;
+  letter-spacing: 0.06em;
+  text-transform: uppercase;
+  font-weight: 600;
+  cursor: pointer;
+  transition: border-color 0.15s, color 0.15s, background 0.15s;
+}
+.ck-launch__sample-cta:hover { border-color: var(--ck-accent); color: var(--ck-accent); background: var(--ck-accent-soft); }
+
 .ck-launch__panel-footer {
   margin-top: 1.75rem;
   padding-top: 1.25rem;

data/app/controllers/completion_kit/onboarding_controller.rb

@@ -14,5 +14,10 @@ module CompletionKit
       cookies[DISMISS_COOKIE] = { value: "1", expires: 1.year.from_now, httponly: true }
       redirect_to prompts_path, notice: "Setup skipped. Pick it back up from Settings → Getting started any time."
     end
+
+    def sample_data
+      Onboarding::SampleData.install!
+      redirect_to onboarding_path, notice: "Loaded a sample dataset and prompt — edit or delete them whenever."
+    end
   end
 end

data/app/helpers/completion_kit/application_helper.rb

@@ -72,6 +72,11 @@ module CompletionKit
       CompletionKit::ProviderCredential::PROVIDER_LABELS[provider.to_s] || provider.to_s.titleize
     end
 
+    def ck_model_option_label(model)
+      return "#{model[:name]} (?)" if model.key?(:judging_confirmed) && !model[:judging_confirmed]
+      model[:name]
+    end
+
     def ck_grouped_models(models, selected = nil)
       if selected.present? && models.none? { |m| m[:id] == selected }
         retired = CompletionKit::Model.find_by(model_id: selected)
@@ -90,7 +95,7 @@ module CompletionKit
       end
 
       ordered_keys = groups.keys.sort_by { |label| [label.start_with?("OpenRouter") ? 1 : 0, label] }
-      grouped = ordered_keys.map { |label| [label, groups[label].map { |m| [m[:name], m[:id]] }] }
+      grouped = ordered_keys.map { |label| [label, groups[label].map { |m| [ck_model_option_label(m), m[:id]] }] }
       grouped_options_for_select(grouped, selected)
     end
 

data/app/jobs/completion_kit/judge_review_job.rb

@@ -71,6 +71,7 @@ module CompletionKit
       )
       review.save!
 
+      confirm_judging_capability(run.judge_model)
       run.send(:broadcast_response_update, response)
       run.send(:broadcast_progress)
       enqueue_completion_check
@@ -78,6 +79,15 @@ module CompletionKit
 
     private
 
+    # A model with supports_judging == nil ("untested") just produced a valid
+    # review — promote it to confirmed. No-op once confirmed (so repeated runs
+    # don't churn the row), and a model already flagged as a bad judge stays so.
+    def confirm_judging_capability(judge_model_id)
+      model = Model.find_by(provider: ApiConfig.provider_for_model(judge_model_id), model_id: judge_model_id)
+      return unless model && model.supports_judging.nil?
+      model.update_columns(supports_judging: true, judging_error: nil)
+    end
+
     def record_terminal_failure!(error)
       response_id = @response_id || arguments.first
       metric_id = @metric_id || arguments.last

data/app/models/completion_kit/model.rb

@@ -8,6 +8,9 @@ module CompletionKit
 
     scope :active, -> { where(status: "active") }
     scope :for_generation, -> { active.where(supports_generation: true) }
-    scope :for_judging, -> { active.where(supports_judging: true) }
+    # Includes models not yet confirmed as judges (supports_judging: nil) — worth
+    # a try, and a successful run flips them to confirmed. Only models known to be
+    # bad judges (false) are excluded.
+    scope :for_judging, -> { active.where(supports_judging: [true, nil]) }
   end
 end

data/app/services/completion_kit/api_config.rb

@@ -61,7 +61,9 @@ module CompletionKit
               end
       query = query.where(provider: provider) if provider.present?
       models = query.order(:provider, :display_name).map do |m|
-        { id: m.model_id, name: m.display_name || m.model_id, provider: m.provider }
+        entry = { id: m.model_id, name: m.display_name || m.model_id, provider: m.provider }
+        entry[:judging_confirmed] = !m.supports_judging.nil? if scope == :judging
+        entry
       end
 
       return models if models.any?

data/app/services/completion_kit/model_discovery_service.rb

@@ -13,8 +13,13 @@ module CompletionKit
     end
 
     def refresh!(&on_progress)
-      models_with_names = fetch_models
-      reconcile(models_with_names)
+      discovered = fetch_models
+      reconcile(discovered)
+      # OpenRouter publishes capability metadata (output modalities, etc.), so we
+      # derive everything from the model list and skip live probing entirely.
+      # Judging stays unknown ("?") until a real run proves it.
+      return if @provider == "openrouter"
+
       probe_new_models(&on_progress)
     end
 
@@ -86,10 +91,19 @@ module CompletionKit
         next nil if entry["deprecated"] == true
         context_length = entry["context_length"].to_i
         next nil if context_length < 8192
-        { id: entry["id"], display_name: entry["name"] }
+        { id: entry["id"], display_name: entry["name"], supports_generation: openrouter_text_output?(entry) }
       end
     end
 
+    # OpenRouter exposes architecture.output_modalities (e.g. ["text"], ["image"],
+    # ["text", "image"]). A model can be used for generation/judging only if it
+    # outputs text. When the field is missing we keep the historical default of
+    # treating the model as text-capable.
+    def openrouter_text_output?(entry)
+      modalities = Array(entry.dig("architecture", "output_modalities")).map(&:to_s)
+      modalities.empty? || modalities.include?("text")
+    end
+
     def fetch_ollama_models
       raise DiscoveryError, "Ollama endpoint URL is required" if @api_endpoint.blank?
       base_url = @api_endpoint.to_s.delete_suffix("/")
@@ -100,35 +114,67 @@ module CompletionKit
       JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: e["id"] } }
     end
 
-    def reconcile(models_with_names)
-      api_model_ids = models_with_names.map { |m| m[:id] }
-      names_by_id = models_with_names.each_with_object({}) { |m, h| h[m[:id]] = m[:display_name] }
+    def reconcile(discovered)
+      api_model_ids = discovered.map { |m| m[:id] }
+      meta_by_id = discovered.index_by { |m| m[:id] }
       existing = Model.where(provider: @provider).index_by(&:model_id)
 
       api_model_ids.each do |model_id|
-        if existing[model_id]
-          attrs = { status: "active", retired_at: nil }
-          attrs[:display_name] = names_by_id[model_id] if names_by_id[model_id].present?
-          existing[model_id].update!(attrs) if existing[model_id].status == "retired" || names_by_id[model_id].present?
+        meta = meta_by_id[model_id]
+        if (model = existing[model_id])
+          reconcile_existing_model(model, meta)
         else
-          attrs = {
-            provider: @provider,
-            model_id: model_id,
-            display_name: names_by_id[model_id],
-            status: "active",
-            discovered_at: Time.current
-          }
-          if %w[openrouter ollama].include?(@provider)
-            attrs[:supports_generation] = true
-            attrs[:probed_at] = nil
-          end
-          Model.create!(attrs)
+          Model.create!(new_model_attrs(model_id, meta))
         end
       end
 
-      active_not_in_api = Model.where(provider: @provider, status: "active")
-                               .where.not(model_id: api_model_ids)
-      active_not_in_api.update_all(status: "retired", retired_at: Time.current)
+      Model.where(provider: @provider, status: "active")
+           .where.not(model_id: api_model_ids)
+           .update_all(status: "retired", retired_at: Time.current)
+    end
+
+    def new_model_attrs(model_id, meta)
+      attrs = {
+        provider: @provider,
+        model_id: model_id,
+        display_name: meta[:display_name],
+        status: "active",
+        discovered_at: Time.current
+      }
+      if @provider == "openrouter"
+        supports_generation = meta[:supports_generation] != false
+        attrs.merge!(
+          supports_generation: supports_generation,
+          supports_judging: nil,
+          probed_at: Time.current,
+          status: supports_generation ? "active" : "failed"
+        )
+      elsif @provider == "ollama"
+        attrs[:supports_generation] = true
+        attrs[:probed_at] = nil
+      end
+      attrs
+    end
+
+    def reconcile_existing_model(model, meta)
+      if @provider == "openrouter"
+        # Re-derive generation capability from the published metadata every refresh
+        # (fixes models discovered before capability metadata was used). Leave
+        # supports_judging alone — it's "learned" from successful runs.
+        supports_generation = meta[:supports_generation] != false
+        model.update!(
+          display_name: meta[:display_name].presence || model.display_name,
+          supports_generation: supports_generation,
+          generation_error: nil,
+          probed_at: Time.current,
+          status: supports_generation ? "active" : "failed",
+          retired_at: nil
+        )
+      else
+        attrs = { status: "active", retired_at: nil }
+        attrs[:display_name] = meta[:display_name] if meta[:display_name].present?
+        model.update!(attrs) if model.status == "retired" || meta[:display_name].present?
+      end
     end
 
     def probe_new_models(&on_progress)
@@ -223,7 +269,6 @@ module CompletionKit
       case @provider
       when "openai" then openai_probe(model_id, input, max_tokens)
       when "anthropic" then anthropic_probe(model_id, input, max_tokens)
-      when "openrouter" then openrouter_probe(model_id, input, max_tokens)
       when "ollama" then ollama_probe(model_id, input, max_tokens)
       else raise ArgumentError, "Unsupported probe provider: #{@provider}"
       end
@@ -290,23 +335,6 @@ module CompletionKit
       end
     end
 
-    def openrouter_probe(model_id, input, max_tokens)
-      conn = Faraday.new(url: "https://openrouter.ai") do |f|
-        f.options.timeout = 30
-        f.options.open_timeout = 5
-        f.request :retry, max: 1, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-      conn.post do |req|
-        req.url "/api/v1/chat/completions"
-        req.headers["Content-Type"] = "application/json"
-        req.headers["Authorization"] = "Bearer #{@api_key}"
-        req.headers["HTTP-Referer"] = "https://completionkit.com"
-        req.headers["X-Title"] = "CompletionKit"
-        req.body = { model: model_id, messages: [{ role: "user", content: input }], max_tokens: max_tokens }.to_json
-      end
-    end
-
     def ollama_probe(model_id, input, max_tokens)
       base_url = @api_endpoint.to_s.delete_suffix("/")
       conn = Faraday.new(url: base_url) do |f|

data/app/services/completion_kit/onboarding/checklist.rb

@@ -50,6 +50,13 @@ module CompletionKit
         steps.all?(&:done?)
       end
 
+      # Whether the "Load sample data" button should show — only while neither
+      # the dataset nor the prompt step is done (SampleData.install! no-ops
+      # otherwise, so the button would do nothing).
+      def sample_loadable?
+        steps.none? { |s| %i[dataset prompt].include?(s.key) && s.done? }
+      end
+
       def progress
         done = steps.count(&:done?)
         { done: done, total: steps.size, percent: ((done.to_f / steps.size) * 100).round }

data/app/services/completion_kit/onboarding/sample_data.rb

@@ -0,0 +1,37 @@
+module CompletionKit
+  module Onboarding
+    # Opt-in starter data for the onboarding page: one dataset + one prompt so a
+    # brand-new install has something to poke at. Idempotent — a no-op once the
+    # workspace already has any prompt or dataset. Deliberately does NOT create a
+    # provider credential (needs a real API key) or a run (user-initiated).
+    module SampleData
+      SAMPLE_CSV = <<~CSV.freeze
+        ticket
+        "My order #4827 arrived with a dented panel. I emailed photos 11 days ago and heard nothing. Today I was told the return window 'closed'. I paid $749. I want a refund or replacement, not store credit."
+        "Tracking says delivered to my porch Tuesday 3:47pm. I was home all day, nothing arrived, neighbours' cameras show no van. Order #5102 — a $315 mixer, wedding gift, wedding is Saturday. Can someone look today?"
+        "WELCOME20 says 'invalid' at checkout but the promo email says it's good through May 31. Same email I'm signed in with. Tried Chrome and Safari. Cart is $186 waiting on you."
+      CSV
+
+      SAMPLE_PROMPT = {
+        name: "Sample: Support reply",
+        description: "A starter prompt — drafts a warm, professional reply to a customer support ticket. Edit it or delete it; it's just here to get you going.",
+        template: "You are a senior customer-support specialist. Write a warm, professional reply to this ticket. Acknowledge the customer's situation, be specific about next steps, and don't be defensive.\n\nTicket:\n{{ticket}}",
+        llm_model: "gpt-4o-mini"
+      }.freeze
+
+      module_function
+
+      def install!
+        return if CompletionKit::Prompt.exists? || CompletionKit::Dataset.exists?
+
+        CompletionKit::Dataset.create!(name: "Sample: Customer tickets", csv_data: SAMPLE_CSV)
+        CompletionKit::Prompt.create!(
+          name: SAMPLE_PROMPT[:name],
+          description: SAMPLE_PROMPT[:description],
+          template: SAMPLE_PROMPT[:template],
+          llm_model: SAMPLE_PROMPT[:llm_model]
+        )
+      end
+    end
+  end
+end

data/app/views/completion_kit/onboarding/show.html.erb

@@ -69,6 +69,15 @@
         <% end %>
       </ol>
 
+      <% if @checklist.sample_loadable? %>
+        <div class="ck-launch__sample">
+          <p class="ck-launch__sample-copy">Just exploring? Drop in a sample dataset and prompt to poke around — they're labelled <code>Sample:</code>, and you can edit or delete them whenever.</p>
+          <%= button_to onboarding_sample_data_path, method: :post, class: "ck-launch__sample-cta", form: { style: "display:inline" } do %>
+            Load sample data <span aria-hidden="true">&rarr;</span>
+          <% end %>
+        </div>
+      <% end %>
+
       <div class="ck-launch__panel-footer">
         <%= button_to dismiss_onboarding_path, method: :post, class: "ck-launch__dismiss", form: { style: "display:inline" } do %>
           Skip setup &mdash; go to the app <span aria-hidden="true">&rarr;</span>

data/app/views/completion_kit/provider_credentials/_models_card.html.erb

@@ -36,7 +36,7 @@
                     Gen<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup ck-info-popup--right">Generation models produce the responses your prompts ask for. Pick one when creating a prompt.</span>
                   </th>
                   <th scope="col" class="ck-model-table__cap">
-                    Judge<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup ck-info-popup--right">Judge models score generated responses against your metrics. Pick one when configuring a run.</span>
+                    Judge<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup ck-info-popup--right">Judge models score generated responses against your metrics. Pick one when configuring a run. A <strong>?</strong> means we haven't confirmed this model works as a judge — it's still selectable, and a successful run promotes it to ✓.</span>
                   </th>
                 </tr>
               </thead>
@@ -54,8 +54,10 @@
                     <td class="ck-model-table__cap">
                       <% if m.supports_judging %>
                         <span class="ck-model-table__tick" aria-label="Supports judging">✓</span>
+                      <% elsif m.supports_judging.nil? %>
+                        <span class="ck-model-table__unknown" aria-label="Untested as judge" title="Untested as a judge — selectable; a successful run confirms it">?</span>
                       <% else %>
-                        <span class="ck-model-table__dash" aria-label="No judging support">—</span>
+                        <span class="ck-model-table__dash" aria-label="Not usable as judge">—</span>
                       <% end %>
                     </td>
                   </tr>

data/config/routes.rb

@@ -3,6 +3,7 @@ CompletionKit::Engine.routes.draw do
 
   get "onboarding", to: "onboarding#show", as: :onboarding
   post "onboarding/dismiss", to: "onboarding#dismiss", as: :dismiss_onboarding
+  post "onboarding/sample-data", to: "onboarding#sample_data", as: :onboarding_sample_data
 
   resources :prompts do
     member do

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.5.4"
+  VERSION = "0.5.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - Damien Bastin
@@ -294,6 +294,7 @@ files:
 - app/services/completion_kit/model_discovery_service.rb
 - app/services/completion_kit/ollama_client.rb
 - app/services/completion_kit/onboarding/checklist.rb
+- app/services/completion_kit/onboarding/sample_data.rb
 - app/services/completion_kit/open_ai_client.rb
 - app/services/completion_kit/open_router_client.rb
 - app/services/completion_kit/prompt_improvement_service.rb