completion-kit 0.26.8 → 0.27.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 42fae0f3b31530598ee43e1d39f3d8dbb1afd53e4c0af73aa2013a33433414f7
-  data.tar.gz: 46ab6507a7181494383f4d960fbd985c4a04587d842d274151c1e639729a96fd
+  metadata.gz: 38c7acaadacc7cb636ee99e3eb7e4c52f68b89ed57122487eb1591be99435716
+  data.tar.gz: 3e5f9c36019c6635ae41b913eea74607f3b8c9f21a511c5ccc3f9ff75422282d
 SHA512:
-  metadata.gz: 6f2be3fdcaa3c439159fbddf36718b85b300ffd67bd0a660ebc002de0fd6f70ac73383f0ae8efe14d6ee2df8f35421cf9712f75ec503c22c92a08a633685214c
-  data.tar.gz: b3b371448f03e9a94d4d3ee52c47546d156b6e9b194cdec20cd4ee10e9f1f12b783f37e4eb1a1446b597f6e3c4e01b8c0f682c69f9710ec567d56d223ddc846a
+  metadata.gz: 80adc55bde36fa3ac782c213c4786e677cd27d1b85bef81c14fbd89e05a82e229681866390c34d7f17cd9d6a153bcb93da705d63a87f7aede39dcb26929e3629
+  data.tar.gz: 533ef4f6c4b29791a66ce9169ab4c980a38838f5e3a5184d0b07cf0a4276ad0fd5990fae69616a2ea43421daebab1ca1ed2f246b4a310a506403db7548935981

data/app/assets/stylesheets/completion_kit/application.css

@@ -961,14 +961,6 @@ tr:hover .ck-chip--publish {
   max-width: 36rem;
 }
 
-.ck-model-list__empty {
-  display: flex;
-  align-items: center;
-  gap: 1rem;
-  flex-wrap: wrap;
-  padding: 0.35rem 0.25rem;
-}
-
 .ck-form-card__footer-header {
   display: flex;
   align-items: center;
@@ -1504,6 +1496,26 @@ tr:hover .ck-chip--publish {
   gap: 0.6rem;
 }
 
+.ck-model-list__catalog {
+  margin-top: 0.75rem;
+  padding-top: 0.6rem;
+  border-top: 1px solid var(--ck-line);
+  font-family: var(--ck-mono);
+  font-size: 0.72rem;
+  letter-spacing: 0.03em;
+}
+
+.ck-model-list__catalog-link {
+  color: var(--ck-accent);
+  font-weight: 500;
+  text-decoration: none;
+  white-space: nowrap;
+}
+
+.ck-model-list__catalog-link:hover {
+  color: var(--ck-accent-hover);
+}
+
 .ck-model-list__summary-stamp {
   font-family: var(--ck-mono);
   font-size: 0.7rem;

data/app/controllers/completion_kit/api/v1/prompts_controller.rb

@@ -25,10 +25,17 @@ module CompletionKit
 
         def update
           if @prompt.runs.exists?
-            new_prompt = @prompt.clone_as_new_version(prompt_params.except(:tag_names).to_h)
-            new_prompt.publish!
-            new_prompt.update!(tag_names: prompt_params[:tag_names]) if prompt_params.key?(:tag_names)
-            render json: new_prompt.reload
+            new_prompt = @prompt.build_next_version(prompt_params.except(:tag_names).to_h)
+            if new_prompt.valid?
+              CompletionKit::ApplicationRecord.transaction do
+                new_prompt.save!
+                new_prompt.publish!
+                new_prompt.update!(tag_names: prompt_params[:tag_names]) if prompt_params.key?(:tag_names)
+              end
+              render json: new_prompt.reload
+            else
+              render_validation_errors(new_prompt)
+            end
           elsif @prompt.update(prompt_params)
             render json: @prompt
           else

data/app/controllers/completion_kit/prompts_controller.rb

@@ -33,10 +33,19 @@ module CompletionKit
     
     def update
       if @prompt.runs.exists?
-        new_prompt = @prompt.clone_as_new_version(prompt_params.except(:tag_names).to_h)
-        new_prompt.publish!
-        new_prompt.update!(tag_names: prompt_params[:tag_names]) if prompt_params.key?(:tag_names)
-        redirect_to prompt_path(new_prompt), notice: "Saved as #{new_prompt.version_label}."
+        new_prompt = @prompt.build_next_version(prompt_params.except(:tag_names).to_h)
+        if new_prompt.valid?
+          CompletionKit::ApplicationRecord.transaction do
+            new_prompt.save!
+            new_prompt.publish!
+            new_prompt.update!(tag_names: prompt_params[:tag_names]) if prompt_params.key?(:tag_names)
+          end
+          redirect_to prompt_path(new_prompt), notice: "Saved as #{new_prompt.version_label}."
+        else
+          @prompt.assign_attributes(prompt_params.except(:tag_names).to_h)
+          @prompt.errors.merge!(new_prompt.errors)
+          render :edit, status: :unprocessable_entity
+        end
       elsif @prompt.update(prompt_params)
         redirect_to prompt_path(@prompt), notice: "Prompt saved."
       else

data/app/jobs/completion_kit/model_discovery_job.rb

@@ -48,7 +48,12 @@ module CompletionKit
         credential.broadcast_discovery_progress
       end
 
-      credential.update_columns(discovery_status: "completed", discovery_error: nil, updated_at: Time.current)
+      credential.update_columns(
+        discovery_status: "completed",
+        discovery_error: nil,
+        catalog_model_count: service.catalog_model_count,
+        updated_at: Time.current
+      )
       credential.reload
       credential.broadcast_discovery_complete
     end

data/app/models/completion_kit/prompt.rb

@@ -5,9 +5,12 @@ module CompletionKit
     has_many :runs, dependent: :destroy
     has_many :responses, through: :runs
 
+    attr_accessor :cloned_from_llm_model
+
     validates :name, presence: true
     validates :template, presence: true
     validates :llm_model, presence: true
+    validate :llm_model_usable_for_generation, if: :llm_model_newly_selected?
     validates :family_key, presence: true
     validates :version_number, presence: true, numericality: { only_integer: true, greater_than: 0 }
 
@@ -52,8 +55,8 @@ module CompletionKit
       self.class.where(family_key: family_key).order(version_number: :desc, created_at: :desc)
     end
 
-    def clone_as_new_version(overrides = {})
-      self.class.create!(
+    def build_next_version(overrides = {})
+      version = self.class.new(
         {
           name: name,
           description: description,
@@ -65,6 +68,14 @@ module CompletionKit
           published_at: nil
         }.merge(overrides.compact)
       )
+      version.cloned_from_llm_model = llm_model
+      version
+    end
+
+    def clone_as_new_version(overrides = {})
+      version = build_next_version(overrides)
+      version.save!
+      version
     end
 
     def publish!
@@ -86,6 +97,22 @@ module CompletionKit
 
     private
 
+    def llm_model_newly_selected?
+      return false if llm_model.blank?
+      return llm_model != cloned_from_llm_model if new_record? && cloned_from_llm_model.present?
+
+      llm_model_changed?
+    end
+
+    def llm_model_usable_for_generation
+      rows = Model.where(model_id: llm_model)
+      return if rows.empty?
+      return if rows.where(supports_generation: true, status: "active").exists?
+      return unless rows.where(supports_generation: false).exists?
+
+      errors.add(:llm_model, "is not available for generating responses")
+    end
+
     def assign_family_key
       self.family_key ||= SecureRandom.uuid
     end

data/app/models/completion_kit/provider_credential.rb

@@ -61,6 +61,10 @@ module CompletionKit
       Model.where(provider: provider).active.count
     end
 
+    def foundry_catalog_available?
+      azure_foundry? && catalog_model_count.to_i.positive?
+    end
+
     def self.discovery_in_progress?
       where(discovery_status: "discovering").exists?
     end

data/app/services/completion_kit/azure_foundry_client.rb

@@ -58,16 +58,26 @@ module CompletionKit
       return [] unless configured?
       return [] unless ProviderEndpoint.safe?(api_endpoint)
 
-      response = build_connection(azure_base_url).get(models_url) do |req|
+      path = foundry_project? ? "#{azure_base_url}/deployments?api-version=v1" : models_path
+      response = build_connection(azure_base_url).get(path) do |req|
         req.headers["api-key"] = api_key
       end
       return [] unless response.success?
 
-      parse_models(response.body)
+      body = JSON.parse(response.body)
+      if foundry_project?
+        body.fetch("value", []).map { |d| { id: d["name"], name: d["name"] } }
+      else
+        body.fetch("data", []).map { |entry| { id: entry["id"], name: entry["id"] } }
+      end
     rescue StandardError
       []
     end
 
+    def foundry_project?
+      api_endpoint.to_s.include?("/api/projects/")
+    end
+
     def configured?
       configuration_errors.empty?
     end
@@ -101,27 +111,8 @@ module CompletionKit
       api_version.blank?
     end
 
-    def foundry_project?
-      api_endpoint.to_s.include?("/api/projects/")
-    end
-
-    def models_url
-      if foundry_project?
-        "#{azure_base_url}/deployments?api-version=v1"
-      elsif v1_mode?
-        "/openai/v1/models"
-      else
-        "/openai/deployments?api-version=#{api_version}"
-      end
-    end
-
-    def parse_models(body)
-      json = JSON.parse(body)
-      if foundry_project?
-        json.fetch("value", []).map { |d| { id: d["name"], name: d["name"] } }
-      else
-        json.fetch("data", []).map { |e| { id: e["id"], name: e["id"] } }
-      end
+    def models_path
+      v1_mode? ? "/openai/v1/models" : "/openai/deployments?api-version=#{api_version}"
     end
 
     def post_chat(model:, prompt:, max_tokens:, temperature:, max_completion: false)

data/app/services/completion_kit/model_discovery_service.rb

@@ -8,11 +8,14 @@ module CompletionKit
 
     AZURE_HOST_SUFFIXES = [".openai.azure.com", ".services.ai.azure.com"].freeze
 
+    attr_reader :catalog_model_count
+
     def initialize(config:)
       @provider = config[:provider]
       @api_key = config[:api_key]
       @api_endpoint = config[:api_endpoint]
       @api_version = config[:api_version]
+      @catalog_model_count = nil
     end
 
     def refresh!(force: false, &on_progress)
@@ -156,18 +159,32 @@ module CompletionKit
     def fetch_azure_foundry_models
       raise DiscoveryError, "An Azure endpoint URL is required." if @api_endpoint.blank?
 
-      response = fetch_connection(azure_base_url).get(azure_foundry_project? ? "#{azure_base_url}/deployments?api-version=v1" : azure_models_path) do |req|
+      path = azure_foundry_project? ? "#{azure_base_url}/deployments?api-version=v1" : azure_models_path
+      response = fetch_connection(azure_base_url).get(path) do |req|
         req.headers["api-key"] = @api_key
       end
       raise DiscoveryError, azure_error_message(response) unless response.success?
+
       body = JSON.parse(response.body)
       if azure_foundry_project?
+        @catalog_model_count = fetch_azure_catalog_count
         body.fetch("value", []).map { |d| { id: d["name"], display_name: d["name"] } }
       else
         body.fetch("data", []).map { |e| { id: e["id"], display_name: e["id"] } }
       end
     end
 
+    def fetch_azure_catalog_count
+      response = fetch_connection(azure_base_url).get("/openai/v1/models") do |req|
+        req.headers["api-key"] = @api_key
+      end
+      return nil unless response.success?
+
+      JSON.parse(response.body).fetch("data", []).length
+    rescue StandardError
+      nil
+    end
+
     def azure_v1_mode?
       @api_version.blank?
     end

data/app/services/completion_kit/promptfoo_importer.rb

@@ -27,6 +27,8 @@ module CompletionKit
         metrics = import_metrics(config)
         Result.new(ok: true, error: nil, prompts: prompts, dataset: dataset, metrics: metrics, providers: providers)
       end
+    rescue ActiveRecord::RecordInvalid => e
+      failure(e.record.errors.full_messages.to_sentence.presence || e.message)
     end
 
     private

data/app/views/completion_kit/provider_credentials/_models_card.html.erb

@@ -71,12 +71,19 @@
           </div>
         </details>
       <% elsif provider_credential.discovery_status != "discovering" %>
-        <div class="ck-model-list__empty">
-          <% unless provider_credential.discovery_status == "failed" %>
-            <span class="ck-meta-copy">No models discovered yet.</span>
-          <% end %>
-          <button type="button" class="<%= ck_button_classes(:light, variant: :outline) %>" onclick="fetch('<%= refresh_provider_credential_path(provider_credential) %>', {method:'POST',headers:{'X-CSRF-Token':document.querySelector('meta[name=csrf-token]').content}});if(window.ckStartDiscoveryPolling)ckStartDiscoveryPolling(8000)">Discover models</button>
+        <div class="ck-model-list__summary">
+          <span class="ck-model-list__summary-label">Available models <span class="ck-model-list__summary-count">0</span></span>
+          <span class="ck-model-list__summary-meta">
+            <button type="button" class="ck-icon-btn ck-model-list__refresh" title="Refresh models" aria-label="Refresh available models" onclick="event.preventDefault();fetch('<%= refresh_provider_credential_path(provider_credential) %>', {method:'POST',headers:{'X-CSRF-Token':document.querySelector('meta[name=csrf-token]').content}});if(window.ckStartDiscoveryPolling)ckStartDiscoveryPolling(8000)"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="13" height="13" aria-hidden="true"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg></button>
+          </span>
         </div>
       <% end %>
+
+      <% if provider_credential.foundry_catalog_available? %>
+        <p class="ck-meta-copy ck-model-list__catalog">
+          <%= pluralize(provider_credential.catalog_model_count, "model") %> available to deploy in
+          <a href="https://ai.azure.com/" target="_blank" rel="noopener" class="ck-model-list__catalog-link">Azure AI Foundry <span aria-hidden="true">&rarr;</span></a>
+        </p>
+      <% end %>
     </div>
 </div>

data/app/views/completion_kit/responses/show.html.erb

@@ -79,7 +79,9 @@
   </div>
   <% if @response.status == "failed" %>
     <% err = @response.error_payload || {} %>
-    <div class="ck-code-scroll-wrap"><pre class="ck-code ck-code--error"><%= err[:provider]&.titleize %><%= " #{err[:status]}" if err[:status].present? %><%= "\n\n" if err[:provider].present? || err[:status].present? %><%= err[:message] %></pre></div>
+    <% head = [err[:provider]&.titleize, err[:status].presence].compact.join(" ").presence %>
+    <% error_text = [head, @response.error_message.presence].compact.join("\n\n").presence || "The provider returned no error detail." %>
+    <div class="ck-code-scroll-wrap"><pre class="ck-code ck-code--error"><%= error_text %></pre></div>
   <% else %>
     <div class="ck-code-scroll-wrap"><pre class="ck-code"><%= ck_format_maybe_json(@response.response_text) %></pre></div>
   <% end %>

data/db/migrate/20260713000001_add_catalog_model_count_to_completion_kit_provider_credentials.rb

@@ -0,0 +1,5 @@
+class AddCatalogModelCountToCompletionKitProviderCredentials < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_provider_credentials, :catalog_model_count, :integer
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.26.8"
+  VERSION = "0.27.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.26.8
+  version: 0.27.1
 platform: ruby
 authors:
 - Damien Bastin
@@ -479,6 +479,7 @@ files:
 - db/migrate/20260629000003_add_passed_to_completion_kit_reviews.rb
 - db/migrate/20260706000001_add_expected_column_to_completion_kit_runs.rb
 - db/migrate/20260708000001_add_api_version_to_completion_kit_provider_credentials.rb
+- db/migrate/20260713000001_add_catalog_model_count_to_completion_kit_provider_credentials.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb