completion-kit 0.25.3 → 0.26.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 522b556370ce964401d721cfca911d3eeed34c73d4f00a10fd817d72aea6ebd9
-  data.tar.gz: 376659f68600f29d4b5a8d5a659b106154794b1c21fce13419115199061f99b5
+  metadata.gz: 4c870f728fb1ea2e6cb3eb0319a5ff3102b23d6aec1d2bf6023215e61d9e8896
+  data.tar.gz: 12dcb5ff03f4efa4d61ac761d213252f9eb29c12f90619147da2daec6e16876f
 SHA512:
-  metadata.gz: 2f4d28f7545dee426f9ecf540a8249a5559c60c75624dae1649c0cf5a9a32eeeb37d3ce0e82d90c3749859bd593aec9c6f874a8284a0b3635ea7b60b4197baae
-  data.tar.gz: 547840aee8d6b9d2ca7495ab9d716116ca5aba721503c6b4d71570789587af94ed0c7d5afe5c73d853362e1e6dbabd583e2042731002336ef63e1b703312db84
+  metadata.gz: 53238e3533ba8fb70bbca85287367a2caf409ab65fb544d04f78c00c5a98ce2938caabcb5ccc75e75f0dfa45a84959aaaabbcd69675f7c152936b125cbcf189f
+  data.tar.gz: 289b3034cf9a8dfc28792303790afffa37d89b852bed6009efc40184b42ab6e74d6f8095e42f4e9204babf45781ac5d5c07e7ccfdb4891ebb568b442ab351a5d

data/app/controllers/completion_kit/api/v1/provider_credentials_controller.rb

@@ -30,8 +30,11 @@ module CompletionKit
         end
 
         def destroy
-          @credential.destroy!
-          head :no_content
+          if @credential.destroy
+            head :no_content
+          else
+            render_validation_errors(@credential)
+          end
         end
 
         private
@@ -43,7 +46,7 @@ module CompletionKit
         end
 
         def credential_params
-          params.permit(:provider, :api_key, :api_endpoint)
+          params.permit(:provider, :api_key, :api_endpoint, :api_version)
         end
       end
     end

data/app/controllers/completion_kit/provider_credentials_controller.rb

@@ -1,6 +1,6 @@
 module CompletionKit
   class ProviderCredentialsController < ApplicationController
-    before_action :set_provider_credential, only: [:edit, :update, :refresh]
+    before_action :set_provider_credential, only: [:edit, :update, :refresh, :destroy]
 
     def index
       @provider_credentials = ProviderCredential.order(:provider)
@@ -36,6 +36,14 @@ module CompletionKit
       end
     end
 
+    def destroy
+      if @provider_credential.destroy
+        redirect_to provider_credentials_path, notice: "#{@provider_credential.display_provider} provider was removed."
+      else
+        redirect_to provider_credentials_path, alert: @provider_credential.errors.full_messages.to_sentence
+      end
+    end
+
     def refresh
       @provider_credential.update_columns(discovery_status: "discovering", discovery_current: 0, discovery_total: 0)
       @provider_credential.reload
@@ -62,7 +70,7 @@ module CompletionKit
     end
 
     def provider_credential_params
-      params.require(:provider_credential).permit(:provider, :api_key, :api_endpoint)
+      params.require(:provider_credential).permit(:provider, :api_key, :api_endpoint, :api_version)
     end
   end
 end

data/app/jobs/completion_kit/model_discovery_job.rb

@@ -18,6 +18,9 @@ module CompletionKit
     discard_on ActiveJob::DeserializationError
 
     rescue_from(StandardError) do |error|
+      if error.is_a?(CompletionKit::ModelDiscoveryService::DiscoveryError)
+        Rails.error.report(error, handled: true, context: { job: self.class.name, provider_credential_id: arguments.first })
+      end
       credential = ProviderCredential.find(arguments.first)
       credential.update_columns(discovery_status: "failed", discovery_error: error.message.to_s.truncate(500))
       credential.reload

data/app/models/completion_kit/provider_credential.rb

@@ -1,19 +1,20 @@
 module CompletionKit
   class ProviderCredential < ApplicationRecord
     include Turbo::Broadcastable
-    PROVIDERS = %w[openai anthropic ollama openrouter].freeze
+    PROVIDERS = %w[openai anthropic ollama openrouter azure_foundry].freeze
     PROVIDER_LABELS = {
       "openai" => "OpenAI",
       "anthropic" => "Anthropic",
       "ollama" => "Ollama / OpenAI-compatible endpoint",
-      "openrouter" => "OpenRouter"
+      "openrouter" => "OpenRouter",
+      "azure_foundry" => "Azure AI Foundry"
     }.freeze
 
     encrypts :api_key
 
     def as_json(options = {})
       {
-        id: id, provider: provider, api_endpoint: api_endpoint,
+        id: id, provider: provider, api_endpoint: api_endpoint, api_version: api_version,
         created_at: created_at, updated_at: updated_at
       }
     end
@@ -24,15 +25,24 @@ module CompletionKit
 
     validates :provider, presence: true, inclusion: { in: PROVIDERS }
     validates :provider, tenant_scoped_uniqueness: true
+    validates :api_endpoint, presence: true, if: :azure_foundry?
+    validates :api_version, presence: true, if: :azure_foundry?
     validate :api_endpoint_not_internal
 
     after_save :enqueue_discovery
+    before_destroy :ensure_not_in_use, prepend: true
+    after_destroy :destroy_discovered_models
+
+    def azure_foundry?
+      provider == "azure_foundry"
+    end
 
     def config_hash
       {
         provider: provider,
         api_key: api_key,
-        api_endpoint: api_endpoint
+        api_endpoint: api_endpoint,
+        api_version: api_version
       }.compact
     end
 
@@ -78,6 +88,17 @@ module CompletionKit
          .maximum(:created_at)
     end
 
+    def in_use?
+      prompt_count.positive? || judge_count.positive?
+    end
+
+    def in_use_message
+      parts = []
+      parts << "#{prompt_count} #{'prompt'.pluralize(prompt_count)}" if prompt_count.positive?
+      parts << "#{judge_count} judge #{'run'.pluralize(judge_count)}" if judge_count.positive?
+      "#{display_provider} is still in use by #{parts.to_sentence}. Remove those references before deleting it."
+    end
+
     def broadcast_discovery_progress
       safely_broadcast do
         broadcast_replace_to(
@@ -108,6 +129,17 @@ module CompletionKit
 
     private
 
+    def ensure_not_in_use
+      return unless in_use?
+
+      errors.add(:base, in_use_message)
+      throw :abort
+    end
+
+    def destroy_discovered_models
+      Model.where(provider: provider).delete_all
+    end
+
     def enqueue_discovery
       update_columns(discovery_status: "discovering", discovery_current: 0, discovery_total: 0)
       ModelDiscoveryJob.perform_later(id)

data/app/services/completion_kit/api_config.rb

@@ -1,6 +1,6 @@
 module CompletionKit
   class ApiConfig
-    PROVIDERS = %w[openai anthropic ollama openrouter].freeze
+    PROVIDERS = %w[openai anthropic ollama openrouter azure_foundry].freeze
 
     def self.for_model(model_name)
       provider = provider_for_model(model_name)

data/app/services/completion_kit/azure_foundry_client.rb

@@ -0,0 +1,109 @@
+module CompletionKit
+  class AzureFoundryClient < LlmClient
+    def temperature_dropped?
+      @temperature_dropped == true
+    end
+
+    def generate_completion(prompt, options = {})
+      @temperature_dropped = false
+      return "Error: Azure provider is not fully configured" unless configured?
+      return "Error: API endpoint resolves to a private address" unless ProviderEndpoint.safe?(api_endpoint)
+
+      model = options[:model]
+      max_tokens = options[:max_tokens] || 1000
+      temperature = options[:temperature] || 0.7
+
+      response = post_chat(model: model, prompt: prompt, max_tokens: max_tokens, temperature: temperature)
+
+      if response.status == 400 && temperature_unsupported?(response.body)
+        @temperature_dropped = true
+        response = post_chat(model: model, prompt: prompt, max_tokens: max_tokens, temperature: nil)
+      end
+
+      if response.status == 429
+        raise CompletionKit::RateLimitError.new(
+          response.body.to_s.truncate(500),
+          provider: "azure_foundry",
+          status: 429,
+          retry_after: response.headers["Retry-After"]&.to_i
+        )
+      end
+
+      if response.success?
+        data = JSON.parse(response.body)
+        content = data.dig("choices", 0, "message", "content").to_s.strip
+        return "Error: model returned empty content" if content.empty?
+        content
+      else
+        "Error: #{response.status} - #{response.body}"
+      end
+    rescue CompletionKit::RateLimitError
+      raise
+    rescue Faraday::Error
+      raise
+    rescue => e
+      "Error: #{e.message}"
+    end
+
+    def available_models
+      return [] unless configured?
+      return [] unless ProviderEndpoint.safe?(api_endpoint)
+
+      response = build_connection(azure_base_url).get("/openai/deployments?api-version=#{api_version}") do |req|
+        req.headers["api-key"] = api_key
+      end
+      return [] unless response.success?
+
+      JSON.parse(response.body).fetch("data", []).map { |entry| { id: entry["id"], name: entry["id"] } }
+    rescue StandardError
+      []
+    end
+
+    def configured?
+      configuration_errors.empty?
+    end
+
+    def configuration_errors
+      errors = []
+      errors << "Azure endpoint is not configured" if api_endpoint.blank?
+      errors << "Azure API key is not configured" if api_key.blank?
+      errors << "Azure api-version is not configured" if api_version.blank?
+      errors
+    end
+
+    private
+
+    def api_key
+      @config[:api_key]
+    end
+
+    def api_endpoint
+      @config[:api_endpoint]
+    end
+
+    def api_version
+      @config[:api_version]
+    end
+
+    def azure_base_url
+      api_endpoint.to_s.strip.delete_suffix("/")
+    end
+
+    def post_chat(model:, prompt:, max_tokens:, temperature:)
+      body = { messages: [{ role: "user", content: prompt }], max_tokens: max_tokens }
+      body[:temperature] = temperature unless temperature.nil?
+
+      build_connection(azure_base_url, timeout: 30, open_timeout: 5).post do |req|
+        req.url "/openai/deployments/#{model}/chat/completions?api-version=#{api_version}"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["api-key"] = api_key
+        req.body = body.to_json
+      end
+    end
+
+    def temperature_unsupported?(body)
+      s = body.to_s
+      s.include?("temperature") && (s.include?("deprecated") || s.include?("not supported") || s.include?("Unsupported parameter"))
+    end
+  end
+end

data/app/services/completion_kit/llm_client.rb

@@ -34,6 +34,8 @@ module CompletionKit
         OllamaClient.new(config)
       when "openrouter"
         OpenRouterClient.new(config)
+      when "azure_foundry"
+        AzureFoundryClient.new(config)
       else
         raise ArgumentError, "Unsupported provider: #{provider_name}"
       end

data/app/services/completion_kit/mcp_tools/provider_credentials.rb

@@ -19,9 +19,10 @@ module CompletionKit
           inputSchema: {
             type: "object",
             properties: {
-              provider: {type: "string", enum: ["openai", "anthropic", "ollama", "openrouter"]},
+              provider: {type: "string", enum: ["openai", "anthropic", "ollama", "openrouter", "azure_foundry"]},
               api_key: {type: "string"},
-              api_endpoint: {type: "string"}
+              api_endpoint: {type: "string"},
+              api_version: {type: "string"}
             },
             required: ["provider", "api_key"]
           },
@@ -33,7 +34,7 @@ module CompletionKit
             type: "object",
             properties: {
               id: {type: "integer"}, provider: {type: "string"},
-              api_key: {type: "string"}, api_endpoint: {type: "string"}
+              api_key: {type: "string"}, api_endpoint: {type: "string"}, api_version: {type: "string"}
             },
             required: ["id"]
           },
@@ -55,7 +56,7 @@ module CompletionKit
       end
 
       def self.create(args)
-        credential = ProviderCredential.new(args.slice("provider", "api_key", "api_endpoint"))
+        credential = ProviderCredential.new(args.slice("provider", "api_key", "api_endpoint", "api_version"))
         if credential.save
           text_result(credential.as_json)
         else
@@ -65,7 +66,7 @@ module CompletionKit
 
       def self.update(args)
         credential = ProviderCredential.find(args["id"])
-        if credential.update(args.except("id").slice("provider", "api_key", "api_endpoint"))
+        if credential.update(args.except("id").slice("provider", "api_key", "api_endpoint", "api_version"))
           text_result(credential.as_json)
         else
           error_result(credential.errors.full_messages.join(", "))
@@ -73,8 +74,12 @@ module CompletionKit
       end
 
       def self.delete(args)
-        ProviderCredential.find(args["id"]).destroy!
-        text_result("Provider credential #{args["id"]} deleted")
+        credential = ProviderCredential.find(args["id"])
+        if credential.destroy
+          text_result("Provider credential #{args["id"]} deleted")
+        else
+          error_result(credential.errors.full_messages.join(", "))
+        end
       end
     end
   end

data/app/services/completion_kit/model_discovery_service.rb

@@ -6,18 +6,18 @@ module CompletionKit
   class ModelDiscoveryService
     class DiscoveryError < StandardError; end
 
+    AZURE_HOST_SUFFIXES = [".openai.azure.com", ".services.ai.azure.com"].freeze
+
     def initialize(config:)
       @provider = config[:provider]
       @api_key = config[:api_key]
       @api_endpoint = config[:api_endpoint]
+      @api_version = config[:api_version]
     end
 
     def refresh!(force: false, &on_progress)
       discovered = fetch_models
       reconcile(discovered)
-      # OpenRouter publishes capability metadata (output modalities, etc.), so we
-      # derive everything from the model list and skip live probing entirely.
-      # Judging stays unknown ("?") until a real run proves it.
       return if @provider == "openrouter"
 
       reset_failed_generation if force
@@ -32,6 +32,7 @@ module CompletionKit
       when "anthropic" then fetch_anthropic_models
       when "openrouter" then fetch_openrouter_models
       when "ollama" then fetch_ollama_models
+      when "azure_foundry" then fetch_azure_foundry_models
       else []
       end
     end
@@ -106,19 +107,72 @@ module CompletionKit
     end
 
     def fetch_ollama_models
-      raise DiscoveryError, "Ollama endpoint URL is required" if @api_endpoint.blank?
+      raise DiscoveryError, "A model endpoint URL is required." if @api_endpoint.blank?
       base_url = ollama_root_url
       response = fetch_connection(base_url).get("/v1/models") do |req|
         req.headers["Authorization"] = "Bearer #{@api_key}" if @api_key.present?
       end
-      raise_fetch_error!(response) unless response.success?
+      raise DiscoveryError, custom_endpoint_error_message(response) unless response.success?
       JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: e["id"] } }
     end
 
+    def custom_endpoint_error_message(response)
+      detail = extract_provider_error_message(response.body)
+      case response.status
+      when 401, 403
+        with_detail("The endpoint rejected the API key (#{response.status}).", detail)
+      when 404
+        custom_endpoint_404_message
+      when 429
+        "The endpoint rate-limited the model-list request (429). Try again shortly."
+      else
+        with_detail("The endpoint at #{custom_endpoint_host} did not return an OpenAI-compatible model list at /v1/models (#{response.status}).", detail)
+      end
+    end
+
+    def custom_endpoint_404_message
+      message = "No OpenAI-compatible model list was found at #{custom_endpoint_host}/v1/models (404). Check that the base URL is correct."
+      return message unless azure_custom_host?
+      "#{message} This looks like an Azure endpoint; add it with the Azure AI Foundry provider, which uses an api-version and an api-key header."
+    end
+
+    def with_detail(message, detail)
+      detail.present? ? "#{message} #{detail}" : message
+    end
+
+    def custom_endpoint_host
+      ProviderEndpoint.parse(@api_endpoint)&.host || @api_endpoint.to_s
+    end
+
+    def azure_custom_host?
+      host = custom_endpoint_host.to_s.downcase
+      AZURE_HOST_SUFFIXES.any? { |suffix| host.end_with?(suffix) }
+    end
+
     def ollama_root_url
       @api_endpoint.to_s.strip.delete_suffix("/").delete_suffix("/v1")
     end
 
+    def fetch_azure_foundry_models
+      raise DiscoveryError, "An Azure endpoint URL is required." if @api_endpoint.blank?
+      raise DiscoveryError, "An Azure api-version is required." if @api_version.blank?
+
+      response = fetch_connection(azure_base_url).get("/openai/deployments?api-version=#{@api_version}") do |req|
+        req.headers["api-key"] = @api_key
+      end
+      raise DiscoveryError, azure_error_message(response) unless response.success?
+      JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: e["id"] } }
+    end
+
+    def azure_base_url
+      @api_endpoint.to_s.strip.delete_suffix("/")
+    end
+
+    def azure_error_message(response)
+      detail = extract_provider_error_message(response.body)
+      with_detail("Azure did not return a deployments list at /openai/deployments (#{response.status}). Check the endpoint base URL and api-version.", detail)
+    end
+
     def reconcile(discovered)
       api_model_ids = discovered.map { |m| m[:id] }
       meta_by_id = discovered.index_by { |m| m[:id] }
@@ -150,7 +204,7 @@ module CompletionKit
         supports_generation = meta[:supports_generation] != false
         attrs.merge!(
           supports_generation: supports_generation,
-          supports_judging: nil,
+          supports_judging: supports_generation,
           probed_at: Time.current,
           status: supports_generation ? "active" : "failed"
         )
@@ -163,14 +217,13 @@ module CompletionKit
 
     def reconcile_existing_model(model, meta)
       if @provider == "openrouter"
-        # Re-derive generation capability from the published metadata every refresh
-        # (fixes models discovered before capability metadata was used). Leave
-        # supports_judging alone — it's "learned" from successful runs.
         supports_generation = meta[:supports_generation] != false
         model.update!(
           display_name: meta[:display_name].presence || model.display_name,
           supports_generation: supports_generation,
+          supports_judging: supports_generation,
           generation_error: nil,
+          judging_error: nil,
           probed_at: Time.current,
           status: supports_generation ? "active" : "failed",
           retired_at: nil
@@ -287,6 +340,7 @@ module CompletionKit
       when "openai" then openai_probe(model_id, input, max_tokens)
       when "anthropic" then anthropic_probe(model_id, input, max_tokens)
       when "ollama" then ollama_probe(model_id, input, max_tokens)
+      when "azure_foundry" then azure_foundry_probe(model_id, input, max_tokens)
       else raise ArgumentError, "Unsupported probe provider: #{@provider}"
       end
     end
@@ -366,5 +420,20 @@ module CompletionKit
         req.body = { model: model_id, messages: [{ role: "user", content: input }], max_tokens: max_tokens }.to_json
       end
     end
+
+    def azure_foundry_probe(model_id, input, max_tokens)
+      conn = Faraday.new(url: azure_base_url) do |f|
+        f.options.timeout = 60
+        f.options.open_timeout = 5
+        f.request :retry, max: 1, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+      conn.post do |req|
+        req.url "/openai/deployments/#{model_id}/chat/completions?api-version=#{@api_version}"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["api-key"] = @api_key
+        req.body = { messages: [{ role: "user", content: input }], max_tokens: max_tokens }.to_json
+      end
+    end
   end
 end

data/app/views/completion_kit/provider_credentials/_form.html.erb

@@ -24,11 +24,31 @@
 
     <div class="ck-field">
       <%= form.label :api_endpoint, "API endpoint", class: "ck-label" %>
-      <%= form.text_field :api_endpoint, class: "ck-input", placeholder: "Only needed for Ollama or custom OpenAI-compatible endpoints", **ck_field_aria(form, :api_endpoint) %>
+      <%= form.text_field :api_endpoint, class: "ck-input", placeholder: "Needed for Ollama, Azure AI Foundry, or custom OpenAI-compatible endpoints", **ck_field_aria(form, :api_endpoint) %>
       <%= ck_field_error(form, :api_endpoint) %>
     </div>
 
+    <div class="ck-field">
+      <%= form.label :api_version, "API version", class: "ck-label" %>
+      <%= form.text_field :api_version, class: "ck-input", placeholder: "Only needed for Azure AI Foundry (e.g. 2024-10-21)", **ck_field_aria(form, :api_version) %>
+      <%= ck_field_error(form, :api_version) %>
+    </div>
+
+    <% if provider_credential.persisted? && provider_credential.in_use? %>
+      <p class="ck-field-hint"><%= provider_credential.in_use_message %></p>
+    <% end %>
+
     <div class="ck-actions">
+      <% if provider_credential.persisted? && !provider_credential.in_use? %>
+        <%= button_to provider_credential_path(provider_credential), method: :delete,
+              form_class: "inline-block",
+              class: "ck-icon-btn",
+              title: "Delete provider",
+              "aria-label": "Delete provider",
+              data: { turbo_confirm: "Delete the #{provider_credential.display_provider} provider and its discovered models? This can't be undone." } do %>
+          <%= heroicon_tag "trash", variant: :outline, size: 16, "aria-hidden": "true" %>
+        <% end %>
+      <% end %>
       <%= link_to "Cancel", provider_credentials_path, class: ck_button_classes(:light, variant: :outline), tabindex: "0" %>
       <%= form.submit(provider_credential.persisted? ? "Save provider" : "Create provider", class: ck_button_classes(:dark)) %>
     </div>

data/config/routes.rb

@@ -51,7 +51,7 @@ CompletionKit::Engine.routes.draw do
     end
   end
 
-  resources :provider_credentials, only: [:index, :new, :create, :edit, :update] do
+  resources :provider_credentials, only: [:index, :new, :create, :edit, :update, :destroy] do
     post :refresh, on: :member
     get :statuses, on: :collection
   end

data/db/migrate/20260708000001_add_api_version_to_completion_kit_provider_credentials.rb

@@ -0,0 +1,5 @@
+class AddApiVersionToCompletionKitProviderCredentials < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_provider_credentials, :api_version, :string
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.25.3"
+  VERSION = "0.26.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.25.3
+  version: 0.26.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -300,6 +300,7 @@ files:
 - app/services/completion_kit/agreement_math.rb
 - app/services/completion_kit/anthropic_client.rb
 - app/services/completion_kit/api_config.rb
+- app/services/completion_kit/azure_foundry_client.rb
 - app/services/completion_kit/checks/contains.rb
 - app/services/completion_kit/checks/equals.rb
 - app/services/completion_kit/checks/expected_resolver.rb
@@ -476,6 +477,7 @@ files:
 - db/migrate/20260629000002_add_check_type_to_completion_kit_metric_versions.rb
 - db/migrate/20260629000003_add_passed_to_completion_kit_reviews.rb
 - db/migrate/20260706000001_add_expected_column_to_completion_kit_runs.rb
+- db/migrate/20260708000001_add_api_version_to_completion_kit_provider_credentials.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb