completion-kit 0.1.0.rc1

data/app/services/completion_kit/model_discovery_service.rb

@@ -0,0 +1,223 @@
+require "faraday"
+require "faraday/retry"
+require "json"
+
+module CompletionKit
+  class ModelDiscoveryService
+    def initialize(config:)
+      @provider = config[:provider]
+      @api_key = config[:api_key]
+      @api_endpoint = config[:api_endpoint]
+    end
+
+    def refresh!(&on_progress)
+      models_with_names = fetch_models
+      reconcile(models_with_names)
+      return if %w[openrouter ollama].include?(@provider)
+      probe_new_models(&on_progress)
+    end
+
+    private
+
+    def fetch_models
+      case @provider
+      when "openai" then fetch_openai_models
+      when "anthropic" then fetch_anthropic_models
+      when "openrouter" then fetch_openrouter_models
+      when "ollama" then fetch_ollama_models
+      else []
+      end
+    end
+
+    def fetch_connection(base_url)
+      Faraday.new(url: base_url) do |f|
+        f.options.timeout = 15
+        f.options.open_timeout = 5
+        f.adapter Faraday.default_adapter
+      end
+    end
+
+    def fetch_openai_models
+      response = fetch_connection("https://api.openai.com").get("/v1/models") do |req|
+        req.headers["Authorization"] = "Bearer #{@api_key}"
+      end
+      return [] unless response.success?
+      JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: nil } }
+    end
+
+    def fetch_anthropic_models
+      response = fetch_connection("https://api.anthropic.com").get("/v1/models?limit=100") do |req|
+        req.headers["x-api-key"] = @api_key
+        req.headers["anthropic-version"] = "2023-06-01"
+      end
+      return [] unless response.success?
+      JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: e["display_name"] } }
+    end
+
+    def fetch_openrouter_models
+      response = fetch_connection("https://openrouter.ai").get("/api/v1/models") do |req|
+        req.headers["Authorization"] = "Bearer #{@api_key}"
+        req.headers["HTTP-Referer"] = "https://completionkit.com"
+        req.headers["X-Title"] = "CompletionKit"
+      end
+      return [] unless response.success?
+      JSON.parse(response.body).fetch("data", []).filter_map do |entry|
+        next nil if entry["deprecated"] == true
+        context_length = entry["context_length"].to_i
+        next nil if context_length < 8192
+        { id: entry["id"], display_name: entry["name"] }
+      end
+    end
+
+    def fetch_ollama_models
+      return [] if @api_endpoint.nil?
+      base_url = @api_endpoint.to_s.delete_suffix("/")
+      response = fetch_connection(base_url).get("/models") do |req|
+        req.headers["Authorization"] = "Bearer #{@api_key}" if @api_key.present?
+      end
+      return [] unless response.success?
+      JSON.parse(response.body).fetch("data", []).map { |e| { id: e["id"], display_name: e["id"] } }
+    end
+
+    def reconcile(models_with_names)
+      api_model_ids = models_with_names.map { |m| m[:id] }
+      names_by_id = models_with_names.each_with_object({}) { |m, h| h[m[:id]] = m[:display_name] }
+      existing = Model.where(provider: @provider).index_by(&:model_id)
+
+      api_model_ids.each do |model_id|
+        if existing[model_id]
+          attrs = { status: "active", retired_at: nil }
+          attrs[:display_name] = names_by_id[model_id] if names_by_id[model_id].present?
+          existing[model_id].update!(attrs) if existing[model_id].status == "retired" || names_by_id[model_id].present?
+        else
+          attrs = {
+            provider: @provider,
+            model_id: model_id,
+            display_name: names_by_id[model_id],
+            status: "active",
+            discovered_at: Time.current
+          }
+          if %w[openrouter ollama].include?(@provider)
+            attrs[:supports_generation] = true
+            attrs[:probed_at] = nil
+          end
+          Model.create!(attrs)
+        end
+      end
+
+      active_not_in_api = Model.where(provider: @provider, status: "active")
+                               .where.not(model_id: api_model_ids)
+      active_not_in_api.update_all(status: "retired", retired_at: Time.current)
+    end
+
+    def probe_new_models(&on_progress)
+      unprobed = Model.where(provider: @provider, supports_generation: nil, status: "active")
+      total = unprobed.count
+      current = 0
+      unprobed.find_each do |model|
+        probe_generation(model)
+        probe_judging(model) if model.supports_generation
+        model.probed_at = Time.current
+        model.status = "failed" if model.supports_generation == false
+        model.save!
+        current += 1
+        on_progress&.call(current, total)
+      end
+    end
+
+    def probe_generation(model)
+      response = send_probe(model.model_id, "Say hello", 20)
+      if response.success?
+        text = extract_text(response)
+        if text.present?
+          model.supports_generation = true
+        else
+          model.supports_generation = false
+          model.generation_error = "Empty response"
+        end
+      else
+        model.supports_generation = false
+        model.generation_error = "#{response.status} - #{response.body.truncate(500)}"
+      end
+    rescue StandardError => e
+      model.supports_generation = false
+      model.generation_error = e.message
+    end
+
+    def probe_judging(model)
+      judge_input = <<~PROMPT
+        You are an expert evaluator. You MUST respond with ONLY two lines in this exact format, nothing else:
+
+        Score: <integer from 1 to 5>
+        Feedback: <one sentence explaining why>
+
+        AI output to evaluate: The sky is blue.
+      PROMPT
+
+      response = send_probe(model.model_id, judge_input, 50)
+      if response.success?
+        text = extract_text(response).to_s
+        if text.match?(/Score:\s*\d/i)
+          model.supports_judging = true
+        else
+          model.supports_judging = false
+          model.judging_error = "Response not in Score/Feedback format: #{text.truncate(200)}"
+        end
+      else
+        model.supports_judging = false
+        model.judging_error = "#{response.status} - #{response.body.truncate(500)}"
+      end
+    rescue StandardError => e
+      model.supports_judging = false
+      model.judging_error = e.message
+    end
+
+    def send_probe(model_id, input, max_tokens)
+      if @provider == "openai"
+        openai_probe(model_id, input, max_tokens)
+      else
+        anthropic_probe(model_id, input, max_tokens)
+      end
+    end
+
+    def extract_text(response)
+      data = JSON.parse(response.body)
+      if @provider == "openai"
+        data.dig("output", 0, "content", 0, "text")
+      else
+        data.dig("content", 0, "text")
+      end
+    end
+
+    def openai_probe(model_id, input, max_tokens)
+      conn = Faraday.new(url: "https://api.openai.com") do |f|
+        f.options.timeout = 15
+        f.options.open_timeout = 5
+        f.request :retry, max: 1, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+      conn.post do |req|
+        req.url "/v1/responses"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["Authorization"] = "Bearer #{@api_key}"
+        req.body = { model: model_id, input: input, max_output_tokens: max_tokens, store: false }.to_json
+      end
+    end
+
+    def anthropic_probe(model_id, input, max_tokens)
+      conn = Faraday.new(url: "https://api.anthropic.com") do |f|
+        f.options.timeout = 15
+        f.options.open_timeout = 5
+        f.request :retry, max: 1, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+      conn.post do |req|
+        req.url "/v1/messages"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["x-api-key"] = @api_key
+        req.headers["anthropic-version"] = "2023-06-01"
+        req.body = { model: model_id, messages: [{ role: "user", content: input }], max_tokens: max_tokens }.to_json
+      end
+    end
+  end
+end

data/app/services/completion_kit/ollama_client.rb

@@ -0,0 +1,80 @@
+module CompletionKit
+  class OllamaClient < LlmClient
+    def generate_completion(prompt, options = {})
+      return "Error: API endpoint not configured" unless configured?
+
+      require "faraday"
+      require "faraday/retry"
+      require "json"
+
+      model = options[:model]
+      max_tokens = options[:max_tokens] || 1000
+      temperature = options[:temperature] || 0.7
+
+      conn = Faraday.new(url: api_endpoint) do |f|
+        f.request :retry, max: 2, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+
+      response = conn.post do |req|
+        req.url "/v1/completions"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
+        req.body = {
+          model: model,
+          prompt: prompt,
+          max_tokens: max_tokens,
+          temperature: temperature
+        }.to_json
+      end
+
+      if response.success?
+        data = JSON.parse(response.body)
+        data["choices"][0]["text"].strip
+      else
+        "Error: #{response.status} - #{response.body}"
+      end
+    rescue => e
+      "Error: #{e.message}"
+    end
+
+    def available_models
+      return [] unless configured?
+
+      require "faraday"
+      require "faraday/retry"
+      require "json"
+
+      response = Faraday.get("#{api_endpoint}/v1/models") do |req|
+        req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
+      end
+
+      return [] unless response.success?
+
+      models = JSON.parse(response.body).fetch("data", []).map { |entry| entry["id"] }.sort
+      models.map { |id| { id: id, name: id } }
+    rescue StandardError
+      []
+    end
+
+    def configured?
+      api_endpoint.present?
+    end
+
+    def configuration_errors
+      errors = []
+      errors << "Ollama API endpoint is not configured" unless api_endpoint.present?
+      errors
+    end
+
+    private
+
+    def api_key
+      @config[:api_key] || ENV["OLLAMA_API_KEY"]
+    end
+
+    def api_endpoint
+      (@config[:api_endpoint] || ENV["OLLAMA_API_ENDPOINT"] || "http://localhost:11434/v1").to_s.delete_suffix("/")
+    end
+  end
+end

data/app/services/completion_kit/open_ai_client.rb

@@ -0,0 +1,71 @@
+module CompletionKit
+  class OpenAiClient < LlmClient
+    STATIC_MODELS = [
+      { id: "gpt-5.4-mini", name: "GPT-5.4 Mini" },
+      { id: "gpt-4.1-mini", name: "GPT-4.1 Mini" },
+      { id: "gpt-4o-mini", name: "GPT-4o Mini" }
+    ].freeze
+
+    def generate_completion(prompt, options = {})
+      return "Error: API key not configured" unless configured?
+
+      require "faraday"
+      require "faraday/retry"
+      require "json"
+
+      model = options[:model] || "gpt-4.1-mini"
+      max_tokens = options[:max_tokens] || 1000
+      temperature = options[:temperature] || 0.7
+
+      conn = Faraday.new(url: "https://api.openai.com") do |f|
+        f.request :retry, max: 2, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+
+      response = conn.post do |req|
+        req.url "/v1/responses"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["Authorization"] = "Bearer #{api_key}"
+        req.body = {
+          model: model,
+          input: prompt,
+          instructions: "You are a helpful assistant.",
+          max_output_tokens: max_tokens,
+          temperature: temperature,
+          store: false
+        }.to_json
+      end
+
+      if response.success?
+        data = JSON.parse(response.body)
+        data["output"][0]["content"][0]["text"].strip
+      else
+        "Error: #{response.status} - #{response.body}"
+      end
+    rescue Faraday::Error => e
+      raise
+    rescue => e
+      "Error: #{e.message}"
+    end
+
+    def available_models
+      STATIC_MODELS
+    end
+
+    def configured?
+      api_key.present?
+    end
+
+    def configuration_errors
+      errors = []
+      errors << "OpenAI API key is not configured" unless api_key.present?
+      errors
+    end
+
+    private
+
+    def api_key
+      @config[:api_key] || ENV["OPENAI_API_KEY"]
+    end
+  end
+end

data/app/services/completion_kit/open_router_client.rb

@@ -0,0 +1,69 @@
+module CompletionKit
+  class OpenRouterClient < LlmClient
+    BASE_URL = "https://openrouter.ai/api/v1".freeze
+    REFERER = "https://completionkit.com".freeze
+    APP_TITLE = "CompletionKit".freeze
+
+    def generate_completion(prompt, options = {})
+      return "Error: API key not configured" unless configured?
+
+      require "faraday"
+      require "faraday/retry"
+      require "json"
+
+      model = options[:model] || "openai/gpt-4o-mini"
+      max_tokens = options[:max_tokens] || 1000
+      temperature = options[:temperature] || 0.7
+
+      conn = Faraday.new(url: BASE_URL) do |f|
+        f.options.timeout = 30
+        f.options.open_timeout = 5
+        f.request :retry, max: 2, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+
+      response = conn.post do |req|
+        req.url "/chat/completions"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["Authorization"] = "Bearer #{api_key}"
+        req.headers["HTTP-Referer"] = REFERER
+        req.headers["X-Title"] = APP_TITLE
+        req.body = {
+          model: model,
+          messages: [{ role: "user", content: prompt }],
+          max_tokens: max_tokens,
+          temperature: temperature
+        }.to_json
+      end
+
+      if response.success?
+        data = JSON.parse(response.body)
+        data.dig("choices", 0, "message", "content").to_s.strip
+      else
+        "Error: #{response.status} - #{response.body}"
+      end
+    rescue => e
+      "Error: #{e.message}"
+    end
+
+    def available_models
+      []
+    end
+
+    def configured?
+      api_key.present?
+    end
+
+    def configuration_errors
+      errors = []
+      errors << "OpenRouter API key is not configured" unless api_key.present?
+      errors
+    end
+
+    private
+
+    def api_key
+      @config[:api_key] || ENV["OPENROUTER_API_KEY"]
+    end
+  end
+end

data/app/services/completion_kit/prompt_improvement_service.rb

@@ -0,0 +1,81 @@
+module CompletionKit
+  class PromptImprovementService
+    def initialize(run)
+      @run = run
+      @prompt = run.prompt
+    end
+
+    def suggest
+      client = LlmClient.for_model(@prompt.llm_model, ApiConfig.for_model(@prompt.llm_model))
+      response = client.generate_completion(build_meta_prompt, model: @prompt.llm_model, max_tokens: 2000, temperature: 0.4)
+      parse_response(response)
+    end
+
+    private
+
+    def build_meta_prompt
+      sections = []
+      sections << "You are an expert prompt engineer. Analyze the following prompt and its test results, then suggest an improved version."
+      sections << ""
+      sections << "## Current Prompt"
+      sections << "```"
+      sections << @prompt.template
+      sections << "```"
+      sections << ""
+      sections << "## Test Results"
+      sections << ""
+
+      reviews_by_response = @run.responses.includes(:reviews).limit(20)
+
+      reviews_by_response.each_with_index do |resp, i|
+        sections << "### Response #{i + 1}"
+        if resp.input_data.present?
+          sections << "Input: #{resp.input_data.truncate(200)}"
+        end
+        sections << "Output: #{resp.response_text.to_s.truncate(300)}"
+        if resp.expected_output.present?
+          sections << "Expected: #{resp.expected_output.truncate(200)}"
+        end
+        resp.reviews.each do |review|
+          sections << "  #{review.metric_name}: #{review.ai_score}/5 — #{review.ai_feedback}"
+        end
+        sections << ""
+      end
+
+      avg = @run.avg_score
+      sections << "## Overall Score: #{avg}/5" if avg
+
+      metric_avgs = @run.metric_averages
+      if metric_avgs.any?
+        sections << "## Metric Averages"
+        metric_avgs.each { |m| sections << "  #{m[:name]}: #{m[:avg]}/5" }
+        sections << ""
+      end
+
+      sections << "## Instructions"
+      sections << "Based on the test results above, suggest an improved version of the prompt."
+      sections << "Focus on addressing the weakest scoring areas while preserving what works well."
+      sections << ""
+      sections << "Respond in EXACTLY this format:"
+      sections << ""
+      sections << "REASONING:"
+      sections << "<2-4 bullet points explaining what you'd change and why>"
+      sections << ""
+      sections << "IMPROVED_PROMPT:"
+      sections << "<the full improved prompt template, preserving all {{variable}} placeholders>"
+
+      sections.join("\n")
+    end
+
+    def parse_response(text)
+      reasoning_match = text.match(/REASONING:\s*\n(.*?)(?=IMPROVED_PROMPT:)/m)
+      prompt_match = text.match(/IMPROVED_PROMPT:\s*\n(.*)/m)
+
+      {
+        "reasoning" => reasoning_match ? reasoning_match[1].strip : "No reasoning provided.",
+        "suggested_template" => prompt_match ? prompt_match[1].strip : text.strip,
+        "original_template" => @prompt.template
+      }
+    end
+  end
+end

data/app/views/completion_kit/api_reference/_example.html.erb

@@ -0,0 +1,6 @@
+<div class="ck-api-example">
+  <pre class="ck-code ck-code--dark"><%= cmd %></pre>
+  <button type="button" class="ck-api-copy" aria-label="Copy example to clipboard" onclick="ckCopyExample(this)" data-real-token="<%= real_token %>" data-display-token="<%= token %>">
+    <%= heroicon_tag "clipboard-document", variant: :outline, size: 14, "aria-hidden": "true" %>
+  </button>
+</div>