completion-kit 0.1.0.rc1

data/app/services/completion_kit/anthropic_client.rb

@@ -0,0 +1,86 @@
+module CompletionKit
+  class AnthropicClient < LlmClient
+    STATIC_MODELS = [
+      { id: "claude-3-7-sonnet-latest", name: "Claude 3.7 Sonnet" },
+      { id: "claude-3-5-haiku-latest", name: "Claude 3.5 Haiku" }
+    ].freeze
+
+    def generate_completion(prompt, options = {})
+      return "Error: API key not configured" unless configured?
+      
+      require "faraday"
+      require "faraday/retry"
+      require "json"
+      
+      model = options[:model] || "claude-3-7-sonnet-latest"
+      max_tokens = options[:max_tokens] || 1000
+      temperature = options[:temperature] || 0.7
+      
+      conn = Faraday.new(url: "https://api.anthropic.com") do |f|
+        f.request :retry, max: 2, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+      
+      response = conn.post do |req|
+        req.url "/v1/messages"
+        req.headers["Content-Type"] = "application/json"
+        req.headers["x-api-key"] = api_key
+        req.headers["anthropic-version"] = "2023-06-01"
+        req.body = {
+          model: model,
+          messages: [
+            { role: "user", content: prompt }
+          ],
+          max_tokens: max_tokens,
+          temperature: temperature
+        }.to_json
+      end
+      
+      if response.success?
+        data = JSON.parse(response.body)
+        data["content"][0]["text"].strip
+      else
+        "Error: #{response.status} - #{response.body}"
+      end
+    rescue => e
+      "Error: #{e.message}"
+    end
+
+    def available_models
+      return STATIC_MODELS unless configured?
+
+      require "faraday"
+      require "faraday/retry"
+      require "json"
+
+      response = Faraday.get("https://api.anthropic.com/v1/models?limit=100") do |req|
+        req.headers["x-api-key"] = api_key
+        req.headers["anthropic-version"] = "2023-06-01"
+      end
+
+      return STATIC_MODELS unless response.success?
+
+      entries = JSON.parse(response.body).fetch("data", [])
+      models = entries.map { |entry| { id: entry["id"], name: entry["display_name"] || entry["id"] } }
+      models.presence || STATIC_MODELS
+    rescue StandardError
+      STATIC_MODELS
+    end
+
+    def configured?
+      api_key.present?
+    end
+
+    def configuration_errors
+      errors = []
+      errors << "Anthropic API key is not configured" unless api_key.present?
+      errors
+    end
+
+    private
+
+    def api_key
+      @config[:api_key] || ENV["ANTHROPIC_API_KEY"]
+    end
+  end
+end

data/app/services/completion_kit/api_config.rb

@@ -0,0 +1,80 @@
+module CompletionKit
+  class ApiConfig
+    def self.for_model(model_name)
+      provider = provider_for_model(model_name)
+      provider ? for_provider(provider) : {}
+    end
+
+    def self.for_provider(provider_name)
+      provider = provider_name.to_s
+      stored = ProviderCredential.find_by(provider: provider)&.config_hash || {}
+
+      defaults = case provider
+                 when "openai"
+                   { provider: "openai", api_key: CompletionKit.config.openai_api_key || ENV["OPENAI_API_KEY"] }
+                 when "anthropic"
+                   { provider: "anthropic", api_key: CompletionKit.config.anthropic_api_key || ENV["ANTHROPIC_API_KEY"] }
+                 when "ollama"
+                   {
+                     provider: "ollama",
+                     api_key: CompletionKit.config.ollama_api_key || ENV["OLLAMA_API_KEY"],
+                     api_endpoint: CompletionKit.config.ollama_api_endpoint || ENV["OLLAMA_API_ENDPOINT"]
+                   }
+                 when "openrouter"
+                   { provider: "openrouter", api_key: ENV["OPENROUTER_API_KEY"] }
+                 else
+                   {}
+                 end
+
+      defaults.merge(stored.compact)
+    end
+
+    def self.provider_for_model(model_name)
+      available_match = available_models.find { |model| model[:id] == model_name.to_s }
+      return available_match[:provider] if available_match
+
+      case model_name.to_s
+      when /\Agpt-/
+        "openai"
+      when /\Aclaude-/
+        "anthropic"
+      else
+        nil
+      end
+    end
+
+    def self.valid_for_model?(model_name)
+      client = LlmClient.for_model(model_name, for_model(model_name))
+      client.configured?
+    end
+
+    def self.errors_for_model(model_name)
+      client = LlmClient.for_model(model_name, for_model(model_name))
+      client.configuration_errors
+    end
+
+    def self.available_models(provider: nil, scope: :generation)
+      query = case scope
+              when :judging then Model.for_judging
+              when :generation then Model.for_generation
+              else Model.active
+              end
+      query = query.where(provider: provider) if provider.present?
+      models = query.order(:provider, :display_name).map do |m|
+        { id: m.model_id, name: m.display_name || m.model_id, provider: m.provider }
+      end
+
+      return models if models.any?
+
+      configured = ProviderCredential.pluck(:provider)
+      providers = provider.present? ? [provider.to_s] : configured
+      providers.flat_map do |provider_name|
+        next [] unless configured.include?(provider_name)
+        client = LlmClient.for_provider(provider_name, for_provider(provider_name))
+        client.available_models.map { |model| model.symbolize_keys.merge(provider: provider_name) }
+      rescue StandardError
+        []
+      end.uniq { |model| model[:id] }
+    end
+  end
+end

data/app/services/completion_kit/csv_processor.rb

@@ -0,0 +1,65 @@
+module CompletionKit
+  class CsvProcessor
+    require 'csv'
+
+    def self.process(run)
+      return [] if run.csv_data.blank?
+
+      begin
+        csv_data = CSV.parse(run.csv_data, headers: true)
+        rows = csv_data.map(&:to_h)
+
+        if rows.empty?
+          run.errors.add(:csv_data, "No data rows found in CSV")
+          return []
+        end
+
+        return [] unless validate_variables(run, rows.first.keys)
+
+        rows
+      rescue CSV::MalformedCSVError => e
+        run.errors.add(:csv_data, "Invalid CSV format: #{e.message}")
+        []
+      end
+    end
+
+    def self.process_self(run)
+      return [] unless run.dataset&.csv_data.present?
+
+      begin
+        csv_data = CSV.parse(run.dataset.csv_data, headers: true)
+        csv_data.map(&:to_h)
+      rescue CSV::MalformedCSVError
+        []
+      end
+    end
+
+    def self.extract_variables(prompt)
+      return [] if prompt.nil? || prompt.template.blank?
+
+      prompt.template.scan(/\{\{([^}]+)\}\}/).flatten.map(&:strip).uniq
+    end
+
+    def self.validate_variables(run, headers)
+      prompt_variables = extract_variables(run.prompt)
+      missing_variables = prompt_variables - headers
+
+      if missing_variables.any?
+        run.errors.add(:csv_data, "Missing required variables in CSV: #{missing_variables.join(', ')}")
+        return false
+      end
+
+      true
+    end
+
+    def self.apply_variables(prompt, variables)
+      result = prompt.template.dup
+
+      variables.each do |name, value|
+        result.gsub!(/\{\{\s*#{Regexp.escape(name.to_s)}\s*\}\}/, value.to_s)
+      end
+
+      result
+    end
+  end
+end

data/app/services/completion_kit/judge_service.rb

@@ -0,0 +1,87 @@
+require "faraday"
+
+module CompletionKit
+  class JudgeService
+    def initialize(config = {})
+      @config = config
+      @judge_model = config[:judge_model] || CompletionKit.config.judge_model
+      @judge_client = LlmClient.for_model(@judge_model, ApiConfig.for_model(@judge_model))
+    end
+
+    def evaluate(output, expected_output = nil, prompt = nil, criteria: nil, evaluation_steps: nil, rubric_text: nil, human_examples: nil, input_data: nil, **_extras)
+      return { score: 1, feedback: "Judge not configured" } unless @judge_client.configured?
+
+      judge_prompt = build_judge_prompt(output, expected_output, prompt,
+        criteria: criteria, evaluation_steps: evaluation_steps,
+        rubric_text: rubric_text, human_examples: human_examples,
+        input_data: input_data)
+
+      response = @judge_client.generate_completion(judge_prompt, model: @judge_model)
+      raise StandardError, response if response.start_with?("Error:")
+      parse_judge_response(response)
+    rescue Faraday::Error
+      raise
+    rescue => e
+      { score: 1, feedback: "Error during evaluation: #{e.message}" }
+    end
+
+    private
+
+    def build_judge_prompt(output, expected_output, prompt, criteria: nil, evaluation_steps: nil, rubric_text: nil, human_examples: nil, input_data: nil)
+      judge_prompt = <<~PROMPT
+        You are an expert evaluator. You MUST respond with ONLY two lines in this exact format, nothing else:
+
+        Score: <integer from 1 to 5>
+        Feedback: <one sentence explaining why>
+
+        Do not include any other text, markdown, or explanation. Just those two lines.
+
+        Use this rubric to choose the score:
+        #{rubric_text.presence || CompletionKit::Metric.default_rubric_text}
+      PROMPT
+
+      if criteria.present?
+        judge_prompt += "\nCriteria: #{criteria}\n"
+      end
+
+      if evaluation_steps.present? && evaluation_steps.any?
+        judge_prompt += "\nEvaluation steps:\n#{evaluation_steps.each_with_index.map { |step, i| "#{i + 1}. #{step}" }.join("\n")}\n"
+      end
+
+      if human_examples.present?
+        judge_prompt += "\nCalibration examples:\n"
+        human_examples.each_with_index do |example, index|
+          judge_prompt += "Example #{index + 1}: score=#{example[:human_score]} output=#{example[:response_text].to_s.truncate(200)}\n"
+        end
+      end
+
+      judge_prompt += <<~PROMPT
+
+        Original prompt: #{prompt || "Not provided"}
+        #{input_data.present? ? "Input data: #{input_data}" : ""}
+        #{expected_output.present? ? "Expected output: #{expected_output}" : ""}
+        AI output to evaluate: #{output}
+      PROMPT
+
+      judge_prompt
+    end
+
+    def parse_judge_response(response)
+      score_match = response.match(/\*{0,2}Score:?\*{0,2}\s*(\d+(?:\.\d+)?)/i)
+      feedback_match = response.match(/\*{0,2}Feedback:?\*{0,2}\s*(.+)/mi)
+
+      score = score_match ? score_match[1].to_f : 1
+      feedback = if feedback_match
+                   feedback_match[1].strip
+                 elsif score_match
+                   "No feedback provided"
+                 else
+                   "Could not parse judge response: #{response.truncate(500)}"
+                 end
+
+      score = [[score, 1].max, 5].min
+
+      { score: score, feedback: feedback }
+    end
+  end
+end

data/app/services/completion_kit/llm_client.rb

@@ -0,0 +1,45 @@
+module CompletionKit
+  class LlmClient
+    def initialize(config = {})
+      @config = config
+    end
+
+    def generate_completion(prompt, options = {})
+      raise NotImplementedError, "Subclasses must implement generate_completion"
+    end
+
+    def available_models
+      raise NotImplementedError, "Subclasses must implement available_models"
+    end
+
+    def configured?
+      raise NotImplementedError, "Subclasses must implement configured?"
+    end
+
+    def configuration_errors
+      []
+    end
+
+    def self.for_provider(provider_name, config = {})
+      case provider_name.to_s
+      when "openai"
+        OpenAiClient.new(config)
+      when "anthropic"
+        AnthropicClient.new(config)
+      when "ollama"
+        OllamaClient.new(config)
+      when "openrouter"
+        OpenRouterClient.new(config)
+      else
+        raise ArgumentError, "Unsupported provider: #{provider_name}"
+      end
+    end
+
+    def self.for_model(model_name, config = {})
+      provider = ApiConfig.provider_for_model(model_name)
+      raise ArgumentError, "Unsupported model: #{model_name}" unless provider
+
+      for_provider(provider, config)
+    end
+  end
+end

data/app/services/completion_kit/mcp_dispatcher.rb

@@ -0,0 +1,53 @@
+module CompletionKit
+  class McpDispatcher
+    class MethodNotFound < StandardError; end
+    class InvalidParams < StandardError; end
+
+    PROTOCOL_VERSION = "2025-03-26"
+
+    def self.initialize_session
+      session_id = SecureRandom.uuid
+      Rails.cache.write("mcp_session:#{session_id}", true, expires_in: 1.hour)
+      {
+        session_id: session_id,
+        protocolVersion: PROTOCOL_VERSION,
+        serverInfo: {name: "CompletionKit", version: CompletionKit::VERSION},
+        capabilities: {tools: {listChanged: false}}
+      }
+    end
+
+    def self.dispatch(method, params)
+      case method
+      when "tools/list"
+        {tools: tool_definitions}
+      when "tools/call"
+        call_tool(params&.dig("name"), params&.dig("arguments") || {})
+      else
+        raise MethodNotFound, "Method not found: #{method}"
+      end
+    end
+
+    def self.tool_definitions
+      McpTools::Prompts.definitions +
+        McpTools::Runs.definitions +
+        McpTools::Responses.definitions +
+        McpTools::Datasets.definitions +
+        McpTools::Metrics.definitions +
+        McpTools::Criteria.definitions +
+        McpTools::ProviderCredentials.definitions
+    end
+
+    def self.call_tool(name, arguments)
+      case name
+      when /\Aprompts_/              then McpTools::Prompts.call(name, arguments)
+      when /\Aruns_/                 then McpTools::Runs.call(name, arguments)
+      when /\Aresponses_/            then McpTools::Responses.call(name, arguments)
+      when /\Adatasets_/             then McpTools::Datasets.call(name, arguments)
+      when /\Ametrics_/              then McpTools::Metrics.call(name, arguments)
+      when /\Acriteria_/             then McpTools::Criteria.call(name, arguments)
+      when /\Aprovider_credentials_/ then McpTools::ProviderCredentials.call(name, arguments)
+      else raise MethodNotFound, "Unknown tool: #{name}"
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/criteria.rb

@@ -0,0 +1,106 @@
+module CompletionKit
+  module McpTools
+    module Criteria
+      TOOLS = {
+        "criteria_list" => {
+          description: "List all criteria",
+          inputSchema: {type: "object", properties: {}, required: []},
+          handler: :list
+        },
+        "criteria_get" => {
+          description: "Get a criteria by ID",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :get
+        },
+        "criteria_create" => {
+          description: "Create a criteria grouping metrics",
+          inputSchema: {
+            type: "object",
+            properties: {
+              name: {type: "string"}, description: {type: "string"},
+              metric_ids: {type: "array", items: {type: "integer"}}
+            },
+            required: ["name"]
+          },
+          handler: :create
+        },
+        "criteria_update" => {
+          description: "Update a criteria",
+          inputSchema: {
+            type: "object",
+            properties: {
+              id: {type: "integer"}, name: {type: "string"}, description: {type: "string"},
+              metric_ids: {type: "array", items: {type: "integer"}}
+            },
+            required: ["id"]
+          },
+          handler: :update
+        },
+        "criteria_delete" => {
+          description: "Delete a criteria",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :delete
+        }
+      }.freeze
+
+      def self.definitions
+        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def self.call(name, arguments)
+        tool = TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def self.list(_args)
+        text_result(CompletionKit::Criteria.order(created_at: :desc).map(&:as_json))
+      end
+
+      def self.get(args)
+        text_result(CompletionKit::Criteria.find(args["id"]).as_json)
+      end
+
+      def self.create(args)
+        criteria = CompletionKit::Criteria.new(args.slice("name", "description"))
+        if criteria.save
+          replace_metric_memberships(criteria, args["metric_ids"])
+          text_result(criteria.reload.as_json)
+        else
+          error_result(criteria.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.update(args)
+        criteria = CompletionKit::Criteria.find(args["id"])
+        if criteria.update(args.except("id", "metric_ids").slice("name", "description"))
+          replace_metric_memberships(criteria, args["metric_ids"]) if args.key?("metric_ids")
+          text_result(criteria.reload.as_json)
+        else
+          error_result(criteria.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.delete(args)
+        CompletionKit::Criteria.find(args["id"]).destroy!
+        text_result("Criteria #{args["id"]} deleted")
+      end
+
+      def self.text_result(data)
+        text = data.is_a?(String) ? data : data.to_json
+        {content: [{type: "text", text: text}]}
+      end
+
+      def self.error_result(message)
+        {content: [{type: "text", text: message}], isError: true}
+      end
+
+      def self.replace_metric_memberships(criteria, metric_ids)
+        return unless metric_ids
+        criteria.criteria_memberships.delete_all
+        Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
+          criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
+        end
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/datasets.rb

@@ -0,0 +1,90 @@
+module CompletionKit
+  module McpTools
+    module Datasets
+      TOOLS = {
+        "datasets_list" => {
+          description: "List all datasets",
+          inputSchema: {type: "object", properties: {}, required: []},
+          handler: :list
+        },
+        "datasets_get" => {
+          description: "Get a dataset by ID",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :get
+        },
+        "datasets_create" => {
+          description: "Create a dataset with CSV data",
+          inputSchema: {
+            type: "object",
+            properties: {name: {type: "string"}, csv_data: {type: "string"}},
+            required: ["name", "csv_data"]
+          },
+          handler: :create
+        },
+        "datasets_update" => {
+          description: "Update a dataset",
+          inputSchema: {
+            type: "object",
+            properties: {id: {type: "integer"}, name: {type: "string"}, csv_data: {type: "string"}},
+            required: ["id"]
+          },
+          handler: :update
+        },
+        "datasets_delete" => {
+          description: "Delete a dataset",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :delete
+        }
+      }.freeze
+
+      def self.definitions
+        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def self.call(name, arguments)
+        tool = TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def self.list(_args)
+        text_result(Dataset.order(created_at: :desc).map(&:as_json))
+      end
+
+      def self.get(args)
+        text_result(Dataset.find(args["id"]).as_json)
+      end
+
+      def self.create(args)
+        dataset = Dataset.new(args.slice("name", "csv_data"))
+        if dataset.save
+          text_result(dataset.as_json)
+        else
+          error_result(dataset.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.update(args)
+        dataset = Dataset.find(args["id"])
+        if dataset.update(args.except("id").slice("name", "csv_data"))
+          text_result(dataset.as_json)
+        else
+          error_result(dataset.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.delete(args)
+        Dataset.find(args["id"]).destroy!
+        text_result("Dataset #{args["id"]} deleted")
+      end
+
+      def self.text_result(data)
+        text = data.is_a?(String) ? data : data.to_json
+        {content: [{type: "text", text: text}]}
+      end
+
+      def self.error_result(message)
+        {content: [{type: "text", text: message}], isError: true}
+      end
+    end
+  end
+end