completion-kit 0.1.0.rc1 → 0.2.0

data/app/models/completion_kit/run.rb

@@ -21,6 +21,14 @@ module CompletionKit
       judge_model.present? && metrics.any? && ApiConfig.valid_for_model?(judge_model)
     end
 
+    def replace_metrics!(metric_ids)
+      return unless metric_ids
+      run_metrics.delete_all
+      Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
+        run_metrics.create!(metric_id: metric_id, position: index + 1)
+      end
+    end
+
     def avg_score
       all_reviews = responses.flat_map(&:reviews)
       scores = all_reviews.map(&:ai_score).compact.map(&:to_f)
@@ -113,16 +121,15 @@ module CompletionKit
             response.response_text,
             response.expected_output,
             prompt.template,
-            criteria: metric.respond_to?(:instruction) ? metric.instruction.to_s : "",
-            evaluation_steps: metric.respond_to?(:evaluation_steps) ? metric.evaluation_steps : nil,
-            rubric_text: metric.respond_to?(:display_rubric_text) ? metric.display_rubric_text : nil,
+            criteria: metric.instruction.to_s,
+            rubric_text: metric.display_rubric_text,
             input_data: response.input_data
           )
 
           response.reviews.find_or_initialize_by(metric_id: metric.id).tap do |review|
             review.assign_attributes(
               metric_name: metric.name,
-              instruction: metric.respond_to?(:instruction) ? metric.instruction.to_s : "",
+              instruction: metric.instruction.to_s,
               status: "evaluated",
               ai_score: evaluation[:score],
               ai_feedback: evaluation[:feedback]

data/app/services/completion_kit/anthropic_client.rb

@@ -7,21 +7,12 @@ module CompletionKit
 
     def generate_completion(prompt, options = {})
       return "Error: API key not configured" unless configured?
-      
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-      
+
       model = options[:model] || "claude-3-7-sonnet-latest"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
-      
-      conn = Faraday.new(url: "https://api.anthropic.com") do |f|
-        f.request :retry, max: 2, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-      
-      response = conn.post do |req|
+
+      response = build_connection("https://api.anthropic.com").post do |req|
         req.url "/v1/messages"
         req.headers["Content-Type"] = "application/json"
         req.headers["x-api-key"] = api_key
@@ -49,11 +40,7 @@ module CompletionKit
     def available_models
       return STATIC_MODELS unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
-      response = Faraday.get("https://api.anthropic.com/v1/models?limit=100") do |req|
+      response = build_connection("https://api.anthropic.com").get("/v1/models?limit=100") do |req|
         req.headers["x-api-key"] = api_key
         req.headers["anthropic-version"] = "2023-06-01"
       end

data/app/services/completion_kit/judge_service.rb

@@ -8,11 +8,11 @@ module CompletionKit
       @judge_client = LlmClient.for_model(@judge_model, ApiConfig.for_model(@judge_model))
     end
 
-    def evaluate(output, expected_output = nil, prompt = nil, criteria: nil, evaluation_steps: nil, rubric_text: nil, human_examples: nil, input_data: nil, **_extras)
+    def evaluate(output, expected_output = nil, prompt = nil, criteria: nil, rubric_text: nil, human_examples: nil, input_data: nil, **_extras)
       return { score: 1, feedback: "Judge not configured" } unless @judge_client.configured?
 
       judge_prompt = build_judge_prompt(output, expected_output, prompt,
-        criteria: criteria, evaluation_steps: evaluation_steps,
+        criteria: criteria,
         rubric_text: rubric_text, human_examples: human_examples,
         input_data: input_data)
 
@@ -27,7 +27,7 @@ module CompletionKit
 
     private
 
-    def build_judge_prompt(output, expected_output, prompt, criteria: nil, evaluation_steps: nil, rubric_text: nil, human_examples: nil, input_data: nil)
+    def build_judge_prompt(output, expected_output, prompt, criteria: nil, rubric_text: nil, human_examples: nil, input_data: nil)
       judge_prompt = <<~PROMPT
         You are an expert evaluator. You MUST respond with ONLY two lines in this exact format, nothing else:
 
@@ -44,10 +44,6 @@ module CompletionKit
         judge_prompt += "\nCriteria: #{criteria}\n"
       end
 
-      if evaluation_steps.present? && evaluation_steps.any?
-        judge_prompt += "\nEvaluation steps:\n#{evaluation_steps.each_with_index.map { |step, i| "#{i + 1}. #{step}" }.join("\n")}\n"
-      end
-
       if human_examples.present?
         judge_prompt += "\nCalibration examples:\n"
         human_examples.each_with_index do |example, index|

data/app/services/completion_kit/llm_client.rb

@@ -1,3 +1,7 @@
+require "faraday"
+require "faraday/retry"
+require "json"
+
 module CompletionKit
   class LlmClient
     def initialize(config = {})
@@ -41,5 +45,16 @@ module CompletionKit
 
       for_provider(provider, config)
     end
+
+    protected
+
+    def build_connection(url, timeout: nil, open_timeout: nil)
+      Faraday.new(url: url) do |f|
+        f.options.timeout = timeout if timeout
+        f.options.open_timeout = open_timeout if open_timeout
+        f.request :retry, max: 2, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+    end
   end
 end

data/app/services/completion_kit/mcp_dispatcher.rb

@@ -33,7 +33,7 @@ module CompletionKit
         McpTools::Responses.definitions +
         McpTools::Datasets.definitions +
         McpTools::Metrics.definitions +
-        McpTools::Criteria.definitions +
+        McpTools::MetricGroups.definitions +
         McpTools::ProviderCredentials.definitions
     end
 
@@ -44,7 +44,7 @@ module CompletionKit
       when /\Aresponses_/            then McpTools::Responses.call(name, arguments)
       when /\Adatasets_/             then McpTools::Datasets.call(name, arguments)
       when /\Ametrics_/              then McpTools::Metrics.call(name, arguments)
-      when /\Acriteria_/             then McpTools::Criteria.call(name, arguments)
+      when /\Ametric_groups_/        then McpTools::MetricGroups.call(name, arguments)
       when /\Aprovider_credentials_/ then McpTools::ProviderCredentials.call(name, arguments)
       else raise MethodNotFound, "Unknown tool: #{name}"
       end

data/app/services/completion_kit/mcp_tools/base.rb

@@ -0,0 +1,23 @@
+module CompletionKit
+  module McpTools
+    module Base
+      def definitions
+        self::TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def call(name, arguments)
+        tool = self::TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def text_result(data)
+        text = data.is_a?(String) ? data : data.to_json
+        {content: [{type: "text", text: text}]}
+      end
+
+      def error_result(message)
+        {content: [{type: "text", text: message}], isError: true}
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/datasets.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Datasets
+      extend Base
+
       TOOLS = {
         "datasets_list" => {
           description: "List all datasets",
@@ -37,15 +39,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(Dataset.order(created_at: :desc).map(&:as_json))
       end
@@ -76,15 +69,6 @@ module CompletionKit
         Dataset.find(args["id"]).destroy!
         text_result("Dataset #{args["id"]} deleted")
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/metric_groups.rb

@@ -0,0 +1,82 @@
+module CompletionKit
+  module McpTools
+    module MetricGroups
+      extend Base
+
+      TOOLS = {
+        "metric_groups_list" => {
+          description: "List all metric groups",
+          inputSchema: {type: "object", properties: {}, required: []},
+          handler: :list
+        },
+        "metric_groups_get" => {
+          description: "Get a metric group by ID",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :get
+        },
+        "metric_groups_create" => {
+          description: "Create a metric group",
+          inputSchema: {
+            type: "object",
+            properties: {
+              name: {type: "string"}, description: {type: "string"},
+              metric_ids: {type: "array", items: {type: "integer"}}
+            },
+            required: ["name"]
+          },
+          handler: :create
+        },
+        "metric_groups_update" => {
+          description: "Update a metric group",
+          inputSchema: {
+            type: "object",
+            properties: {
+              id: {type: "integer"}, name: {type: "string"}, description: {type: "string"},
+              metric_ids: {type: "array", items: {type: "integer"}}
+            },
+            required: ["id"]
+          },
+          handler: :update
+        },
+        "metric_groups_delete" => {
+          description: "Delete a metric group",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :delete
+        }
+      }.freeze
+
+      def self.list(_args)
+        text_result(CompletionKit::MetricGroup.order(created_at: :desc).map(&:as_json))
+      end
+
+      def self.get(args)
+        text_result(CompletionKit::MetricGroup.find(args["id"]).as_json)
+      end
+
+      def self.create(args)
+        metric_group = CompletionKit::MetricGroup.new(args.slice("name", "description"))
+        if metric_group.save
+          metric_group.replace_metrics!(args["metric_ids"])
+          text_result(metric_group.reload.as_json)
+        else
+          error_result(metric_group.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.update(args)
+        metric_group = CompletionKit::MetricGroup.find(args["id"])
+        if metric_group.update(args.except("id", "metric_ids").slice("name", "description"))
+          metric_group.replace_metrics!(args["metric_ids"]) if args.key?("metric_ids")
+          text_result(metric_group.reload.as_json)
+        else
+          error_result(metric_group.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.delete(args)
+        CompletionKit::MetricGroup.find(args["id"]).destroy!
+        text_result("Metric group #{args["id"]} deleted")
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/metrics.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Metrics
+      extend Base
+
       TOOLS = {
         "metrics_list" => {
           description: "List all metrics",
@@ -18,7 +20,6 @@ module CompletionKit
             type: "object",
             properties: {
               name: {type: "string"}, instruction: {type: "string"},
-              evaluation_steps: {type: "array", items: {type: "string"}},
               rubric_bands: {type: "array", items: {type: "object", properties: {stars: {type: "integer"}, description: {type: "string"}}}}
             },
             required: ["name"]
@@ -31,7 +32,6 @@ module CompletionKit
             type: "object",
             properties: {
               id: {type: "integer"}, name: {type: "string"}, instruction: {type: "string"},
-              evaluation_steps: {type: "array", items: {type: "string"}},
               rubric_bands: {type: "array", items: {type: "object", properties: {stars: {type: "integer"}, description: {type: "string"}}}}
             },
             required: ["id"]
@@ -45,15 +45,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(Metric.order(created_at: :desc).map(&:as_json))
       end
@@ -63,7 +54,7 @@ module CompletionKit
       end
 
       def self.create(args)
-        metric = Metric.new(args.slice("name", "instruction", "evaluation_steps", "rubric_bands"))
+        metric = Metric.new(args.slice("name", "instruction", "rubric_bands"))
         if metric.save
           text_result(metric.as_json)
         else
@@ -73,7 +64,7 @@ module CompletionKit
 
       def self.update(args)
         metric = Metric.find(args["id"])
-        if metric.update(args.except("id").slice("name", "instruction", "evaluation_steps", "rubric_bands"))
+        if metric.update(args.except("id").slice("name", "instruction", "rubric_bands"))
           text_result(metric.as_json)
         else
           error_result(metric.errors.full_messages.join(", "))
@@ -84,15 +75,6 @@ module CompletionKit
         Metric.find(args["id"]).destroy!
         text_result("Metric #{args["id"]} deleted")
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/prompts.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Prompts
+      extend Base
+
       TOOLS = {
         "prompts_list" => {
           description: "List all prompts",
@@ -48,15 +50,6 @@ module CompletionKit
         },
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(Prompt.order(created_at: :desc).map(&:as_json))
       end
@@ -98,15 +91,6 @@ module CompletionKit
         prompt.publish!
         text_result(prompt.reload.as_json)
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/provider_credentials.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module ProviderCredentials
+      extend Base
+
       TOOLS = {
         "provider_credentials_list" => {
           description: "List all provider credentials (API keys are not exposed)",
@@ -44,15 +46,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(ProviderCredential.order(created_at: :desc).map(&:as_json))
       end
@@ -83,15 +76,6 @@ module CompletionKit
         ProviderCredential.find(args["id"]).destroy!
         text_result("Provider credential #{args["id"]} deleted")
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/responses.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Responses
+      extend Base
+
       TOOLS = {
         "responses_list" => {
           description: "List responses for a run",
@@ -18,15 +20,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(args)
         run = Run.find(args["run_id"])
         text_result(run.responses.includes(:reviews).map(&:as_json))
@@ -36,10 +29,6 @@ module CompletionKit
         run = Run.find(args["run_id"])
         text_result(run.responses.find(args["id"]).as_json)
       end
-
-      def self.text_result(data)
-        {content: [{type: "text", text: data.to_json}]}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Runs
+      extend Base
+
       TOOLS = {
         "runs_list" => {
           description: "List all runs",
@@ -55,15 +57,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(Run.order(created_at: :desc).map(&:as_json))
       end
@@ -75,7 +68,7 @@ module CompletionKit
       def self.create(args)
         run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model"))
         if run.save
-          replace_run_metrics(run, args["metric_ids"])
+          run.replace_metrics!(args["metric_ids"])
           text_result(run.reload.as_json)
         else
           error_result(run.errors.full_messages.join(", "))
@@ -85,7 +78,7 @@ module CompletionKit
       def self.update(args)
         run = Run.find(args["id"])
         if run.update(args.except("id", "metric_ids").slice("name", "dataset_id", "judge_model"))
-          replace_run_metrics(run, args["metric_ids"]) if args.key?("metric_ids")
+          run.replace_metrics!(args["metric_ids"]) if args.key?("metric_ids")
           text_result(run.reload.as_json)
         else
           error_result(run.errors.full_messages.join(", "))
@@ -108,23 +101,6 @@ module CompletionKit
         JudgeJob.perform_later(run.id)
         text_result(run.reload.as_json)
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
-
-      def self.replace_run_metrics(run, metric_ids)
-        return unless metric_ids
-        run.run_metrics.delete_all
-        Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
-          run.run_metrics.create!(metric_id: metric_id, position: index + 1)
-        end
-      end
     end
   end
 end

data/app/services/completion_kit/ollama_client.rb

@@ -3,20 +3,11 @@ module CompletionKit
     def generate_completion(prompt, options = {})
       return "Error: API endpoint not configured" unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
       model = options[:model]
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      conn = Faraday.new(url: api_endpoint) do |f|
-        f.request :retry, max: 2, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-
-      response = conn.post do |req|
+      response = build_connection(api_endpoint).post do |req|
         req.url "/v1/completions"
         req.headers["Content-Type"] = "application/json"
         req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
@@ -41,11 +32,7 @@ module CompletionKit
     def available_models
       return [] unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
-      response = Faraday.get("#{api_endpoint}/v1/models") do |req|
+      response = build_connection(api_endpoint).get("/v1/models") do |req|
         req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
       end
 

data/app/services/completion_kit/open_ai_client.rb

@@ -9,20 +9,11 @@ module CompletionKit
     def generate_completion(prompt, options = {})
       return "Error: API key not configured" unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
       model = options[:model] || "gpt-4.1-mini"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      conn = Faraday.new(url: "https://api.openai.com") do |f|
-        f.request :retry, max: 2, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-
-      response = conn.post do |req|
+      response = build_connection("https://api.openai.com").post do |req|
         req.url "/v1/responses"
         req.headers["Content-Type"] = "application/json"
         req.headers["Authorization"] = "Bearer #{api_key}"

data/app/services/completion_kit/open_router_client.rb

@@ -7,22 +7,11 @@ module CompletionKit
     def generate_completion(prompt, options = {})
       return "Error: API key not configured" unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
       model = options[:model] || "openai/gpt-4o-mini"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      conn = Faraday.new(url: BASE_URL) do |f|
-        f.options.timeout = 30
-        f.options.open_timeout = 5
-        f.request :retry, max: 2, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-
-      response = conn.post do |req|
+      response = build_connection(BASE_URL, timeout: 30, open_timeout: 5).post do |req|
         req.url "/chat/completions"
         req.headers["Content-Type"] = "application/json"
         req.headers["Authorization"] = "Bearer #{api_key}"

data/app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb

@@ -0,0 +1,15 @@
+module CompletionKit
+  class TenantScopedUniquenessValidator < ActiveRecord::Validations::UniquenessValidator
+    def validate_each(record, attribute, value)
+      extra = Array(CompletionKit.config.tenant_scope_columns)
+      return super if extra.empty? && options[:scope].nil?
+
+      merged = options.merge(
+        scope: Array(options[:scope]) + extra,
+        attributes: [attribute],
+        class: @klass
+      )
+      self.class.superclass.new(merged).validate(record)
+    end
+  end
+end