completion-kit 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8b122b978bb3d74051e14d734203da11cd951ef6ea0cb60b0459215812000e9
-  data.tar.gz: 458b93ab81bf13dcaf1fd7431e6934899855b4e6513ef16e82b4cd085dcb3e67
+  metadata.gz: c87481def48cfd6193ee591e9ac1ab1a2be6b7de63a3275c82f7f7175804abfc
+  data.tar.gz: 333b23de10b7e81daeac7c118f2c4ae13b2624304a1d04404b206aa5dffce092
 SHA512:
-  metadata.gz: d3c49afda3bb03eda67c89df4b5a8144da2259e8002ff35e906fa7516e0d115c0c7e3549a6b2de78f5478b72fefdcf758a0f6e042f83d3befc430c0bb26e7fa0
-  data.tar.gz: ed5fb37dcacd8c3bc1947d75b7ff83e4abde0fbcc418944ac7f3f4dc837679eb6d5cce5c144546ddd59aa4bc6548bdb83ac03c4b13428b916ef6e40b0f510e10
+  metadata.gz: '0538aae286aeffe40644e580dc84a27ee1bc2a187fe0a060f33bc4dfb30e04a769f9daa2b8a59b19633d35b8f79172a40bcd03d75d136c5dc1d4a8af92e5aa4a'
+  data.tar.gz: 1843ea626685029288ec9533edc07a800bca18b8e435de22d1b6c32c59e4b2d52c08999833f7b123d983855e445a95ae814cb12a31aa21c041bac11d7a70c466

data/README.md

@@ -178,6 +178,22 @@ bin/rails db:migrate
 git add db/migrate/ && git commit -m "install new engine migration"
 ```
 
+## Multi-tenant host apps (advanced)
+
+For hosts that mount CompletionKit in a multi-tenant app, two optional hooks scope engine records per tenant without forking the engine:
+
+```ruby
+CompletionKit.configure do |config|
+  config.tenant_scope = -> {
+    org = Current.organization&.id
+    org ? where(organization_id: org) : where("1=0")
+  }
+  config.tenant_scope_columns = [:organization_id]
+end
+```
+
+`tenant_scope` runs as each engine model's `default_scope` (use `unscoped` to bypass). `tenant_scope_columns` is appended to every engine uniqueness validation. Adding the tenant columns and composite unique indexes lives in your host migrations. Both defaults (`nil`, `[]`) are no-ops.
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, testing, and pull request guidelines.

data/app/controllers/completion_kit/api/v1/metric_groups_controller.rb

@@ -15,7 +15,7 @@ module CompletionKit
         def create
           metric_group = MetricGroup.new(metric_group_params.except(:metric_ids))
           if metric_group.save
-            replace_metric_memberships(metric_group, params[:metric_ids]) if params.key?(:metric_ids)
+            metric_group.replace_metrics!(params[:metric_ids]) if params.key?(:metric_ids)
             render json: metric_group.reload, status: :created
           else
             render json: {errors: metric_group.errors}, status: :unprocessable_entity
@@ -24,7 +24,7 @@ module CompletionKit
 
         def update
           if @metric_group.update(metric_group_params.except(:metric_ids))
-            replace_metric_memberships(@metric_group, params[:metric_ids]) if params.key?(:metric_ids)
+            @metric_group.replace_metrics!(params[:metric_ids]) if params.key?(:metric_ids)
             render json: @metric_group.reload
           else
             render json: {errors: @metric_group.errors}, status: :unprocessable_entity
@@ -47,15 +47,6 @@ module CompletionKit
         def metric_group_params
           params.permit(:name, :description, metric_ids: [])
         end
-
-        def replace_metric_memberships(metric_group, metric_ids)
-          return unless metric_ids
-
-          metric_group.metric_group_memberships.delete_all
-          Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
-            metric_group.metric_group_memberships.create!(metric_id: metric_id, position: index + 1)
-          end
-        end
       end
     end
   end

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -15,7 +15,7 @@ module CompletionKit
         def create
           run = Run.new(run_params.except(:metric_ids))
           if run.save
-            replace_run_metrics(run, params[:metric_ids])
+            run.replace_metrics!(params[:metric_ids])
             render json: run.reload, status: :created
           else
             render json: {errors: run.errors}, status: :unprocessable_entity
@@ -24,7 +24,7 @@ module CompletionKit
 
         def update
           if @run.update(run_params.except(:metric_ids))
-            replace_run_metrics(@run, params[:metric_ids]) if params.key?(:metric_ids)
+            @run.replace_metrics!(params[:metric_ids]) if params.key?(:metric_ids)
             render json: @run.reload
           else
             render json: {errors: @run.errors}, status: :unprocessable_entity
@@ -57,14 +57,6 @@ module CompletionKit
         def run_params
           params.permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, metric_ids: [])
         end
-
-        def replace_run_metrics(run, metric_ids)
-          return unless metric_ids
-          run.run_metrics.delete_all
-          Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
-            run.run_metrics.create!(metric_id: metric_id, position: index + 1)
-          end
-        end
       end
     end
   end

data/app/controllers/completion_kit/metric_groups_controller.rb

@@ -23,7 +23,7 @@ module CompletionKit
       @metrics = Metric.order(:name)
 
       if @metric_group.save
-        replace_metric_memberships
+        @metric_group.replace_metrics!(metric_group_params[:metric_ids])
         redirect_to metric_group_path(@metric_group), notice: "Metric group was successfully created."
       else
         render :new, status: :unprocessable_entity
@@ -34,7 +34,7 @@ module CompletionKit
       @metrics = Metric.order(:name)
 
       if @metric_group.update(metric_group_params.except(:metric_ids))
-        replace_metric_memberships
+        @metric_group.replace_metrics!(metric_group_params[:metric_ids])
         redirect_to metric_group_path(@metric_group), notice: "Metric group was successfully updated."
       else
         render :edit, status: :unprocessable_entity
@@ -55,13 +55,5 @@ module CompletionKit
     def metric_group_params
       params.require(:metric_group).permit(:name, :description, metric_ids: [])
     end
-
-    def replace_metric_memberships
-      metric_ids = Array(metric_group_params[:metric_ids]).reject(&:blank?)
-      @metric_group.metric_group_memberships.delete_all
-      metric_ids.each_with_index do |metric_id, index|
-        @metric_group.metric_group_memberships.create!(metric_id: metric_id, position: index + 1)
-      end
-    end
   end
 end

data/app/controllers/completion_kit/runs_controller.rb

@@ -35,7 +35,7 @@ module CompletionKit
     def create
       @run = Run.new(run_params.except(:metric_ids))
       if @run.save
-        replace_run_metrics(@run, params[:run][:metric_ids])
+        @run.replace_metrics!(params[:run][:metric_ids])
         redirect_to run_path(@run), notice: "Run was successfully created."
       else
         load_form_collections
@@ -46,10 +46,10 @@ module CompletionKit
     def update
       if @run.responses.any?
         new_run = Run.create!(run_params.except(:metric_ids).to_h.merge(status: "pending"))
-        replace_run_metrics(new_run, params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
+        new_run.replace_metrics!(params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
         redirect_to run_path(new_run), notice: "Saved as a new run. The previous run and its results are preserved."
       elsif @run.update(run_params.except(:metric_ids))
-        replace_run_metrics(@run, params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
+        @run.replace_metrics!(params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
         redirect_to run_path(@run), notice: "Run saved."
       else
         load_form_collections
@@ -120,12 +120,5 @@ module CompletionKit
       params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, metric_ids: [])
     end
 
-    def replace_run_metrics(run, metric_ids)
-      return unless metric_ids
-      run.run_metrics.delete_all
-      Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
-        run.run_metrics.create!(metric_id: metric_id, position: index + 1)
-      end
-    end
   end
 end

data/app/helpers/completion_kit/application_helper.rb

@@ -76,15 +76,8 @@ module CompletionKit
       end
     end
 
-    PROVIDER_LABELS = {
-      "openai" => "OpenAI",
-      "anthropic" => "Anthropic",
-      "ollama" => "Ollama / local endpoint",
-      "openrouter" => "OpenRouter"
-    }.freeze
-
     def ck_provider_label(provider)
-      PROVIDER_LABELS[provider.to_s] || provider.to_s.titleize
+      CompletionKit::ProviderCredential::PROVIDER_LABELS[provider.to_s] || provider.to_s.titleize
     end
 
     def ck_grouped_models(models, selected = nil)

data/app/models/completion_kit/application_record.rb

@@ -1,5 +1,12 @@
 module CompletionKit
   class ApplicationRecord < ActiveRecord::Base
     self.abstract_class = true
+
+    TenantScopedUniquenessValidator = CompletionKit::TenantScopedUniquenessValidator
+
+    default_scope do
+      scope_proc = CompletionKit.config.tenant_scope
+      scope_proc ? instance_exec(&scope_proc) : all
+    end
   end
 end

data/app/models/completion_kit/metric.rb

@@ -15,7 +15,7 @@ module CompletionKit
     serialize :rubric_bands, coder: JSON
 
     validates :name, presence: true
-    validates :key, uniqueness: true, allow_nil: true
+    validates :key, tenant_scoped_uniqueness: { allow_nil: true }
 
     before_validation :generate_key
     before_validation :normalize_rubric_bands

data/app/models/completion_kit/metric_group.rb

@@ -11,6 +11,14 @@ module CompletionKit
       metric_group_memberships.includes(:metric).map(&:metric).compact
     end
 
+    def replace_metrics!(metric_ids)
+      return unless metric_ids
+      metric_group_memberships.delete_all
+      Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
+        metric_group_memberships.create!(metric_id: metric_id, position: index + 1)
+      end
+    end
+
     def as_json(options = {})
       {
         id: id, name: name, description: description,

data/app/models/completion_kit/model.rb

@@ -3,7 +3,7 @@ module CompletionKit
     STATUSES = %w[active retired failed].freeze
 
     validates :provider, presence: true
-    validates :model_id, presence: true, uniqueness: { scope: :provider }
+    validates :model_id, presence: true, tenant_scoped_uniqueness: { scope: :provider }
     validates :status, presence: true, inclusion: { in: STATUSES }
 
     scope :active, -> { where(status: "active") }

data/app/models/completion_kit/provider_credential.rb

@@ -22,7 +22,8 @@ module CompletionKit
       PROVIDER_LABELS[provider] || provider.titleize
     end
 
-    validates :provider, presence: true, inclusion: { in: PROVIDERS }, uniqueness: true
+    validates :provider, presence: true, inclusion: { in: PROVIDERS }
+    validates :provider, tenant_scoped_uniqueness: true
 
     after_save :enqueue_discovery
 

data/app/models/completion_kit/run.rb

@@ -21,6 +21,14 @@ module CompletionKit
       judge_model.present? && metrics.any? && ApiConfig.valid_for_model?(judge_model)
     end
 
+    def replace_metrics!(metric_ids)
+      return unless metric_ids
+      run_metrics.delete_all
+      Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
+        run_metrics.create!(metric_id: metric_id, position: index + 1)
+      end
+    end
+
     def avg_score
       all_reviews = responses.flat_map(&:reviews)
       scores = all_reviews.map(&:ai_score).compact.map(&:to_f)
@@ -113,15 +121,15 @@ module CompletionKit
             response.response_text,
             response.expected_output,
             prompt.template,
-            criteria: metric.respond_to?(:instruction) ? metric.instruction.to_s : "",
-            rubric_text: metric.respond_to?(:display_rubric_text) ? metric.display_rubric_text : nil,
+            criteria: metric.instruction.to_s,
+            rubric_text: metric.display_rubric_text,
             input_data: response.input_data
           )
 
           response.reviews.find_or_initialize_by(metric_id: metric.id).tap do |review|
             review.assign_attributes(
               metric_name: metric.name,
-              instruction: metric.respond_to?(:instruction) ? metric.instruction.to_s : "",
+              instruction: metric.instruction.to_s,
               status: "evaluated",
               ai_score: evaluation[:score],
               ai_feedback: evaluation[:feedback]

data/app/services/completion_kit/anthropic_client.rb

@@ -7,21 +7,12 @@ module CompletionKit
 
     def generate_completion(prompt, options = {})
       return "Error: API key not configured" unless configured?
-      
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-      
+
       model = options[:model] || "claude-3-7-sonnet-latest"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
-      
-      conn = Faraday.new(url: "https://api.anthropic.com") do |f|
-        f.request :retry, max: 2, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-      
-      response = conn.post do |req|
+
+      response = build_connection("https://api.anthropic.com").post do |req|
         req.url "/v1/messages"
         req.headers["Content-Type"] = "application/json"
         req.headers["x-api-key"] = api_key
@@ -49,11 +40,7 @@ module CompletionKit
     def available_models
       return STATIC_MODELS unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
-      response = Faraday.get("https://api.anthropic.com/v1/models?limit=100") do |req|
+      response = build_connection("https://api.anthropic.com").get("/v1/models?limit=100") do |req|
         req.headers["x-api-key"] = api_key
         req.headers["anthropic-version"] = "2023-06-01"
       end

data/app/services/completion_kit/llm_client.rb

@@ -1,3 +1,7 @@
+require "faraday"
+require "faraday/retry"
+require "json"
+
 module CompletionKit
   class LlmClient
     def initialize(config = {})
@@ -41,5 +45,16 @@ module CompletionKit
 
       for_provider(provider, config)
     end
+
+    protected
+
+    def build_connection(url, timeout: nil, open_timeout: nil)
+      Faraday.new(url: url) do |f|
+        f.options.timeout = timeout if timeout
+        f.options.open_timeout = open_timeout if open_timeout
+        f.request :retry, max: 2, interval: 0.5
+        f.adapter Faraday.default_adapter
+      end
+    end
   end
 end

data/app/services/completion_kit/mcp_tools/base.rb

@@ -0,0 +1,23 @@
+module CompletionKit
+  module McpTools
+    module Base
+      def definitions
+        self::TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def call(name, arguments)
+        tool = self::TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def text_result(data)
+        text = data.is_a?(String) ? data : data.to_json
+        {content: [{type: "text", text: text}]}
+      end
+
+      def error_result(message)
+        {content: [{type: "text", text: message}], isError: true}
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/datasets.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Datasets
+      extend Base
+
       TOOLS = {
         "datasets_list" => {
           description: "List all datasets",
@@ -37,15 +39,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(Dataset.order(created_at: :desc).map(&:as_json))
       end
@@ -76,15 +69,6 @@ module CompletionKit
         Dataset.find(args["id"]).destroy!
         text_result("Dataset #{args["id"]} deleted")
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/metric_groups.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module MetricGroups
+      extend Base
+
       TOOLS = {
         "metric_groups_list" => {
           description: "List all metric groups",
@@ -43,15 +45,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(CompletionKit::MetricGroup.order(created_at: :desc).map(&:as_json))
       end
@@ -63,7 +56,7 @@ module CompletionKit
       def self.create(args)
         metric_group = CompletionKit::MetricGroup.new(args.slice("name", "description"))
         if metric_group.save
-          replace_metric_memberships(metric_group, args["metric_ids"])
+          metric_group.replace_metrics!(args["metric_ids"])
           text_result(metric_group.reload.as_json)
         else
           error_result(metric_group.errors.full_messages.join(", "))
@@ -73,7 +66,7 @@ module CompletionKit
       def self.update(args)
         metric_group = CompletionKit::MetricGroup.find(args["id"])
         if metric_group.update(args.except("id", "metric_ids").slice("name", "description"))
-          replace_metric_memberships(metric_group, args["metric_ids"]) if args.key?("metric_ids")
+          metric_group.replace_metrics!(args["metric_ids"]) if args.key?("metric_ids")
           text_result(metric_group.reload.as_json)
         else
           error_result(metric_group.errors.full_messages.join(", "))
@@ -84,23 +77,6 @@ module CompletionKit
         CompletionKit::MetricGroup.find(args["id"]).destroy!
         text_result("Metric group #{args["id"]} deleted")
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
-
-      def self.replace_metric_memberships(metric_group, metric_ids)
-        return unless metric_ids
-        metric_group.metric_group_memberships.delete_all
-        Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
-          metric_group.metric_group_memberships.create!(metric_id: metric_id, position: index + 1)
-        end
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/metrics.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Metrics
+      extend Base
+
       TOOLS = {
         "metrics_list" => {
           description: "List all metrics",
@@ -43,15 +45,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(Metric.order(created_at: :desc).map(&:as_json))
       end
@@ -82,15 +75,6 @@ module CompletionKit
         Metric.find(args["id"]).destroy!
         text_result("Metric #{args["id"]} deleted")
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/prompts.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Prompts
+      extend Base
+
       TOOLS = {
         "prompts_list" => {
           description: "List all prompts",
@@ -48,15 +50,6 @@ module CompletionKit
         },
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(Prompt.order(created_at: :desc).map(&:as_json))
       end
@@ -98,15 +91,6 @@ module CompletionKit
         prompt.publish!
         text_result(prompt.reload.as_json)
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/provider_credentials.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module ProviderCredentials
+      extend Base
+
       TOOLS = {
         "provider_credentials_list" => {
           description: "List all provider credentials (API keys are not exposed)",
@@ -44,15 +46,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(ProviderCredential.order(created_at: :desc).map(&:as_json))
       end
@@ -83,15 +76,6 @@ module CompletionKit
         ProviderCredential.find(args["id"]).destroy!
         text_result("Provider credential #{args["id"]} deleted")
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/responses.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Responses
+      extend Base
+
       TOOLS = {
         "responses_list" => {
           description: "List responses for a run",
@@ -18,15 +20,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(args)
         run = Run.find(args["run_id"])
         text_result(run.responses.includes(:reviews).map(&:as_json))
@@ -36,10 +29,6 @@ module CompletionKit
         run = Run.find(args["run_id"])
         text_result(run.responses.find(args["id"]).as_json)
       end
-
-      def self.text_result(data)
-        {content: [{type: "text", text: data.to_json}]}
-      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -1,6 +1,8 @@
 module CompletionKit
   module McpTools
     module Runs
+      extend Base
+
       TOOLS = {
         "runs_list" => {
           description: "List all runs",
@@ -55,15 +57,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.definitions
-        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
-      end
-
-      def self.call(name, arguments)
-        tool = TOOLS.fetch(name)
-        send(tool[:handler], arguments)
-      end
-
       def self.list(_args)
         text_result(Run.order(created_at: :desc).map(&:as_json))
       end
@@ -75,7 +68,7 @@ module CompletionKit
       def self.create(args)
         run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model"))
         if run.save
-          replace_run_metrics(run, args["metric_ids"])
+          run.replace_metrics!(args["metric_ids"])
           text_result(run.reload.as_json)
         else
           error_result(run.errors.full_messages.join(", "))
@@ -85,7 +78,7 @@ module CompletionKit
       def self.update(args)
         run = Run.find(args["id"])
         if run.update(args.except("id", "metric_ids").slice("name", "dataset_id", "judge_model"))
-          replace_run_metrics(run, args["metric_ids"]) if args.key?("metric_ids")
+          run.replace_metrics!(args["metric_ids"]) if args.key?("metric_ids")
           text_result(run.reload.as_json)
         else
           error_result(run.errors.full_messages.join(", "))
@@ -108,23 +101,6 @@ module CompletionKit
         JudgeJob.perform_later(run.id)
         text_result(run.reload.as_json)
       end
-
-      def self.text_result(data)
-        text = data.is_a?(String) ? data : data.to_json
-        {content: [{type: "text", text: text}]}
-      end
-
-      def self.error_result(message)
-        {content: [{type: "text", text: message}], isError: true}
-      end
-
-      def self.replace_run_metrics(run, metric_ids)
-        return unless metric_ids
-        run.run_metrics.delete_all
-        Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
-          run.run_metrics.create!(metric_id: metric_id, position: index + 1)
-        end
-      end
     end
   end
 end

data/app/services/completion_kit/ollama_client.rb

@@ -3,20 +3,11 @@ module CompletionKit
     def generate_completion(prompt, options = {})
       return "Error: API endpoint not configured" unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
       model = options[:model]
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      conn = Faraday.new(url: api_endpoint) do |f|
-        f.request :retry, max: 2, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-
-      response = conn.post do |req|
+      response = build_connection(api_endpoint).post do |req|
         req.url "/v1/completions"
         req.headers["Content-Type"] = "application/json"
         req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
@@ -41,11 +32,7 @@ module CompletionKit
     def available_models
       return [] unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
-      response = Faraday.get("#{api_endpoint}/v1/models") do |req|
+      response = build_connection(api_endpoint).get("/v1/models") do |req|
         req.headers["Authorization"] = "Bearer #{api_key}" if api_key.present?
       end
 

data/app/services/completion_kit/open_ai_client.rb

@@ -9,20 +9,11 @@ module CompletionKit
     def generate_completion(prompt, options = {})
       return "Error: API key not configured" unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
       model = options[:model] || "gpt-4.1-mini"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      conn = Faraday.new(url: "https://api.openai.com") do |f|
-        f.request :retry, max: 2, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-
-      response = conn.post do |req|
+      response = build_connection("https://api.openai.com").post do |req|
         req.url "/v1/responses"
         req.headers["Content-Type"] = "application/json"
         req.headers["Authorization"] = "Bearer #{api_key}"

data/app/services/completion_kit/open_router_client.rb

@@ -7,22 +7,11 @@ module CompletionKit
     def generate_completion(prompt, options = {})
       return "Error: API key not configured" unless configured?
 
-      require "faraday"
-      require "faraday/retry"
-      require "json"
-
       model = options[:model] || "openai/gpt-4o-mini"
       max_tokens = options[:max_tokens] || 1000
       temperature = options[:temperature] || 0.7
 
-      conn = Faraday.new(url: BASE_URL) do |f|
-        f.options.timeout = 30
-        f.options.open_timeout = 5
-        f.request :retry, max: 2, interval: 0.5
-        f.adapter Faraday.default_adapter
-      end
-
-      response = conn.post do |req|
+      response = build_connection(BASE_URL, timeout: 30, open_timeout: 5).post do |req|
         req.url "/chat/completions"
         req.headers["Content-Type"] = "application/json"
         req.headers["Authorization"] = "Bearer #{api_key}"

data/app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb

@@ -0,0 +1,15 @@
+module CompletionKit
+  class TenantScopedUniquenessValidator < ActiveRecord::Validations::UniquenessValidator
+    def validate_each(record, attribute, value)
+      extra = Array(CompletionKit.config.tenant_scope_columns)
+      return super if extra.empty? && options[:scope].nil?
+
+      merged = options.merge(
+        scope: Array(options[:scope]) + extra,
+        attributes: [attribute],
+        class: @klass
+      )
+      self.class.superclass.new(merged).validate(record)
+    end
+  end
+end

data/db/migrate/20260403000003_add_applied_at_to_completion_kit_suggestions.rb

@@ -1,5 +1,6 @@
 class AddAppliedAtToCompletionKitSuggestions < ActiveRecord::Migration[7.1]
   def change
+    return if column_exists?(:completion_kit_suggestions, :applied_at)
     add_column :completion_kit_suggestions, :applied_at, :datetime
   end
 end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.1.0"
+  VERSION = "0.2.1"
 end

data/lib/completion_kit.rb

@@ -8,6 +8,7 @@ module CompletionKit
     attr_accessor :openai_api_key, :anthropic_api_key, :ollama_api_key, :ollama_api_endpoint
     attr_accessor :judge_model, :high_quality_threshold, :medium_quality_threshold
     attr_accessor :username, :password, :auth_strategy, :api_token
+    attr_accessor :tenant_scope, :tenant_scope_columns
 
     def initialize
       @openai_api_key = ENV['OPENAI_API_KEY']
@@ -19,6 +20,10 @@ module CompletionKit
       @high_quality_threshold = 4
       @medium_quality_threshold = 3
     end
+
+    def tenant_scope_columns
+      @tenant_scope_columns ||= []
+    end
   end
 
   class << self

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Damien Bastin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-04-19 00:00:00.000000000 Z
+date: 2026-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -259,6 +259,7 @@ files:
 - app/services/completion_kit/judge_service.rb
 - app/services/completion_kit/llm_client.rb
 - app/services/completion_kit/mcp_dispatcher.rb
+- app/services/completion_kit/mcp_tools/base.rb
 - app/services/completion_kit/mcp_tools/datasets.rb
 - app/services/completion_kit/mcp_tools/metric_groups.rb
 - app/services/completion_kit/mcp_tools/metrics.rb
@@ -271,6 +272,7 @@ files:
 - app/services/completion_kit/open_ai_client.rb
 - app/services/completion_kit/open_router_client.rb
 - app/services/completion_kit/prompt_improvement_service.rb
+- app/validators/completion_kit/tenant_scoped_uniqueness_validator.rb
 - app/views/completion_kit/api_reference/_example.html.erb
 - app/views/completion_kit/api_reference/index.html.erb
 - app/views/completion_kit/datasets/_form.html.erb