completion-kit 0.1.0.rc1

data/app/services/completion_kit/mcp_tools/metrics.rb

@@ -0,0 +1,98 @@
+module CompletionKit
+  module McpTools
+    module Metrics
+      TOOLS = {
+        "metrics_list" => {
+          description: "List all metrics",
+          inputSchema: {type: "object", properties: {}, required: []},
+          handler: :list
+        },
+        "metrics_get" => {
+          description: "Get a metric by ID",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :get
+        },
+        "metrics_create" => {
+          description: "Create a metric with evaluation criteria",
+          inputSchema: {
+            type: "object",
+            properties: {
+              name: {type: "string"}, instruction: {type: "string"},
+              evaluation_steps: {type: "array", items: {type: "string"}},
+              rubric_bands: {type: "array", items: {type: "object", properties: {stars: {type: "integer"}, description: {type: "string"}}}}
+            },
+            required: ["name"]
+          },
+          handler: :create
+        },
+        "metrics_update" => {
+          description: "Update a metric",
+          inputSchema: {
+            type: "object",
+            properties: {
+              id: {type: "integer"}, name: {type: "string"}, instruction: {type: "string"},
+              evaluation_steps: {type: "array", items: {type: "string"}},
+              rubric_bands: {type: "array", items: {type: "object", properties: {stars: {type: "integer"}, description: {type: "string"}}}}
+            },
+            required: ["id"]
+          },
+          handler: :update
+        },
+        "metrics_delete" => {
+          description: "Delete a metric",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :delete
+        }
+      }.freeze
+
+      def self.definitions
+        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def self.call(name, arguments)
+        tool = TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def self.list(_args)
+        text_result(Metric.order(created_at: :desc).map(&:as_json))
+      end
+
+      def self.get(args)
+        text_result(Metric.find(args["id"]).as_json)
+      end
+
+      def self.create(args)
+        metric = Metric.new(args.slice("name", "instruction", "evaluation_steps", "rubric_bands"))
+        if metric.save
+          text_result(metric.as_json)
+        else
+          error_result(metric.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.update(args)
+        metric = Metric.find(args["id"])
+        if metric.update(args.except("id").slice("name", "instruction", "evaluation_steps", "rubric_bands"))
+          text_result(metric.as_json)
+        else
+          error_result(metric.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.delete(args)
+        Metric.find(args["id"]).destroy!
+        text_result("Metric #{args["id"]} deleted")
+      end
+
+      def self.text_result(data)
+        text = data.is_a?(String) ? data : data.to_json
+        {content: [{type: "text", text: text}]}
+      end
+
+      def self.error_result(message)
+        {content: [{type: "text", text: message}], isError: true}
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/prompts.rb

@@ -0,0 +1,112 @@
+module CompletionKit
+  module McpTools
+    module Prompts
+      TOOLS = {
+        "prompts_list" => {
+          description: "List all prompts",
+          inputSchema: {type: "object", properties: {}, required: []},
+          handler: :list
+        },
+        "prompts_get" => {
+          description: "Get a prompt by ID",
+          inputSchema: {type: "object", properties: {id: {type: "integer", description: "Prompt ID"}}, required: ["id"]},
+          handler: :get
+        },
+        "prompts_create" => {
+          description: "Create a prompt",
+          inputSchema: {
+            type: "object",
+            properties: {
+              name: {type: "string"}, description: {type: "string"},
+              template: {type: "string"}, llm_model: {type: "string"}
+            },
+            required: ["name", "template", "llm_model"]
+          },
+          handler: :create
+        },
+        "prompts_update" => {
+          description: "Update a prompt",
+          inputSchema: {
+            type: "object",
+            properties: {
+              id: {type: "integer"}, name: {type: "string"}, description: {type: "string"},
+              template: {type: "string"}, llm_model: {type: "string"}
+            },
+            required: ["id"]
+          },
+          handler: :update
+        },
+        "prompts_delete" => {
+          description: "Delete a prompt",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :delete
+        },
+        "prompts_publish" => {
+          description: "Publish a prompt version, making it the current version",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :publish
+        },
+      }.freeze
+
+      def self.definitions
+        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def self.call(name, arguments)
+        tool = TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def self.list(_args)
+        text_result(Prompt.order(created_at: :desc).map(&:as_json))
+      end
+
+      def self.get(args)
+        text_result(Prompt.find(args["id"]).as_json)
+      end
+
+      def self.create(args)
+        prompt = Prompt.new(args.slice("name", "description", "template", "llm_model"))
+        if prompt.save
+          text_result(prompt.as_json)
+        else
+          error_result(prompt.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.update(args)
+        prompt = Prompt.find(args["id"])
+        attrs = args.except("id").slice("name", "description", "template", "llm_model")
+        if prompt.runs.exists?
+          new_prompt = prompt.clone_as_new_version(attrs)
+          new_prompt.publish!
+          text_result(new_prompt.as_json)
+        elsif prompt.update(attrs)
+          text_result(prompt.as_json)
+        else
+          error_result(prompt.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.delete(args)
+        Prompt.find(args["id"]).destroy!
+        text_result("Prompt #{args["id"]} deleted")
+      end
+
+      def self.publish(args)
+        prompt = Prompt.find(args["id"])
+        prompt.publish!
+        text_result(prompt.reload.as_json)
+      end
+
+      def self.text_result(data)
+        text = data.is_a?(String) ? data : data.to_json
+        {content: [{type: "text", text: text}]}
+      end
+
+      def self.error_result(message)
+        {content: [{type: "text", text: message}], isError: true}
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/provider_credentials.rb

@@ -0,0 +1,97 @@
+module CompletionKit
+  module McpTools
+    module ProviderCredentials
+      TOOLS = {
+        "provider_credentials_list" => {
+          description: "List all provider credentials (API keys are not exposed)",
+          inputSchema: {type: "object", properties: {}, required: []},
+          handler: :list
+        },
+        "provider_credentials_get" => {
+          description: "Get a provider credential by ID (API key is not exposed)",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :get
+        },
+        "provider_credentials_create" => {
+          description: "Create a provider credential",
+          inputSchema: {
+            type: "object",
+            properties: {
+              provider: {type: "string", enum: ["openai", "anthropic", "ollama", "openrouter"]},
+              api_key: {type: "string"},
+              api_endpoint: {type: "string"}
+            },
+            required: ["provider", "api_key"]
+          },
+          handler: :create
+        },
+        "provider_credentials_update" => {
+          description: "Update a provider credential",
+          inputSchema: {
+            type: "object",
+            properties: {
+              id: {type: "integer"}, provider: {type: "string"},
+              api_key: {type: "string"}, api_endpoint: {type: "string"}
+            },
+            required: ["id"]
+          },
+          handler: :update
+        },
+        "provider_credentials_delete" => {
+          description: "Delete a provider credential",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :delete
+        }
+      }.freeze
+
+      def self.definitions
+        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def self.call(name, arguments)
+        tool = TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def self.list(_args)
+        text_result(ProviderCredential.order(created_at: :desc).map(&:as_json))
+      end
+
+      def self.get(args)
+        text_result(ProviderCredential.find(args["id"]).as_json)
+      end
+
+      def self.create(args)
+        credential = ProviderCredential.new(args.slice("provider", "api_key", "api_endpoint"))
+        if credential.save
+          text_result(credential.as_json)
+        else
+          error_result(credential.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.update(args)
+        credential = ProviderCredential.find(args["id"])
+        if credential.update(args.except("id").slice("provider", "api_key", "api_endpoint"))
+          text_result(credential.as_json)
+        else
+          error_result(credential.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.delete(args)
+        ProviderCredential.find(args["id"]).destroy!
+        text_result("Provider credential #{args["id"]} deleted")
+      end
+
+      def self.text_result(data)
+        text = data.is_a?(String) ? data : data.to_json
+        {content: [{type: "text", text: text}]}
+      end
+
+      def self.error_result(message)
+        {content: [{type: "text", text: message}], isError: true}
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/responses.rb

@@ -0,0 +1,45 @@
+module CompletionKit
+  module McpTools
+    module Responses
+      TOOLS = {
+        "responses_list" => {
+          description: "List responses for a run",
+          inputSchema: {type: "object", properties: {run_id: {type: "integer"}}, required: ["run_id"]},
+          handler: :list
+        },
+        "responses_get" => {
+          description: "Get a specific response",
+          inputSchema: {
+            type: "object",
+            properties: {run_id: {type: "integer"}, id: {type: "integer"}},
+            required: ["run_id", "id"]
+          },
+          handler: :get
+        }
+      }.freeze
+
+      def self.definitions
+        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def self.call(name, arguments)
+        tool = TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def self.list(args)
+        run = Run.find(args["run_id"])
+        text_result(run.responses.includes(:reviews).map(&:as_json))
+      end
+
+      def self.get(args)
+        run = Run.find(args["run_id"])
+        text_result(run.responses.find(args["id"]).as_json)
+      end
+
+      def self.text_result(data)
+        {content: [{type: "text", text: data.to_json}]}
+      end
+    end
+  end
+end

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -0,0 +1,130 @@
+module CompletionKit
+  module McpTools
+    module Runs
+      TOOLS = {
+        "runs_list" => {
+          description: "List all runs",
+          inputSchema: {type: "object", properties: {}, required: []},
+          handler: :list
+        },
+        "runs_get" => {
+          description: "Get a run by ID",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :get
+        },
+        "runs_create" => {
+          description: "Create a run",
+          inputSchema: {
+            type: "object",
+            properties: {
+              name: {type: "string"}, prompt_id: {type: "integer"},
+              dataset_id: {type: "integer"}, judge_model: {type: "string"},
+              metric_ids: {type: "array", items: {type: "integer"}}
+            },
+            required: ["name", "prompt_id"]
+          },
+          handler: :create
+        },
+        "runs_update" => {
+          description: "Update a run",
+          inputSchema: {
+            type: "object",
+            properties: {
+              id: {type: "integer"}, name: {type: "string"},
+              dataset_id: {type: "integer"}, judge_model: {type: "string"},
+              metric_ids: {type: "array", items: {type: "integer"}}
+            },
+            required: ["id"]
+          },
+          handler: :update
+        },
+        "runs_delete" => {
+          description: "Delete a run",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :delete
+        },
+        "runs_generate" => {
+          description: "Generate responses for a run using its prompt and dataset",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :generate
+        },
+        "runs_judge" => {
+          description: "Judge responses for a run using configured metrics",
+          inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
+          handler: :judge
+        }
+      }.freeze
+
+      def self.definitions
+        TOOLS.map { |name, config| {name: name, description: config[:description], inputSchema: config[:inputSchema]} }
+      end
+
+      def self.call(name, arguments)
+        tool = TOOLS.fetch(name)
+        send(tool[:handler], arguments)
+      end
+
+      def self.list(_args)
+        text_result(Run.order(created_at: :desc).map(&:as_json))
+      end
+
+      def self.get(args)
+        text_result(Run.find(args["id"]).as_json)
+      end
+
+      def self.create(args)
+        run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model"))
+        if run.save
+          replace_run_metrics(run, args["metric_ids"])
+          text_result(run.reload.as_json)
+        else
+          error_result(run.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.update(args)
+        run = Run.find(args["id"])
+        if run.update(args.except("id", "metric_ids").slice("name", "dataset_id", "judge_model"))
+          replace_run_metrics(run, args["metric_ids"]) if args.key?("metric_ids")
+          text_result(run.reload.as_json)
+        else
+          error_result(run.errors.full_messages.join(", "))
+        end
+      end
+
+      def self.delete(args)
+        Run.find(args["id"]).destroy!
+        text_result("Run #{args["id"]} deleted")
+      end
+
+      def self.generate(args)
+        run = Run.find(args["id"])
+        GenerateJob.perform_later(run.id)
+        text_result(run.reload.as_json)
+      end
+
+      def self.judge(args)
+        run = Run.find(args["id"])
+        JudgeJob.perform_later(run.id)
+        text_result(run.reload.as_json)
+      end
+
+      def self.text_result(data)
+        text = data.is_a?(String) ? data : data.to_json
+        {content: [{type: "text", text: text}]}
+      end
+
+      def self.error_result(message)
+        {content: [{type: "text", text: message}], isError: true}
+      end
+
+      def self.replace_run_metrics(run, metric_ids)
+        return unless metric_ids
+        run.run_metrics.delete_all
+        Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
+          run.run_metrics.create!(metric_id: metric_id, position: index + 1)
+        end
+      end
+    end
+  end
+end