completion-kit 0.12.4 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfff016e2f30d073116ddcac7e1deb59dd6a6f5cbbc56452004912691a28d07b
-  data.tar.gz: 536e3a578f45d762c37d0115a390b034f581a5e3e2dcd7f036200aef8ef6db22
+  metadata.gz: 7a3fda7b41271efa8b169adff5bc72e3f347fa67641aa26ab2d52f84ad4d0526
+  data.tar.gz: fd9fcca6cc6ac22e32a3f8ba7f917fd8b0ccb30a270f45aa63fb7fc3ee59343b
 SHA512:
-  metadata.gz: 1d7498497e03fdcd2797eb361250eb70bb378884f9593f617dc4e8a5dc9e61dbad15feef83bfd1b811b56f07bdd1a7573e65c291c05bd5233ff4b0442f16e089
-  data.tar.gz: 729db67d4ab9151e3103c4ae75cfdfc720311608cdb3f7aba15609bbabc5221d292528fb514a54b9c058e62ce98f4f1e74971edde05aba53073313f427c6803f
+  metadata.gz: '02917101ab65551a18974c80b382ba508b5f0a80f67182bf630998aacfebc4584f85d03d9fc0c59d6de7b89c1881391340cd81c3cd1631bca6b545067f57a513'
+  data.tar.gz: 255c33ab87a13e1e4ed5387c8cff70c952cf0c68c3d3829ddc4c401208bc1d94019f121fd0cfbd7ec425a4669f3420b96b988ade2bba50f0113f5e714957295e

data/app/controllers/completion_kit/api/v1/datasets_controller.rb

@@ -45,7 +45,10 @@ module CompletionKit
         end
 
         def dataset_params
-          params.permit(:name, :csv_data, tag_names: [])
+          permitted = params.permit(:name, :csv_data, tag_names: [])
+          upload = params[:file]
+          permitted[:csv_data] = upload.read.to_s.force_encoding("UTF-8") if upload.respond_to?(:read)
+          permitted
         end
       end
     end

data/app/services/completion_kit/mcp_tools/datasets.rb

@@ -1,8 +1,12 @@
+require "faraday"
+
 module CompletionKit
   module McpTools
     module Datasets
       extend Base
 
+      MAX_CSV_BYTES = 10 * 1024 * 1024
+
       TOOLS = {
         "datasets_list" => {
           description: "List all datasets",
@@ -36,6 +40,19 @@ module CompletionKit
           description: "Delete a dataset",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :delete
+        },
+        "datasets_create_from_url" => {
+          description: "Create a dataset by downloading CSV from a URL instead of inlining it. Use this for large datasets: pass a public http(s) URL and the server fetches the CSV directly, so the data never has to pass through the tool-call arguments. The URL is SSRF-checked and the download is capped at 10MB.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              name: {type: "string"},
+              url: {type: "string", description: "Public http(s) URL of the CSV file to download."},
+              tag_names: {type: "array", items: {type: "string"}}
+            },
+            required: ["name", "url"]
+          },
+          handler: :create_from_url
         }
       }.freeze
 
@@ -71,6 +88,35 @@ module CompletionKit
         Dataset.find(args["id"]).destroy!
         text_result("Dataset #{args["id"]} deleted")
       end
+
+      def self.create_from_url(args)
+        issues = ProviderEndpoint.validate(args["url"])
+        return error_result("URL is not allowed (#{issues.join(", ")}).") if issues.any?
+
+        response = csv_connection.get(args["url"])
+        return error_result("Could not download CSV (HTTP #{response.status}).") unless response.success?
+
+        body = response.body.to_s
+        return error_result("CSV is larger than the #{MAX_CSV_BYTES / (1024 * 1024)}MB limit.") if body.bytesize > MAX_CSV_BYTES
+
+        dataset = Dataset.new(name: args["name"], csv_data: body.dup.force_encoding("UTF-8"))
+        dataset.tag_names = args["tag_names"] if args.key?("tag_names")
+        if dataset.save
+          text_result(dataset.reload.as_json)
+        else
+          error_result(dataset.errors.full_messages.join(", "))
+        end
+      rescue Faraday::Error => e
+        error_result("Could not download CSV: #{e.message}")
+      end
+
+      def self.csv_connection
+        Faraday.new do |f|
+          f.options.timeout = 30
+          f.options.open_timeout = 5
+          f.adapter Faraday.default_adapter
+        end
+      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/judges.rb

@@ -4,19 +4,6 @@ module CompletionKit
       extend Base
 
       TOOLS = {
-        "judges_suggest" => {
-          description: "Ask the model to rewrite the metric's judge instruction in N variants targeted at the recent disagreements. Each variant is saved as a draft MetricVersion with source=\"suggestion\". Returns the persisted drafts. Stripe-metering hooks fire via ActiveSupport::Notifications under completion_kit.judge_suggestion.generated.",
-          inputSchema: {
-            type: "object",
-            properties: {
-              metric_id: { type: "integer" },
-              count: { type: "integer", description: "How many variants to request (default 1, max 3). One focused rewrite beats five reworded copies." },
-              model: { type: "string", description: "Override the model used to generate variants. Defaults to CompletionKit.config.judge_model." }
-            },
-            required: ["metric_id"]
-          },
-          handler: :suggest
-        },
         "judges_replay" => {
           description: "Run the current judge against a dataset (judge-only run). Wraps runs_create with prompt_id omitted and output_column supplied. Re-judges existing dataset outputs so you can compare against human verdicts.",
           inputSchema: {
@@ -47,15 +34,6 @@ module CompletionKit
         }
       }.freeze
 
-      def self.suggest(args)
-        metric = CompletionKit::Metric.find(args["metric_id"])
-        generator = CompletionKit::MetricVariantGenerator.new(metric, count: args["count"].to_i, model: args["model"])
-        variants = generator.call
-        return error_result("Variant generator returned no parseable variants. Try again or change the model.") if variants.empty?
-        versions = generator.persist!(variants)
-        text_result(versions.map(&:as_json))
-      end
-
       def self.replay(args)
         metric = CompletionKit::Metric.find(args["metric_id"])
         dataset = CompletionKit::Dataset.find(args["dataset_id"])

data/app/services/completion_kit/mcp_tools/metrics.rb

@@ -44,6 +44,19 @@ module CompletionKit
           description: "Delete a metric",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :delete
+        },
+        "metrics_suggest_variants" => {
+          description: "Ask the model to rewrite the metric's judge instruction in N variants targeted at the recent disagreements. Each variant is saved as a draft MetricVersion with source=\"suggestion\". Returns the persisted drafts. Stripe-metering hooks fire via ActiveSupport::Notifications under completion_kit.judge_suggestion.generated.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              metric_id: {type: "integer"},
+              count: {type: "integer", description: "How many variants to request (default 1, max 3). One focused rewrite beats five reworded copies."},
+              model: {type: "string", description: "Override the model used to generate variants. Defaults to CompletionKit.config.judge_model."}
+            },
+            required: ["metric_id"]
+          },
+          handler: :suggest_variants
         }
       }.freeze
 
@@ -79,6 +92,15 @@ module CompletionKit
         Metric.find(args["id"]).destroy!
         text_result("Metric #{args["id"]} deleted")
       end
+
+      def self.suggest_variants(args)
+        metric = Metric.find(args["metric_id"])
+        generator = MetricVariantGenerator.new(metric, count: args["count"].to_i, model: args["model"])
+        variants = generator.call
+        return error_result("Variant generator returned no parseable variants. Try again or change the model.") if variants.empty?
+        versions = generator.persist!(variants)
+        text_result(versions.map(&:as_json))
+      end
     end
   end
 end

data/app/services/completion_kit/mcp_tools/prompts.rb

@@ -50,6 +50,15 @@ module CompletionKit
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :publish
         },
+        "prompts_suggest_improvement" => {
+          description: "Suggest an improved version of a prompt, grounded in a run's test results and judge feedback. Analyzes the run's responses, scores, and reviews, then returns reasoning plus a rewritten template (preserving {{variables}}) and persists it as a Suggestion. Requires a run that has a prompt (not a judge-only run).",
+          inputSchema: {
+            type: "object",
+            properties: {run_id: {type: "integer", description: "The run whose results ground the improvement."}},
+            required: ["run_id"]
+          },
+          handler: :suggest_improvement
+        },
       }.freeze
 
       def self.list(_args)
@@ -96,6 +105,26 @@ module CompletionKit
         prompt.publish!
         text_result(prompt.reload.as_json)
       end
+
+      def self.suggest_improvement(args)
+        run = Run.find(args["run_id"])
+        return error_result("Judge-only runs don't have a prompt to improve.") if run.prompt.nil?
+
+        result = PromptImprovementService.new(run).suggest
+        suggestion = run.suggestions.create!(
+          prompt: run.prompt,
+          reasoning: result["reasoning"],
+          suggested_template: result["suggested_template"],
+          original_template: result["original_template"]
+        )
+        text_result(
+          suggestion_id: suggestion.id,
+          prompt_id: run.prompt.id,
+          reasoning: suggestion.reasoning,
+          suggested_template: suggestion.suggested_template,
+          original_template: suggestion.original_template
+        )
+      end
     end
   end
 end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.12.4"
+  VERSION = "0.14.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.12.4
+  version: 0.14.0
 platform: ruby
 authors:
 - Damien Bastin