completion-kit 0.19.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38312b44903cb1fa31fc64cb00d76713e4bc7c85eb51c9b17ebae6c988778509
-  data.tar.gz: 3bef91185c33760cbb9be5711e59a591115d53dde31b606f3eb92b334979d7ac
+  metadata.gz: 44b946858390ad2aa5d3a3bcb2a68ba666a6725ba5e77725887f99087603e322
+  data.tar.gz: 52499c569d0a73b0b714547e9aeed1535abe452cfd093288356d1580ee83d7ce
 SHA512:
-  metadata.gz: 718185f7ad5d9644f32f068f1172abd4cb11b4de0a3c28099d3549d4678a437c9a98048c480e50005308e242671fe6d4746828ddee2b8a95e0efb561756d88c6
-  data.tar.gz: 54eccc031ee3b7d02bf543f06a86d9db6e00883925c61a5a85afb261734c999382a2c77076cbdf42bb9481a89cf4074b871f5cb347e60f21cb65e06772f17b8e
+  metadata.gz: 406ae145ee5963e7a1a1a32b7669591f8c04ae49cb1166d6e054d657a77aa0fca218c78a4d2f7b31dc5518ee9aa75a7f5bae28889f81753ccdcdf86822b4dfb7
+  data.tar.gz: 2c9f542f56209a7fde4d8d6cc822e7096b43b178d7fef9ef5c65cc95e971c512ad91b14c1713fefd7d076164bff99e74ec587600c9900ce55994e67ff4a273b2

data/app/controllers/completion_kit/api/v1/imports_controller.rb

@@ -0,0 +1,29 @@
+module CompletionKit
+  module Api
+    module V1
+      class ImportsController < BaseController
+        def promptfoo
+          content = params[:config].presence || request.raw_post
+          result = PromptfooImporter.call(content)
+
+          if result.ok
+            render json: import_summary(result), status: :created
+          else
+            render_error(result.error, status: :unprocessable_entity)
+          end
+        end
+
+        private
+
+        def import_summary(result)
+          {
+            prompts: result.prompts,
+            dataset: result.dataset,
+            metrics: result.metrics,
+            providers: result.providers
+          }
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/imports_controller.rb

@@ -0,0 +1,31 @@
+module CompletionKit
+  class ImportsController < ApplicationController
+    def new
+    end
+
+    def create
+      content = uploaded_content
+
+      if content.blank?
+        flash.now[:alert] = "Paste or upload a promptfooconfig.yaml to import."
+        return render :new, status: :unprocessable_entity
+      end
+
+      @result = PromptfooImporter.call(content)
+
+      if @result.ok
+        render :create
+      else
+        flash.now[:alert] = @result.error
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    private
+
+    def uploaded_content
+      file = params[:file]
+      file.respond_to?(:read) ? file.read : params[:config]
+    end
+  end
+end

data/app/models/completion_kit/response.rb

@@ -63,15 +63,21 @@ module CompletionKit
     end
 
     def broadcast_row_update
-      run.broadcast_response_update(self)
+      safely_broadcast { run.broadcast_response_update(self) }
     end
 
     def broadcast_run_progress
-      run.broadcast_progress
+      safely_broadcast { run.broadcast_progress }
     end
 
     def should_broadcast_progress?
       saved_change_to_status? && terminal?
     end
+
+    def safely_broadcast
+      yield
+    rescue StandardError => e
+      Rails.logger.error("[CompletionKit] response ##{id} broadcast failed: #{e.class}: #{e.message}")
+    end
   end
 end

data/app/models/completion_kit/review.rb

@@ -40,15 +40,21 @@ module CompletionKit
     private
 
     def broadcast_parent_row_update
-      response.run.broadcast_response_update(response)
+      safely_broadcast { response.run.broadcast_response_update(response) }
     end
 
     def broadcast_run_progress
-      response.run.broadcast_progress
+      safely_broadcast { response.run.broadcast_progress }
     end
 
     def should_broadcast_progress?
       saved_change_to_status? && terminal?
     end
+
+    def safely_broadcast
+      yield
+    rescue StandardError => e
+      Rails.logger.error("[CompletionKit] review ##{id} broadcast failed: #{e.class}: #{e.message}")
+    end
   end
 end

data/app/services/completion_kit/mcp_dispatcher.rb

@@ -36,7 +36,8 @@ module CompletionKit
         McpTools::ProviderCredentials.definitions +
         McpTools::Tags.definitions +
         McpTools::Agreements.definitions +
-        McpTools::Judges.definitions
+        McpTools::Judges.definitions +
+        McpTools::Imports.definitions
     end
 
     def self.call_tool(name, arguments)
@@ -52,6 +53,7 @@ module CompletionKit
       when /\Atags_/                 then McpTools::Tags.call(name, arguments)
       when /\Aagreements_/           then McpTools::Agreements.call(name, arguments)
       when /\Ajudges_/               then McpTools::Judges.call(name, arguments)
+      when /\Apromptfoo_/            then McpTools::Imports.call(name, arguments)
       else raise MethodNotFound, "Unknown tool: #{name}"
       end
     end

data/app/services/completion_kit/mcp_tools/imports.rb

@@ -0,0 +1,33 @@
+module CompletionKit
+  module McpTools
+    module Imports
+      extend Base
+
+      TOOLS = {
+        "promptfoo_import" => {
+          description: "Import a promptfooconfig.yaml. Creates a prompt, a dataset from the test vars, and metrics from the assert blocks (llm-rubric/g-eval become judge metrics; contains/equals/regex/is-json become deterministic check metrics). Returns a summary of what mapped and what was skipped and why; nothing is dropped silently.",
+          inputSchema: {
+            type: "object",
+            properties: {
+              config: {type: "string", description: "The full promptfooconfig.yaml contents."}
+            },
+            required: ["config"]
+          },
+          handler: :promptfoo_import
+        }
+      }.freeze
+
+      def self.promptfoo_import(args)
+        result = PromptfooImporter.call(args["config"])
+        return error_result(result.error) unless result.ok
+
+        text_result(
+          prompts: result.prompts,
+          dataset: result.dataset,
+          metrics: result.metrics,
+          providers: result.providers
+        )
+      end
+    end
+  end
+end

data/app/services/completion_kit/promptfoo_importer.rb

@@ -0,0 +1,174 @@
+require "yaml"
+require "csv"
+
+module CompletionKit
+  class PromptfooImporter
+    Result = Data.define(:ok, :error, :prompts, :dataset, :metrics, :providers)
+
+    JUDGE_ASSERTS = %w[llm-rubric g-eval model-graded-closedqa factuality].freeze
+
+    def self.call(content)
+      new(content).call
+    end
+
+    def initialize(content)
+      @content = content.to_s
+    end
+
+    def call
+      config = parse
+      return failure("Could not parse YAML: #{@parse_error}") if config.nil?
+      return failure("Top-level YAML must be a mapping of promptfoo config keys.") unless config.is_a?(Hash)
+
+      ApplicationRecord.transaction do
+        providers = import_providers(config)
+        prompts = import_prompts(config, default_model(config))
+        dataset = import_dataset(config)
+        metrics = import_metrics(config)
+        Result.new(ok: true, error: nil, prompts: prompts, dataset: dataset, metrics: metrics, providers: providers)
+      end
+    end
+
+    private
+
+    def parse
+      YAML.safe_load(@content, aliases: true)
+    rescue Psych::Exception => e
+      @parse_error = e.message
+      nil
+    end
+
+    def failure(message)
+      Result.new(ok: false, error: message, prompts: nil, dataset: nil, metrics: nil, providers: nil)
+    end
+
+    def import_providers(config)
+      entries = Array(config["providers"]).map { |p| provider_name(p) }.compact
+      configured = ProviderCredential.pluck(:provider).to_set
+      matched = []
+      unmatched = []
+      entries.each do |entry|
+        provider = entry.split(":").first
+        (configured.include?(provider) ? matched : unmatched) << entry
+      end
+      { matched: matched.uniq, unmatched: unmatched.uniq }
+    end
+
+    def provider_name(provider)
+      provider.is_a?(Hash) ? provider["id"] : provider
+    end
+
+    def default_model(config)
+      first = Array(config["providers"]).map { |p| provider_name(p) }.compact.first
+      return "gpt-4o" if first.nil?
+
+      parts = first.split(":")
+      parts.length > 1 ? parts.last : first
+    end
+
+    def import_prompts(config, model)
+      created = []
+      skipped = []
+      Array(config["prompts"]).each_with_index do |raw, index|
+        if raw.is_a?(String) && raw.start_with?("file://")
+          skipped << { value: raw, reason: "prompt file reference; paste the file's contents to import it" }
+          next
+        end
+        template = prompt_template(raw)
+        if template.nil?
+          skipped << { value: raw.inspect, reason: "unsupported prompt shape (only inline string templates import)" }
+          next
+        end
+
+        prompt = Prompt.create!(name: "Imported prompt #{index + 1}", template: template, llm_model: model)
+        created << prompt.name
+      end
+      { created: created, skipped: skipped }
+    end
+
+    def prompt_template(raw)
+      return raw if raw.is_a?(String)
+      raw["raw"] || raw["template"] if raw.is_a?(Hash)
+    end
+
+    def import_dataset(config)
+      tests = Array(config["tests"])
+      vars_rows = tests.map { |t| (t.is_a?(Hash) ? t["vars"] : nil) || {} }.select { |v| v.is_a?(Hash) }
+      return { skipped: "no tests with vars to import" } if vars_rows.empty?
+
+      columns = vars_rows.flat_map(&:keys).uniq
+      csv = CSV.generate do |out|
+        out << columns
+        vars_rows.each { |vars| out << columns.map { |c| vars[c] } }
+      end
+      dataset = Dataset.create!(name: "Imported dataset", csv_data: csv)
+      { created: dataset.name, rows: vars_rows.length, columns: columns }
+    end
+
+    def import_metrics(config)
+      created = []
+      skipped = []
+      asserts(config).each do |assert|
+        attrs = metric_attributes(assert)
+        if attrs.nil?
+          skipped << { type: assert["type"], reason: "no CompletionKit metric maps to this assert type" }
+          next
+        end
+
+        metric = Metric.create!(attrs)
+        created << { name: metric.name, type: metric.metric_type }
+      rescue ActiveRecord::RecordInvalid => e
+        skipped << { type: assert["type"], reason: e.record.errors.full_messages.join(", ") }
+      end
+      { created: created, skipped: skipped }
+    end
+
+    def asserts(config)
+      default = config.dig("defaultTest", "assert")
+      per_test = Array(config["tests"]).flat_map { |t| t.is_a?(Hash) ? Array(t["assert"]) : [] }
+      (Array(default) + per_test).select { |a| a.is_a?(Hash) && a["type"] }.uniq
+    end
+
+    def metric_attributes(assert)
+      type = assert["type"].to_s
+      value = assert["value"]
+      return { name: unique_name("Rubric"), metric_type: "llm_judge", instruction: value.to_s } if JUDGE_ASSERTS.include?(type)
+
+      config, label = check_mapping(type, value.to_s)
+      return nil if config.nil?
+
+      { name: unique_name(label), metric_type: "check", check_config: config }
+    end
+
+    def check_mapping(type, text)
+      case type
+      when "contains"
+        [{ "check_kind" => "contains", "target" => "response_text", "value" => text }, "Contains #{text.truncate(30).inspect}"]
+      when "icontains"
+        [{ "check_kind" => "contains", "target" => "response_text", "value" => text, "case_sensitive" => false }, "Contains #{text.truncate(30).inspect}"]
+      when "not-contains"
+        [{ "check_kind" => "not_contains", "target" => "response_text", "value" => text }, "Does not contain #{text.truncate(30).inspect}"]
+      when "not-icontains"
+        [{ "check_kind" => "not_contains", "target" => "response_text", "value" => text, "case_sensitive" => false }, "Does not contain #{text.truncate(30).inspect}"]
+      when "equals"
+        [{ "check_kind" => "equals", "target" => "response_text", "value" => text }, "Equals #{text.truncate(30).inspect}"]
+      when "regex"
+        [{ "check_kind" => "regex", "target" => "response_text", "pattern" => text }, "Matches /#{text.truncate(30)}/"]
+      when "is-json"
+        [{ "check_kind" => "valid_json", "target" => "response_text" }, "Valid JSON"]
+      end
+    end
+
+    def unique_name(base)
+      @used_names ||= Metric.pluck(:name).to_set
+      candidate = base
+      counter = 2
+      while @used_names.include?(candidate)
+        candidate = "#{base} (#{counter})"
+        counter += 1
+      end
+      @used_names << candidate
+      candidate
+    end
+  end
+end

data/app/views/completion_kit/imports/create.html.erb

@@ -0,0 +1,89 @@
+<ol class="ck-breadcrumb">
+  <li><%= link_to "Metrics", metrics_path %></li>
+  <li><%= link_to "Import from Promptfoo", import_path %></li>
+  <li>Summary</li>
+</ol>
+
+<section class="ck-page-header">
+  <div>
+    <h1 class="ck-title">Import summary</h1>
+    <p class="ck-lead">Here is what mapped into CompletionKit and what could not be translated.</p>
+  </div>
+  <div class="ck-actions">
+    <%= link_to "Import another", import_path, class: ck_button_classes(:light, variant: :outline) %>
+  </div>
+</section>
+
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">Prompts</p>
+  <% if @result.prompts[:created].any? %>
+    <ul class="ck-list ck-list--compact">
+      <% @result.prompts[:created].each do |name| %>
+        <li><%= link_to name, prompts_path, class: "ck-link" %></li>
+      <% end %>
+    </ul>
+  <% else %>
+    <p class="ck-meta-copy">No prompts imported.</p>
+  <% end %>
+  <% if @result.prompts[:skipped].any? %>
+    <p class="ck-kicker">Skipped prompts</p>
+    <ul class="ck-list ck-list--compact">
+      <% @result.prompts[:skipped].each do |item| %>
+        <li><code><%= item[:value] %></code>: <span class="ck-meta-copy"><%= item[:reason] %></span></li>
+      <% end %>
+    </ul>
+  <% end %>
+</section>
+
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">Dataset</p>
+  <% if @result.dataset[:created] %>
+    <p><%= link_to @result.dataset[:created], datasets_path, class: "ck-link" %>, <%= pluralize(@result.dataset[:rows], "row") %>. Columns: <%= @result.dataset[:columns].join(", ") %>.</p>
+  <% else %>
+    <p class="ck-meta-copy">Skipped: <%= @result.dataset[:skipped] %>.</p>
+  <% end %>
+</section>
+
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">Metrics</p>
+  <% if @result.metrics[:created].any? %>
+    <ul class="ck-list ck-list--compact">
+      <% @result.metrics[:created].each do |metric| %>
+        <li>
+          <%= link_to metric[:name], metrics_path, class: "ck-link" %>
+          <span class="ck-chip ck-chip--soft"><%= metric[:type] == "check" ? "Check" : "Judge" %></span>
+        </li>
+      <% end %>
+    </ul>
+  <% else %>
+    <p class="ck-meta-copy">No metrics imported.</p>
+  <% end %>
+  <% if @result.metrics[:skipped].any? %>
+    <p class="ck-kicker">Skipped asserts</p>
+    <ul class="ck-list ck-list--compact">
+      <% @result.metrics[:skipped].each do |item| %>
+        <li><code><%= item[:type] %></code>: <span class="ck-meta-copy"><%= item[:reason] %></span></li>
+      <% end %>
+    </ul>
+  <% end %>
+</section>
+
+<section class="ck-card ck-card--spaced">
+  <p class="ck-kicker">Providers</p>
+  <% if @result.providers[:matched].any? %>
+    <p class="ck-meta-copy">Matched to configured credentials:</p>
+    <div class="ck-chip-row">
+      <% @result.providers[:matched].each do |name| %>
+        <span class="ck-chip ck-chip--done"><%= name %></span>
+      <% end %>
+    </div>
+  <% end %>
+  <% if @result.providers[:unmatched].any? %>
+    <p class="ck-meta-copy">Referenced but not configured here. <%= link_to "Add a provider", provider_credentials_path, class: "ck-link" %> to run against these.</p>
+    <div class="ck-chip-row">
+      <% @result.providers[:unmatched].each do |name| %>
+        <span class="ck-chip ck-chip--warning"><%= name %></span>
+      <% end %>
+    </div>
+  <% end %>
+</section>

data/app/views/completion_kit/imports/new.html.erb

@@ -0,0 +1,31 @@
+<ol class="ck-breadcrumb">
+  <li><%= link_to "Metrics", metrics_path %></li>
+  <li>Import from Promptfoo</li>
+</ol>
+
+<section class="ck-page-header">
+  <div>
+    <h1 class="ck-title">Import from Promptfoo</h1>
+    <p class="ck-lead">Paste or upload a <code>promptfooconfig.yaml</code> to bring its prompts, test vars, and assertions into CompletionKit. This maps the eval and judge half of Promptfoo. Anything it cannot translate is reported back on the next page, never dropped silently.</p>
+  </div>
+</section>
+
+<%= form_with url: import_path, method: :post, multipart: true do |form| %>
+  <div class="ck-card ck-form-card">
+    <div class="ck-field">
+      <%= form.label :config, "Paste config", class: "ck-label" %>
+      <%= form.text_area :config, rows: 16, class: "ck-input ck-input--area ck-input--code", placeholder: "prompts:\n  - \"Summarize {{article}}.\"\ntests:\n  - vars: { article: \"...\" }\ndefaultTest:\n  assert:\n    - type: contains\n      value: \"summary\"" %>
+    </div>
+
+    <div class="ck-field">
+      <%= form.label :file, "Or upload a file", class: "ck-label" %>
+      <%= form.file_field :file, accept: ".yaml,.yml", class: "ck-input" %>
+      <p class="ck-field-hint">A <code>.yaml</code> file takes precedence over the pasted text.</p>
+    </div>
+
+    <div class="ck-actions">
+      <%= link_to "Cancel", metrics_path, class: ck_button_classes(:light, variant: :outline), tabindex: "0" %>
+      <%= form.submit "Import", class: ck_button_classes(:dark) %>
+    </div>
+  </div>
+<% end %>

data/app/views/completion_kit/metrics/index.html.erb

@@ -4,6 +4,7 @@
     <p class="ck-lead">Scoring dimensions used to evaluate each response. Each defines one thing to assess, by an LLM judge on a 1-5 scale or a deterministic check.</p>
   </div>
   <div class="ck-actions">
+    <%= link_to "Import from Promptfoo", import_path, class: ck_button_classes(:light, variant: :outline) %>
     <%= link_to "New metric", new_metric_path, class: ck_button_classes(:dark) %>
   </div>
 </section>

data/config/routes.rb

@@ -59,6 +59,9 @@ CompletionKit::Engine.routes.draw do
 
   get "api_reference", to: "api_reference#index", as: :api_reference
 
+  get "import", to: "imports#new", as: :import
+  post "import", to: "imports#create"
+
   namespace :api do
     namespace :v1 do
       resources :prompts do
@@ -95,6 +98,7 @@ CompletionKit::Engine.routes.draw do
       resources :tags
       resources :provider_credentials
       resources :agreements, only: [:index, :destroy]
+      post "imports/promptfoo", to: "imports#promptfoo"
     end
   end
 

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.19.0"
+  VERSION = "0.20.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -238,6 +238,7 @@ files:
 - app/controllers/completion_kit/api/v1/agreements_controller.rb
 - app/controllers/completion_kit/api/v1/base_controller.rb
 - app/controllers/completion_kit/api/v1/datasets_controller.rb
+- app/controllers/completion_kit/api/v1/imports_controller.rb
 - app/controllers/completion_kit/api/v1/metric_groups_controller.rb
 - app/controllers/completion_kit/api/v1/metric_versions_controller.rb
 - app/controllers/completion_kit/api/v1/metrics_controller.rb
@@ -251,6 +252,7 @@ files:
 - app/controllers/completion_kit/dashboard_controller.rb
 - app/controllers/completion_kit/dashboard_dismissals_controller.rb
 - app/controllers/completion_kit/datasets_controller.rb
+- app/controllers/completion_kit/imports_controller.rb
 - app/controllers/completion_kit/mcp_controller.rb
 - app/controllers/completion_kit/metric_groups_controller.rb
 - app/controllers/completion_kit/metrics_controller.rb
@@ -317,6 +319,7 @@ files:
 - app/services/completion_kit/mcp_tools/agreements.rb
 - app/services/completion_kit/mcp_tools/base.rb
 - app/services/completion_kit/mcp_tools/datasets.rb
+- app/services/completion_kit/mcp_tools/imports.rb
 - app/services/completion_kit/mcp_tools/judges.rb
 - app/services/completion_kit/mcp_tools/metric_groups.rb
 - app/services/completion_kit/mcp_tools/metric_versions.rb
@@ -339,6 +342,7 @@ files:
 - app/services/completion_kit/open_router_client.rb
 - app/services/completion_kit/prompt_improvement_service.rb
 - app/services/completion_kit/prompt_improvement_validator.rb
+- app/services/completion_kit/promptfoo_importer.rb
 - app/services/completion_kit/provider_endpoint.rb
 - app/services/completion_kit/run_comparison.rb
 - app/services/completion_kit/starter_metrics.rb
@@ -364,6 +368,8 @@ files:
 - app/views/completion_kit/datasets/index.html.erb
 - app/views/completion_kit/datasets/new.html.erb
 - app/views/completion_kit/datasets/show.html.erb
+- app/views/completion_kit/imports/create.html.erb
+- app/views/completion_kit/imports/new.html.erb
 - app/views/completion_kit/metric_groups/_form.html.erb
 - app/views/completion_kit/metric_groups/edit.html.erb
 - app/views/completion_kit/metric_groups/index.html.erb