completion-kit 0.1.0.rc1

data/app/controllers/completion_kit/prompts_controller.rb

@@ -0,0 +1,69 @@
+module CompletionKit
+  class PromptsController < ApplicationController
+    before_action :set_prompt, only: [:show, :edit, :update, :destroy, :publish]
+    
+    def index
+      @prompts = Prompt.current_versions.includes(:runs).order(created_at: :desc)
+    end
+    
+    def show
+      @runs = Run.where(prompt_id: @prompt.family_versions.select(:id))
+                 .includes(:prompt, :dataset, responses: :reviews)
+                 .order(created_at: :desc)
+    end
+    
+    def new
+      @prompt = Prompt.new
+    end
+
+    def edit
+    end
+
+    def create
+      @prompt = Prompt.new(prompt_params)
+
+      if @prompt.save
+        redirect_to prompt_path(@prompt), notice: "Prompt version was successfully created."
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+    
+    def update
+      if @prompt.runs.exists?
+        new_prompt = @prompt.clone_as_new_version(prompt_params.to_h)
+        new_prompt.publish!
+        redirect_to prompt_path(new_prompt), notice: "Saved as #{new_prompt.version_label}."
+      elsif @prompt.update(prompt_params)
+        redirect_to prompt_path(@prompt), notice: "Prompt saved."
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+    
+    def destroy
+      @prompt.destroy
+      redirect_to prompts_path, notice: "Prompt version was successfully destroyed."
+    end
+
+    def publish
+      @prompt.publish!
+      redirect_to prompt_path(@prompt), notice: "#{@prompt.display_name} is now the current version."
+    end
+
+    private
+    
+    def set_prompt
+      @prompt = Prompt.find(params[:id])
+    end
+    
+    def prompt_params
+      params.require(:prompt).permit(
+        :name,
+        :description,
+        :template,
+        :llm_model
+      )
+    end
+  end
+end

data/app/controllers/completion_kit/provider_credentials_controller.rb

@@ -0,0 +1,63 @@
+module CompletionKit
+  class ProviderCredentialsController < ApplicationController
+    before_action :set_provider_credential, only: [:edit, :update, :refresh]
+
+    def index
+      @provider_credentials = ProviderCredential.order(:provider)
+    end
+
+    def new
+      @provider_credential = ProviderCredential.new(provider: params[:provider])
+    end
+
+    def create
+      @provider_credential = ProviderCredential.new(provider_credential_params)
+
+      if @provider_credential.save
+        redirect_to provider_credentials_path, notice: "Provider credential was successfully created."
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def edit
+    end
+
+    def update
+      if @provider_credential.update(provider_credential_params)
+        redirect_to provider_credentials_path, notice: "Provider credential was successfully updated."
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def refresh
+      @provider_credential.update_columns(discovery_status: "discovering", discovery_current: 0, discovery_total: 0)
+      @provider_credential.reload
+      @provider_credential.broadcast_discovery_progress
+      ModelDiscoveryJob.perform_later(@provider_credential.id)
+      head :ok
+    end
+
+    def refresh_all
+      ProviderCredential.find_each do |cred|
+        cred.update_columns(discovery_status: "discovering", discovery_current: 0, discovery_total: 0)
+        cred.reload
+        cred.broadcast_discovery_progress
+        ModelDiscoveryJob.perform_later(cred.id)
+      end
+
+      head :ok
+    end
+
+    private
+
+    def set_provider_credential
+      @provider_credential = ProviderCredential.find(params[:id])
+    end
+
+    def provider_credential_params
+      params.require(:provider_credential).permit(:provider, :api_key, :api_endpoint)
+    end
+  end
+end

data/app/controllers/completion_kit/responses_controller.rb

@@ -0,0 +1,44 @@
+module CompletionKit
+  class ResponsesController < ApplicationController
+    before_action :set_run
+    before_action :set_response
+
+    def show
+      @sort = params[:sort]
+      ordered_ids = ordered_response_ids
+      current_index = ordered_ids.index(@response.id)
+      @response_number = current_index + 1
+      @reviews = @response.reviews.includes(:metric)
+      @prev_response = current_index > 0 ? ordered_ids[current_index - 1] : nil
+      @next_response = ordered_ids[current_index + 1]
+    end
+
+    private
+
+    def set_run
+      @run = Run.find(params[:run_id])
+    end
+
+    def set_response
+      @response = @run.responses.find(params[:id])
+    end
+
+    def ordered_response_ids
+      if @run.judge_configured? && @sort == "score_asc"
+        @run.responses
+          .left_joins(:reviews)
+          .group("completion_kit_responses.id")
+          .order(Arel.sql("AVG(completion_kit_reviews.ai_score) ASC NULLS LAST"))
+          .pluck(:id)
+      elsif @run.judge_configured? && @sort != "none"
+        @run.responses
+          .left_joins(:reviews)
+          .group("completion_kit_responses.id")
+          .order(Arel.sql("AVG(completion_kit_reviews.ai_score) DESC NULLS LAST"))
+          .pluck(:id)
+      else
+        @run.responses.order(:id).pluck(:id)
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/runs_controller.rb

@@ -0,0 +1,131 @@
+module CompletionKit
+  class RunsController < ApplicationController
+    before_action :set_run, only: [:show, :edit, :update, :destroy, :generate, :judge, :suggest, :suggestion, :apply_suggestion]
+    before_action :load_form_collections, only: [:new, :edit, :create, :update]
+
+    def index
+      @runs = Run.includes(:prompt, :dataset, responses: :reviews).order(created_at: :desc)
+    end
+
+    def show
+      @responses = if @run.judge_configured? && params[:sort] == "score_asc"
+                     @run.responses
+                       .left_joins(:reviews)
+                       .includes(:reviews)
+                       .group("completion_kit_responses.id")
+                       .order(Arel.sql("AVG(completion_kit_reviews.ai_score) ASC NULLS LAST"))
+                   elsif @run.judge_configured?
+                     @run.responses
+                       .left_joins(:reviews)
+                       .includes(:reviews)
+                       .group("completion_kit_responses.id")
+                       .order(Arel.sql("AVG(completion_kit_reviews.ai_score) DESC NULLS LAST"))
+                   else
+                     @run.responses.includes(:reviews).order(:id)
+                   end
+    end
+
+    def new
+      @run = Run.new(prompt_id: params[:prompt_id])
+    end
+
+    def edit
+    end
+
+    def create
+      @run = Run.new(run_params.except(:metric_ids))
+      if @run.save
+        replace_run_metrics(@run, params[:run][:metric_ids])
+        redirect_to run_path(@run), notice: "Run was successfully created."
+      else
+        load_form_collections
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def update
+      if @run.responses.any?
+        new_run = Run.create!(run_params.except(:metric_ids).to_h.merge(status: "pending"))
+        replace_run_metrics(new_run, params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
+        redirect_to run_path(new_run), notice: "Saved as a new run. The previous run and its results are preserved."
+      elsif @run.update(run_params.except(:metric_ids))
+        replace_run_metrics(@run, params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
+        redirect_to run_path(@run), notice: "Run saved."
+      else
+        load_form_collections
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      @run.destroy
+      redirect_to runs_path, notice: "Run was successfully destroyed."
+    end
+
+    def generate
+      @run.update!(status: "generating", progress_current: 0, progress_total: 0, error_message: nil)
+      GenerateJob.perform_later(@run.id)
+      redirect_to run_path(@run)
+    end
+
+    def judge
+      if params[:run]
+        @run.update(judge_model: params[:run][:judge_model])
+      end
+      JudgeJob.perform_later(@run.id)
+      redirect_to run_path(@run)
+    end
+
+    def suggest
+      service = PromptImprovementService.new(@run)
+      result = service.suggest
+      @run.suggestions.create!(
+        prompt: @run.prompt,
+        reasoning: result["reasoning"],
+        suggested_template: result["suggested_template"],
+        original_template: result["original_template"]
+      )
+      redirect_to suggestion_run_path(@run)
+    end
+
+    def suggestion
+      @suggestion = @run.suggestions.order(created_at: :desc).first
+      return redirect_to run_path(@run), alert: "No suggestion available. Generate one first." unless @suggestion
+    end
+
+    def apply_suggestion
+      suggestion = @run.suggestions.order(created_at: :desc).first
+      return redirect_to run_path(@run), alert: "No suggestion to apply." unless suggestion
+
+      new_prompt = @run.prompt.clone_as_new_version(template: suggestion.suggested_template)
+      new_prompt.publish!
+      suggestion.update!(applied_at: Time.current)
+      redirect_to prompt_path(new_prompt), notice: "Suggestion applied."
+    end
+
+    private
+
+    def set_run
+      @run = Run.find(params[:id])
+    end
+
+    def load_form_collections
+      @prompts = Prompt.order(:name)
+      @datasets = Dataset.order(:name)
+      @criterias = Criteria.includes(:metrics).order(:name)
+      @all_metrics = Metric.order(:name)
+    end
+
+    def run_params
+      params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, metric_ids: [])
+    end
+
+    def replace_run_metrics(run, metric_ids)
+      return unless metric_ids
+      run.run_metrics.delete_all
+      Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
+        run.run_metrics.create!(metric_id: metric_id, position: index + 1)
+      end
+    end
+  end
+end

data/app/helpers/completion_kit/application_helper.rb

@@ -0,0 +1,193 @@
+module CompletionKit
+  module ApplicationHelper
+    def ck_button_classes(tone = :dark, variant: :solid)
+      base = "ck-button"
+
+      styles = case [tone, variant]
+               when [:dark, :solid]
+                 "ck-button--primary"
+               when [:light, :outline]
+                 "ck-button--secondary"
+               when [:green, :solid]
+                 "ck-button--success"
+               when [:red, :outline]
+                 "ck-button--danger"
+               when [:amber, :outline]
+                 "ck-button--warning"
+               when [:blue, :outline]
+                 "ck-button--info"
+               else
+                 "ck-button--primary"
+               end
+
+      "#{base} #{styles}"
+    end
+
+    def ck_badge_classes(kind)
+      case kind.to_s
+      when "high"
+        "ck-badge ck-badge--high"
+      when "medium"
+        "ck-badge ck-badge--medium"
+      when "low"
+        "ck-badge ck-badge--low"
+      when "pending"
+        "ck-badge ck-badge--pending"
+      when "running"
+        "ck-badge ck-badge--running"
+      when "generating", "judging"
+        "ck-badge ck-badge--running"
+      when "completed"
+        "ck-badge ck-badge--high"
+      when "failed"
+        "ck-badge ck-badge--low"
+      else
+        "ck-badge ck-badge--pending"
+      end
+    end
+
+    def ck_run_dot(run)
+      case run.status
+      when "generating", "judging" then "ck-dot ck-dot--running"
+      when "failed" then "ck-dot ck-dot--failed"
+      when "completed" then "ck-dot ck-dot--completed"
+      else "ck-dot ck-dot--pending"
+      end
+    end
+
+    def ck_run_status_label(run)
+      case run.status
+      when "pending" then "Ready to run"
+      when "generating"
+        if run.progress_total.to_i > 0
+          "Generating responses (#{run.progress_current}/#{run.progress_total})"
+        else
+          "Generating responses…"
+        end
+      when "judging"
+        if run.progress_total.to_i > 0
+          "Judging (#{run.progress_current}/#{run.progress_total} evaluations)"
+        else
+          "Judging…"
+        end
+      when "completed" then "Completed"
+      when "failed" then "Failed"
+      else run.status.capitalize
+      end
+    end
+
+    PROVIDER_LABELS = {
+      "openai" => "OpenAI",
+      "anthropic" => "Anthropic",
+      "ollama" => "Ollama / local endpoint",
+      "openrouter" => "OpenRouter"
+    }.freeze
+
+    def ck_provider_label(provider)
+      PROVIDER_LABELS[provider.to_s] || provider.to_s.titleize
+    end
+
+    def ck_grouped_models(models, selected = nil)
+      if selected.present? && models.none? { |m| m[:id] == selected }
+        retired = CompletionKit::Model.find_by(model_id: selected)
+        if retired
+          models = models + [{ id: retired.model_id, name: "#{retired.display_name || retired.model_id} (retired)", provider: retired.provider }]
+        end
+      end
+
+      groups = models.group_by do |m|
+        if m[:provider] == "openrouter"
+          upstream = m[:id].to_s.split("/", 2).first
+          "OpenRouter — #{upstream}"
+        else
+          ck_provider_label(m[:provider])
+        end
+      end
+
+      ordered_keys = groups.keys.sort_by { |label| [label.start_with?("OpenRouter") ? 1 : 0, label] }
+      grouped = ordered_keys.map { |label| [label, groups[label].map { |m| [m[:name], m[:id]] }] }
+      grouped_options_for_select(grouped, selected)
+    end
+
+    def ck_model_options_html(scope)
+      models = CompletionKit::ApiConfig.available_models(scope: scope)
+      return "" if models.empty?
+      ck_grouped_models(models)
+    end
+
+    def ck_score_kind(score)
+      return :pending if score.nil?
+      return :high if score >= CompletionKit.config.high_quality_threshold
+      return :medium if score >= CompletionKit.config.medium_quality_threshold
+
+      :low
+    end
+
+    def ck_word_diff_old(old_text, new_text)
+      diff_tokens(old_text, new_text, :old)
+    end
+
+    def ck_word_diff_new(old_text, new_text)
+      diff_tokens(old_text, new_text, :new)
+    end
+
+    private
+
+    def diff_tokens(old_text, new_text, side)
+      old_words = tokenize_for_diff(old_text)
+      new_words = tokenize_for_diff(new_text)
+      lcs = lcs_table(old_words, new_words)
+      result = []
+      i = old_words.length
+      j = new_words.length
+
+      changes = []
+      while i > 0 || j > 0
+        if i > 0 && j > 0 && old_words[i - 1] == new_words[j - 1]
+          changes.unshift([:equal, old_words[i - 1]])
+          i -= 1
+          j -= 1
+        elsif j > 0 && (i == 0 || lcs[i][j - 1] >= lcs[i - 1][j])
+          changes.unshift([:add, new_words[j - 1]])
+          j -= 1
+        else
+          changes.unshift([:remove, old_words[i - 1]])
+          i -= 1
+        end
+      end
+
+      changes.each do |type, token|
+        escaped = ERB::Util.html_escape(token)
+        if type == :equal
+          result << escaped
+        elsif type == :remove && side == :old
+          result << content_tag(:span, escaped, class: "ck-diff-del")
+        elsif type == :add && side == :new
+          result << content_tag(:span, escaped, class: "ck-diff-ins")
+        end
+      end
+
+      result.join.html_safe
+    end
+
+    def tokenize_for_diff(text)
+      text.to_s.scan(/\S+|\n| +/)
+    end
+
+    def lcs_table(a, b)
+      m = a.length
+      n = b.length
+      table = Array.new(m + 1) { Array.new(n + 1, 0) }
+      (1..m).each do |i|
+        (1..n).each do |j|
+          table[i][j] = if a[i - 1] == b[j - 1]
+                          table[i - 1][j - 1] + 1
+                        else
+                          [table[i - 1][j], table[i][j - 1]].max
+                        end
+        end
+      end
+      table
+    end
+  end
+end

data/app/jobs/completion_kit/application_job.rb

@@ -0,0 +1,4 @@
+module CompletionKit
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/completion_kit/generate_job.rb

@@ -0,0 +1,12 @@
+module CompletionKit
+  class GenerateJob < ApplicationJob
+    queue_as :default
+
+    def perform(run_id)
+      run = Run.find_by(id: run_id)
+      return unless run
+
+      run.generate_responses!
+    end
+  end
+end

data/app/jobs/completion_kit/judge_job.rb

@@ -0,0 +1,12 @@
+module CompletionKit
+  class JudgeJob < ApplicationJob
+    queue_as :default
+
+    def perform(run_id)
+      run = Run.find_by(id: run_id)
+      return unless run
+
+      run.judge_responses!
+    end
+  end
+end

data/app/jobs/completion_kit/model_discovery_job.rb

@@ -0,0 +1,29 @@
+module CompletionKit
+  class ModelDiscoveryJob < ApplicationJob
+    queue_as :default
+
+    def perform(provider_credential_id)
+      credential = ProviderCredential.find_by(id: provider_credential_id)
+      return unless credential
+
+      credential.update_columns(discovery_status: "discovering", discovery_current: 0, discovery_total: 0)
+      credential.reload
+      credential.broadcast_discovery_progress
+
+      service = ModelDiscoveryService.new(config: credential.config_hash)
+      service.refresh! do |current, total|
+        credential.update_columns(discovery_current: current, discovery_total: total)
+        credential.reload
+        credential.broadcast_discovery_progress
+      end
+
+      credential.update_columns(discovery_status: "completed", updated_at: Time.current)
+      credential.reload
+      credential.broadcast_discovery_complete
+    rescue StandardError
+      credential.update_columns(discovery_status: "failed")
+      credential.reload
+      credential.broadcast_discovery_progress
+    end
+  end
+end

data/app/mailers/completion_kit/application_mailer.rb

@@ -0,0 +1,6 @@
+module CompletionKit
+  class ApplicationMailer < ActionMailer::Base
+    default from: "from@example.com"
+    layout "mailer"
+  end
+end

data/app/models/completion_kit/application_record.rb

@@ -0,0 +1,5 @@
+module CompletionKit
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/completion_kit/criteria.rb

@@ -0,0 +1,22 @@
+module CompletionKit
+  class Criteria < ApplicationRecord
+    self.table_name = "completion_kit_criteria"
+
+    has_many :criteria_memberships, -> { order(:position, :id) }, dependent: :destroy
+    has_many :metrics, through: :criteria_memberships
+
+    validates :name, presence: true
+
+    def ordered_metrics
+      criteria_memberships.includes(:metric).map(&:metric).compact
+    end
+
+    def as_json(options = {})
+      {
+        id: id, name: name, description: description,
+        created_at: created_at, updated_at: updated_at,
+        metric_ids: metric_ids
+      }
+    end
+  end
+end

data/app/models/completion_kit/criteria_membership.rb

@@ -0,0 +1,20 @@
+module CompletionKit
+  class CriteriaMembership < ApplicationRecord
+    self.table_name = "completion_kit_criteria_memberships"
+
+    belongs_to :criteria, class_name: "CompletionKit::Criteria", foreign_key: "criteria_id"
+    belongs_to :metric
+
+    validates :metric_id, uniqueness: { scope: :criteria_id }
+
+    before_validation :set_default_position
+
+    private
+
+    def set_default_position
+      return if position.present? || criteria.blank?
+
+      self.position = criteria.criteria_memberships.maximum(:position).to_i + 1
+    end
+  end
+end

data/app/models/completion_kit/dataset.rb

@@ -0,0 +1,24 @@
+module CompletionKit
+  class Dataset < ApplicationRecord
+    has_many :runs, dependent: :restrict_with_error
+
+    validates :name, presence: true
+    validates :csv_data, presence: true
+
+    def as_json(options = {})
+      {
+        id: id, name: name, csv_data: csv_data,
+        created_at: created_at, updated_at: updated_at
+      }
+    end
+
+    def row_count
+      return 0 if csv_data.blank?
+
+      require "csv"
+      ::CSV.parse(csv_data, headers: true).length
+    rescue ::CSV::MalformedCSVError
+      0
+    end
+  end
+end