completion-kit 0.1.0.rc1

data/config/routes.rb

@@ -0,0 +1,55 @@
+CompletionKit::Engine.routes.draw do
+  root to: "prompts#index"
+
+  resources :prompts do
+    member do
+      post :publish
+    end
+  end
+
+  resources :datasets
+  resources :metrics
+  resources :criteria, controller: "criteria"
+
+  resources :runs do
+    member do
+      post :generate
+      post :judge
+      get :suggestion
+      post :suggest
+      post :apply_suggestion
+    end
+    resources :responses, only: [:show]
+  end
+
+  resources :provider_credentials, only: [:index, :new, :create, :edit, :update] do
+    post :refresh, on: :member
+  end
+  post "refresh_models", to: "provider_credentials#refresh_all", as: :refresh_models
+
+  get "api_reference", to: "api_reference#index", as: :api_reference
+
+  namespace :api do
+    namespace :v1 do
+      resources :prompts do
+        member do
+          post :publish
+        end
+      end
+      resources :runs do
+        member do
+          post :generate
+          post :judge
+        end
+        resources :responses, only: [:index, :show]
+      end
+      resources :datasets
+      resources :metrics
+      resources :criteria, controller: "criteria"
+      resources :provider_credentials
+    end
+  end
+
+  post "mcp", to: "mcp#handle"
+  delete "mcp", to: "mcp#destroy"
+end

data/db/migrate/20260311000001_create_completion_kit_tables.rb

@@ -0,0 +1,87 @@
+class CreateCompletionKitTables < ActiveRecord::Migration[7.1]
+  def change
+    create_table :completion_kit_prompts do |t|
+      t.string :name, null: false
+      t.text :description
+      t.text :template, null: false
+      t.string :llm_model, null: false
+      t.string :family_key, null: false
+      t.integer :version_number, null: false
+      t.boolean :current, default: true, null: false
+      t.datetime :published_at
+      t.timestamps
+    end
+
+    add_index :completion_kit_prompts, :family_key
+    add_index :completion_kit_prompts, [:family_key, :version_number], unique: true, name: "idx_ck_prompts_family_version"
+    add_index :completion_kit_prompts, [:family_key, :current], name: "idx_ck_prompts_family_current"
+
+    create_table :completion_kit_datasets do |t|
+      t.string :name, null: false
+      t.text :csv_data, null: false
+      t.timestamps
+    end
+
+    create_table :completion_kit_criteria do |t|
+      t.string :name, null: false
+      t.text :description
+      t.timestamps
+    end
+
+    create_table :completion_kit_metrics do |t|
+      t.string :name, null: false
+      t.text :criteria
+      t.text :evaluation_steps
+      t.text :rubric_bands
+      t.string :key
+      t.timestamps
+    end
+
+    add_index :completion_kit_metrics, :key, unique: true
+
+    create_table :completion_kit_criteria_memberships do |t|
+      t.references :criteria, null: false, foreign_key: { to_table: :completion_kit_criteria }
+      t.references :metric, null: false, foreign_key: { to_table: :completion_kit_metrics }
+      t.integer :position
+      t.timestamps
+    end
+
+    create_table :completion_kit_provider_credentials do |t|
+      t.string :provider, null: false
+      t.text :api_key
+      t.text :api_endpoint
+      t.timestamps
+    end
+
+    add_index :completion_kit_provider_credentials, :provider, unique: true
+
+    create_table :completion_kit_runs do |t|
+      t.string :name
+      t.references :prompt, null: false, foreign_key: { to_table: :completion_kit_prompts }
+      t.references :dataset, foreign_key: { to_table: :completion_kit_datasets }
+      t.references :criteria, foreign_key: { to_table: :completion_kit_criteria }
+      t.string :judge_model
+      t.string :status
+      t.timestamps
+    end
+
+    create_table :completion_kit_responses do |t|
+      t.references :run, null: false, foreign_key: { to_table: :completion_kit_runs }
+      t.text :input_data
+      t.text :response_text
+      t.text :expected_output
+      t.timestamps
+    end
+
+    create_table :completion_kit_reviews do |t|
+      t.references :response, null: false, foreign_key: { to_table: :completion_kit_responses }
+      t.references :metric, foreign_key: { to_table: :completion_kit_metrics }
+      t.string :metric_name
+      t.text :criteria
+      t.string :status
+      t.decimal :ai_score, precision: 4, scale: 1
+      t.text :ai_feedback
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20260326000001_rename_criteria_to_instruction_on_metrics_and_reviews.rb

@@ -0,0 +1,6 @@
+class RenameCriteriaToInstructionOnMetricsAndReviews < ActiveRecord::Migration[7.0]
+  def change
+    rename_column :completion_kit_metrics, :criteria, :instruction
+    rename_column :completion_kit_reviews, :criteria, :instruction
+  end
+end

data/db/migrate/20260327000001_add_progress_to_runs.rb

@@ -0,0 +1,6 @@
+class AddProgressToRuns < ActiveRecord::Migration[7.0]
+  def change
+    add_column :completion_kit_runs, :progress_current, :integer, default: 0
+    add_column :completion_kit_runs, :progress_total, :integer, default: 0
+  end
+end

data/db/migrate/20260327100001_replace_criteria_with_direct_metrics_on_runs.rb

@@ -0,0 +1,12 @@
+class ReplaceCriteriaWithDirectMetricsOnRuns < ActiveRecord::Migration[7.0]
+  def change
+    create_table :completion_kit_run_metrics do |t|
+      t.references :run, null: false, foreign_key: { to_table: :completion_kit_runs }
+      t.references :metric, null: false, foreign_key: { to_table: :completion_kit_metrics }
+      t.integer :position
+      t.timestamps
+    end
+
+    remove_reference :completion_kit_runs, :criteria, foreign_key: { to_table: :completion_kit_criteria }
+  end
+end

data/db/migrate/20260328000001_add_error_message_to_runs.rb

@@ -0,0 +1,5 @@
+class AddErrorMessageToRuns < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_runs, :error_message, :text
+  end
+end

data/db/migrate/20260329000001_create_completion_kit_models.rb

@@ -0,0 +1,20 @@
+class CreateCompletionKitModels < ActiveRecord::Migration[7.1]
+  def change
+    create_table :completion_kit_models do |t|
+      t.string :provider, null: false
+      t.string :model_id, null: false
+      t.string :display_name
+      t.string :status, null: false, default: "active"
+      t.boolean :supports_generation
+      t.boolean :supports_judging
+      t.text :generation_error
+      t.text :judging_error
+      t.datetime :probed_at
+      t.datetime :discovered_at
+      t.datetime :retired_at
+      t.timestamps
+    end
+
+    add_index :completion_kit_models, [:provider, :model_id], unique: true
+  end
+end

data/db/migrate/20260401170001_add_discovery_columns_to_completion_kit_provider_credentials.rb

@@ -0,0 +1,7 @@
+class AddDiscoveryColumnsToCompletionKitProviderCredentials < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_provider_credentials, :discovery_status, :string
+    add_column :completion_kit_provider_credentials, :discovery_current, :integer, default: 0
+    add_column :completion_kit_provider_credentials, :discovery_total, :integer, default: 0
+  end
+end

data/db/migrate/20260403000001_add_temperature_to_completion_kit_runs.rb

@@ -0,0 +1,5 @@
+class AddTemperatureToCompletionKitRuns < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_runs, :temperature, :float, default: 1.0
+  end
+end

data/db/migrate/20260403000002_create_completion_kit_suggestions.rb

@@ -0,0 +1,13 @@
+class CreateCompletionKitSuggestions < ActiveRecord::Migration[7.1]
+  def change
+    create_table :completion_kit_suggestions do |t|
+      t.references :run, null: false
+      t.references :prompt, null: false
+      t.text :reasoning
+      t.text :suggested_template
+      t.text :original_template
+      t.datetime :applied_at
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20260403000003_add_applied_at_to_completion_kit_suggestions.rb

@@ -0,0 +1,5 @@
+class AddAppliedAtToCompletionKitSuggestions < ActiveRecord::Migration[7.1]
+  def change
+    add_column :completion_kit_suggestions, :applied_at, :datetime
+  end
+end

data/lib/completion-kit.rb

@@ -0,0 +1 @@
+require "completion_kit"

data/lib/completion_kit/engine.rb

@@ -0,0 +1,35 @@
+require "turbo-rails"
+require "heroicons-rails"
+
+module CompletionKit
+  class Engine < ::Rails::Engine
+    isolate_namespace CompletionKit
+
+    initializer("completion_kit.inflections", before: :load_config_initializers) do
+      ActiveSupport::Inflector.inflections(:en) do |inflect|
+        inflect.irregular "criterion", "criteria"
+      end
+    end
+
+    paths.add "app/services", eager_load: true
+
+    def self.register_assets(app)
+      app.config.assets.precompile += %w( completion_kit/application.css completion_kit/evaluation_steps_controller.js completion_kit/logo.svg )
+    end
+
+    initializer("completion_kit.assets") { |app| Engine.register_assets(app) }
+
+    config.after_initialize do
+      cfg = CompletionKit.config
+      unless cfg.username || cfg.auth_strategy
+        Rails.logger.warn "[CompletionKit] WARNING: No authentication configured. All routes are publicly accessible."
+      end
+    end
+
+    config.generators do |g|
+      g.test_framework :rspec
+      g.fixture_replacement :factory_bot
+      g.factory_bot dir: "spec/factories"
+    end
+  end
+end

data/lib/completion_kit/version.rb

@@ -0,0 +1,3 @@
+module CompletionKit
+  VERSION = "0.1.0.rc1"
+end

data/lib/completion_kit.rb

@@ -0,0 +1,55 @@
+require "completion_kit/version"
+require "completion_kit/engine"
+
+module CompletionKit
+  class ConfigurationError < StandardError; end
+
+  class Configuration
+    attr_accessor :openai_api_key, :anthropic_api_key, :ollama_api_key, :ollama_api_endpoint
+    attr_accessor :judge_model, :high_quality_threshold, :medium_quality_threshold
+    attr_accessor :username, :password, :auth_strategy, :api_token
+
+    def initialize
+      @openai_api_key = ENV['OPENAI_API_KEY']
+      @anthropic_api_key = ENV['ANTHROPIC_API_KEY']
+      @ollama_api_key = ENV['OLLAMA_API_KEY']
+      @ollama_api_endpoint = ENV['OLLAMA_API_ENDPOINT']
+
+      @judge_model = "gpt-4.1"
+      @high_quality_threshold = 4
+      @medium_quality_threshold = 3
+    end
+  end
+
+  class << self
+    def config
+      @config ||= Configuration.new
+    end
+
+    def configure
+      yield(config) if block_given?
+    end
+
+    def current_prompt(identifier)
+      Prompt.current_for(identifier)
+    end
+
+    def current_prompt_payload(identifier)
+      prompt = current_prompt(identifier)
+
+      {
+        name: prompt.name,
+        family_key: prompt.family_key,
+        version_number: prompt.version_number,
+        template: prompt.template,
+        generation_model: prompt.llm_model
+      }
+    end
+
+    def render_current_prompt(identifier, variables = {})
+      prompt = current_prompt(identifier)
+      CsvProcessor.apply_variables(prompt, variables.stringify_keys)
+    end
+
+  end
+end

data/lib/generators/completion_kit/install_generator.rb

@@ -0,0 +1,21 @@
+module CompletionKit
+  class InstallGenerator < Rails::Generators::Base
+    source_root File.expand_path('templates', __dir__)
+    
+    def create_initializer
+      template 'initializer.rb', 'config/initializers/completion_kit.rb'
+    end
+    
+    def mount_engine
+      route "mount CompletionKit::Engine => '/completion_kit'"
+    end
+    
+    def copy_migrations
+      rake 'completion_kit:install:migrations'
+    end
+
+    def show_readme
+      readme 'README'
+    end
+  end
+end

data/lib/generators/completion_kit/templates/README

@@ -0,0 +1,20 @@
+=============================================
+CompletionKit Installation Instructions
+=============================================
+
+Thank you for installing CompletionKit!
+
+To complete the setup:
+
+1. Run database migrations:
+   $ bin/rails db:migrate
+
+2. Configure your API keys in config/initializers/completion_kit.rb
+
+3. Access the CompletionKit interface at:
+   http://your-app-url/completion_kit
+
+For more information, see the full documentation at:
+https://github.com/completionkit/completion_kit
+
+Happy prompt testing!

data/lib/generators/completion_kit/templates/initializer.rb

@@ -0,0 +1,43 @@
+# CompletionKit Configuration
+
+# Configure CompletionKit
+CompletionKit.configure do |config|
+  # 1. Environment variables:
+  # config.openai_api_key       = ENV['OPENAI_API_KEY']
+  # config.anthropic_api_key    = ENV['ANTHROPIC_API_KEY']
+  # config.ollama_api_key       = ENV['OLLAMA_API_KEY']
+  # config.ollama_api_endpoint  = ENV['OLLAMA_API_ENDPOINT']
+
+  # 2. Dotenv (.env file):
+  # require 'dotenv/load'
+  # config.openai_api_key = ENV['OPENAI_API_KEY'] if ENV['OPENAI_API_KEY']
+
+  # 3. Rails secrets (config/secrets.yml):
+  # secrets.yml ->
+  # development:
+  #   completion_kit:
+  #     openai_api_key: 'your-api-key'
+  # config.openai_api_key    = Rails.application.secrets.completion_kit[:openai_api_key]
+  # config.anthropic_api_key = Rails.application.secrets.completion_kit[:anthropic_api_key]
+  # config.ollama_api_key    = Rails.application.secrets.completion_kit[:ollama_api_key]
+  # config.ollama_api_endpoint = Rails.application.secrets.completion_kit[:ollama_api_endpoint]
+
+  # 4. Rails credentials (config/credentials.yml.enc):
+  # credentials.yml.enc ->
+  # completion_kit:
+  #   openai_api_key: 'your-api-key'
+  # config.openai_api_key    = Rails.application.credentials.completion_kit[:openai_api_key]
+  # config.anthropic_api_key = Rails.application.credentials.completion_kit[:anthropic_api_key]
+  # config.ollama_api_key    = Rails.application.credentials.completion_kit[:ollama_api_key]
+  # config.ollama_api_endpoint = Rails.application.credentials.completion_kit[:ollama_api_endpoint]
+
+  # 5. Direct assignment:
+  # config.openai_api_key = 'your-api-key-here'
+
+  # API Authentication
+  # config.api_token = ENV['COMPLETION_KIT_API_TOKEN']
+
+  # Web UI Authentication
+  # config.username = "admin"
+  # config.password = ENV['COMPLETION_KIT_PASSWORD']
+end