completion-kit 0.1.0.rc1

data/app/controllers/completion_kit/api/v1/base_controller.rb

@@ -0,0 +1,29 @@
+module CompletionKit
+  module Api
+    module V1
+      class BaseController < ActionController::API
+        before_action :authenticate_api!
+
+        private
+
+        def authenticate_api!
+          token = CompletionKit.config.api_token
+          unless token
+            render json: {error: "API token not configured"}, status: :unauthorized
+            return
+          end
+
+          provided = request.headers["Authorization"]&.match(/\ABearer (.+)\z/)&.[](1)
+          unless provided && ActiveSupport::SecurityUtils.secure_compare(provided, token)
+            render json: {error: "Unauthorized"}, status: :unauthorized
+          end
+        end
+
+        def not_found
+          render json: {error: "Record not found"}, status: :not_found
+        end
+
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api/v1/criteria_controller.rb

@@ -0,0 +1,62 @@
+module CompletionKit
+  module Api
+    module V1
+      class CriteriaController < BaseController
+        before_action :set_criteria, only: [:show, :update, :destroy]
+
+        def index
+          render json: Criteria.order(created_at: :desc)
+        end
+
+        def show
+          render json: @criteria
+        end
+
+        def create
+          criteria = Criteria.new(criteria_params.except(:metric_ids))
+          if criteria.save
+            replace_metric_memberships(criteria, params[:metric_ids]) if params.key?(:metric_ids)
+            render json: criteria.reload, status: :created
+          else
+            render json: {errors: criteria.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def update
+          if @criteria.update(criteria_params.except(:metric_ids))
+            replace_metric_memberships(@criteria, params[:metric_ids]) if params.key?(:metric_ids)
+            render json: @criteria.reload
+          else
+            render json: {errors: @criteria.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def destroy
+          @criteria.destroy!
+          head :no_content
+        end
+
+        private
+
+        def set_criteria
+          @criteria = Criteria.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def criteria_params
+          params.permit(:name, :description, metric_ids: [])
+        end
+
+        def replace_metric_memberships(criteria, metric_ids)
+          return unless metric_ids
+
+          criteria.criteria_memberships.delete_all
+          Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
+            criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
+          end
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api/v1/datasets_controller.rb

@@ -0,0 +1,51 @@
+module CompletionKit
+  module Api
+    module V1
+      class DatasetsController < BaseController
+        before_action :set_dataset, only: [:show, :update, :destroy]
+
+        def index
+          render json: Dataset.order(created_at: :desc)
+        end
+
+        def show
+          render json: @dataset
+        end
+
+        def create
+          dataset = Dataset.new(dataset_params)
+          if dataset.save
+            render json: dataset, status: :created
+          else
+            render json: {errors: dataset.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def update
+          if @dataset.update(dataset_params)
+            render json: @dataset
+          else
+            render json: {errors: @dataset.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def destroy
+          @dataset.destroy!
+          head :no_content
+        end
+
+        private
+
+        def set_dataset
+          @dataset = Dataset.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def dataset_params
+          params.permit(:name, :csv_data)
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api/v1/metrics_controller.rb

@@ -0,0 +1,51 @@
+module CompletionKit
+  module Api
+    module V1
+      class MetricsController < BaseController
+        before_action :set_metric, only: [:show, :update, :destroy]
+
+        def index
+          render json: Metric.order(created_at: :desc)
+        end
+
+        def show
+          render json: @metric
+        end
+
+        def create
+          metric = Metric.new(metric_params)
+          if metric.save
+            render json: metric, status: :created
+          else
+            render json: {errors: metric.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def update
+          if @metric.update(metric_params)
+            render json: @metric
+          else
+            render json: {errors: @metric.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def destroy
+          @metric.destroy!
+          head :no_content
+        end
+
+        private
+
+        def set_metric
+          @metric = Metric.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def metric_params
+          params.permit(:name, :instruction, evaluation_steps: [], rubric_bands: [:stars, :description])
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api/v1/prompts_controller.rb

@@ -0,0 +1,64 @@
+module CompletionKit
+  module Api
+    module V1
+      class PromptsController < BaseController
+        before_action :set_prompt, only: [:show, :update, :destroy, :publish]
+
+        def index
+          render json: Prompt.order(created_at: :desc)
+        end
+
+        def show
+          render json: @prompt
+        end
+
+        def create
+          prompt = Prompt.new(prompt_params)
+          if prompt.save
+            render json: prompt, status: :created
+          else
+            render json: {errors: prompt.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def update
+          if @prompt.runs.exists?
+            new_prompt = @prompt.clone_as_new_version(prompt_params.to_h)
+            new_prompt.publish!
+            render json: new_prompt
+          elsif @prompt.update(prompt_params)
+            render json: @prompt
+          else
+            render json: {errors: @prompt.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def destroy
+          @prompt.destroy!
+          head :no_content
+        end
+
+        def publish
+          @prompt.publish!
+          render json: @prompt.reload
+        end
+
+        private
+
+        def set_prompt
+          @prompt = if params[:id].to_s.match?(/\A\d+\z/)
+                      Prompt.find(params[:id])
+                    else
+                      Prompt.current_for(params[:id])
+                    end
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def prompt_params
+          params.permit(:name, :description, :template, :llm_model)
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api/v1/provider_credentials_controller.rb

@@ -0,0 +1,51 @@
+module CompletionKit
+  module Api
+    module V1
+      class ProviderCredentialsController < BaseController
+        before_action :set_credential, only: [:show, :update, :destroy]
+
+        def index
+          render json: ProviderCredential.order(created_at: :desc)
+        end
+
+        def show
+          render json: @credential
+        end
+
+        def create
+          credential = ProviderCredential.new(credential_params)
+          if credential.save
+            render json: credential, status: :created
+          else
+            render json: {errors: credential.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def update
+          if @credential.update(credential_params)
+            render json: @credential
+          else
+            render json: {errors: @credential.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def destroy
+          @credential.destroy!
+          head :no_content
+        end
+
+        private
+
+        def set_credential
+          @credential = ProviderCredential.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def credential_params
+          params.permit(:provider, :api_key, :api_endpoint)
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api/v1/responses_controller.rb

@@ -0,0 +1,32 @@
+module CompletionKit
+  module Api
+    module V1
+      class ResponsesController < BaseController
+        before_action :set_run
+        before_action :set_response, only: [:show]
+
+        def index
+          render json: @run.responses.includes(:reviews)
+        end
+
+        def show
+          render json: @response
+        end
+
+        private
+
+        def set_run
+          @run = Run.find(params[:run_id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def set_response
+          @response = @run.responses.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -0,0 +1,71 @@
+module CompletionKit
+  module Api
+    module V1
+      class RunsController < BaseController
+        before_action :set_run, only: [:show, :update, :destroy, :generate, :judge]
+
+        def index
+          render json: Run.order(created_at: :desc)
+        end
+
+        def show
+          render json: @run
+        end
+
+        def create
+          run = Run.new(run_params.except(:metric_ids))
+          if run.save
+            replace_run_metrics(run, params[:metric_ids])
+            render json: run.reload, status: :created
+          else
+            render json: {errors: run.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def update
+          if @run.update(run_params.except(:metric_ids))
+            replace_run_metrics(@run, params[:metric_ids]) if params.key?(:metric_ids)
+            render json: @run.reload
+          else
+            render json: {errors: @run.errors}, status: :unprocessable_entity
+          end
+        end
+
+        def destroy
+          @run.destroy!
+          head :no_content
+        end
+
+        def generate
+          GenerateJob.perform_later(@run.id)
+          render json: @run.reload, status: :accepted
+        end
+
+        def judge
+          JudgeJob.perform_later(@run.id)
+          render json: @run.reload, status: :accepted
+        end
+
+        private
+
+        def set_run
+          @run = Run.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          not_found
+        end
+
+        def run_params
+          params.permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, metric_ids: [])
+        end
+
+        def replace_run_metrics(run, metric_ids)
+          return unless metric_ids
+          run.run_metrics.delete_all
+          Array(metric_ids).reject(&:blank?).each_with_index do |metric_id, index|
+            run.run_metrics.create!(metric_id: metric_id, position: index + 1)
+          end
+        end
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/api_reference_controller.rb

@@ -0,0 +1,9 @@
+module CompletionKit
+  class ApiReferenceController < ApplicationController
+    def index
+      @published_prompts = Prompt.current_versions.order(name: :asc)
+      @token = CompletionKit.config.api_token
+      @base_url = request.base_url + request.script_name
+    end
+  end
+end

data/app/controllers/completion_kit/application_controller.rb

@@ -0,0 +1,31 @@
+module CompletionKit
+  class ApplicationController < ActionController::Base
+    helper Heroicons::IconsHelper
+    layout "completion_kit/application"
+
+    before_action :authenticate_completion_kit!
+
+    private
+
+    def authenticate_completion_kit!
+      cfg = CompletionKit.config
+
+      if (cfg.username && !cfg.password) || (cfg.password && !cfg.username)
+        raise CompletionKit::ConfigurationError,
+          "Both username and password are required for built-in auth."
+      end
+
+      if cfg.auth_strategy
+        cfg.auth_strategy.call(self)
+      elsif cfg.username && cfg.password
+        authenticate_or_request_with_http_basic("CompletionKit") do |u, p|
+          ActiveSupport::SecurityUtils.secure_compare(u, cfg.username) &
+            ActiveSupport::SecurityUtils.secure_compare(p, cfg.password)
+        end
+      elsif Rails.env.production?
+        render plain: "CompletionKit authentication not configured. See README for setup instructions.",
+               status: :forbidden
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/criteria_controller.rb

@@ -0,0 +1,67 @@
+module CompletionKit
+  class CriteriaController < ApplicationController
+    before_action :set_criteria, only: [:show, :edit, :update, :destroy]
+
+    def index
+      @criterias = Criteria.includes(:metrics).order(:name)
+    end
+
+    def show
+    end
+
+    def new
+      @criteria = Criteria.new
+      @metrics = Metric.order(:name)
+    end
+
+    def edit
+      @metrics = Metric.order(:name)
+    end
+
+    def create
+      @criteria = Criteria.new(criteria_params.except(:metric_ids))
+      @metrics = Metric.order(:name)
+
+      if @criteria.save
+        replace_metric_memberships
+        redirect_to criterion_path(@criteria), notice: "Criteria was successfully created."
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def update
+      @metrics = Metric.order(:name)
+
+      if @criteria.update(criteria_params.except(:metric_ids))
+        replace_metric_memberships
+        redirect_to criterion_path(@criteria), notice: "Criteria was successfully updated."
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      @criteria.destroy
+      redirect_to criteria_path, notice: "Criteria was successfully destroyed."
+    end
+
+    private
+
+    def set_criteria
+      @criteria = Criteria.find(params[:id])
+    end
+
+    def criteria_params
+      params.require(:criteria).permit(:name, :description, metric_ids: [])
+    end
+
+    def replace_metric_memberships
+      metric_ids = Array(criteria_params[:metric_ids]).reject(&:blank?)
+      @criteria.criteria_memberships.delete_all
+      metric_ids.each_with_index do |metric_id, index|
+        @criteria.criteria_memberships.create!(metric_id: metric_id, position: index + 1)
+      end
+    end
+  end
+end

data/app/controllers/completion_kit/datasets_controller.rb

@@ -0,0 +1,53 @@
+module CompletionKit
+  class DatasetsController < ApplicationController
+    before_action :set_dataset, only: [:show, :edit, :update, :destroy]
+
+    def index
+      @datasets = Dataset.includes(:runs).order(created_at: :desc)
+    end
+
+    def show
+      @runs = @dataset.runs.includes(:prompt, :responses).order(created_at: :desc)
+    end
+
+    def new
+      @dataset = Dataset.new
+    end
+
+    def edit
+    end
+
+    def create
+      @dataset = Dataset.new(dataset_params)
+
+      if @dataset.save
+        redirect_to datasets_path, notice: "Dataset was successfully created."
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def update
+      if @dataset.update(dataset_params)
+        redirect_to @dataset, notice: "Dataset was successfully updated."
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      @dataset.destroy
+      redirect_to datasets_path, notice: "Dataset was successfully destroyed."
+    end
+
+    private
+
+    def set_dataset
+      @dataset = Dataset.find(params[:id])
+    end
+
+    def dataset_params
+      params.require(:dataset).permit(:name, :csv_data)
+    end
+  end
+end

data/app/controllers/completion_kit/mcp_controller.rb

@@ -0,0 +1,57 @@
+module CompletionKit
+  class McpController < Api::V1::BaseController
+    def handle
+      request_body = JSON.parse(request.body.read)
+
+      if request_body["method"] == "initialize"
+        result = McpDispatcher.initialize_session
+        session_id = result.delete(:session_id)
+        response.headers["Mcp-Session-Id"] = session_id
+        render json: jsonrpc_response(request_body["id"], result)
+        return
+      end
+
+      if request_body["method"] == "notifications/initialized"
+        head :ok
+        return
+      end
+
+      session_id = request.headers["Mcp-Session-Id"]
+      unless session_id && Rails.cache.exist?("mcp_session:#{session_id}")
+        render json: jsonrpc_error(request_body["id"], -32000, "Session not initialized. Send initialize first."), status: :bad_request
+        return
+      end
+
+      result = McpDispatcher.dispatch(request_body["method"], request_body["params"])
+      render json: jsonrpc_response(request_body["id"], result)
+    rescue JSON::ParserError
+      render json: jsonrpc_error(nil, -32700, "Parse error"), status: :bad_request
+    rescue McpDispatcher::MethodNotFound => e
+      render json: jsonrpc_error(request_body.dig("id"), -32601, e.message), status: :ok
+    rescue McpDispatcher::InvalidParams => e
+      render json: jsonrpc_error(request_body.dig("id"), -32602, e.message), status: :ok
+    rescue ActiveRecord::RecordNotFound => e
+      render json: jsonrpc_error(request_body.dig("id"), -32602, e.message), status: :ok
+    rescue ActiveRecord::RecordInvalid, ActiveRecord::InvalidForeignKey => e
+      render json: jsonrpc_error(request_body.dig("id"), -32602, e.message), status: :ok
+    rescue StandardError => e
+      render json: jsonrpc_error(request_body.dig("id"), -32603, e.message), status: :ok
+    end
+
+    def destroy
+      session_id = request.headers["Mcp-Session-Id"]
+      Rails.cache.delete("mcp_session:#{session_id}") if session_id
+      head :ok
+    end
+
+    private
+
+    def jsonrpc_response(id, result)
+      {jsonrpc: "2.0", id: id, result: result}
+    end
+
+    def jsonrpc_error(id, code, message)
+      {jsonrpc: "2.0", id: id, error: {code: code, message: message}}
+    end
+  end
+end

data/app/controllers/completion_kit/metrics_controller.rb

@@ -0,0 +1,52 @@
+module CompletionKit
+  class MetricsController < ApplicationController
+    before_action :set_metric, only: [:show, :edit, :update, :destroy]
+
+    def index
+      @metrics = Metric.order(:name)
+    end
+
+    def show
+    end
+
+    def new
+      @metric = Metric.new
+    end
+
+    def edit
+    end
+
+    def create
+      @metric = Metric.new(metric_params)
+
+      if @metric.save
+        redirect_to metric_path(@metric), notice: "Metric was successfully created."
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def update
+      if @metric.update(metric_params)
+        redirect_to metric_path(@metric), notice: "Metric was successfully updated."
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      @metric.destroy
+      redirect_to metrics_path, notice: "Metric was successfully destroyed."
+    end
+
+    private
+
+    def set_metric
+      @metric = Metric.find(params[:id])
+    end
+
+    def metric_params
+      params.require(:metric).permit(:name, :instruction, evaluation_steps: [], rubric_bands: [:stars, :description])
+    end
+  end
+end