completion-kit 0.1.0.rc1

data/app/models/completion_kit/metric.rb

@@ -0,0 +1,97 @@
+module CompletionKit
+  class Metric < ApplicationRecord
+    DEFAULT_RUBRIC_BANDS = [
+      { "stars" => 5, "description" => "Fully meets or exceeds all criteria. No meaningful issues." },
+      { "stars" => 4, "description" => "Meets criteria well. Minor issues only." },
+      { "stars" => 3, "description" => "Meets criteria adequately. Some room for improvement." },
+      { "stars" => 2, "description" => "Partially meets criteria. Significant gaps or frequent errors." },
+      { "stars" => 1, "description" => "Fails to meet the criteria. Major errors or completely off-target." }
+    ].freeze
+
+    has_many :criteria_memberships, dependent: :destroy
+    has_many :criterias, through: :criteria_memberships, source: :criteria
+    has_many :reviews, dependent: :nullify
+
+    serialize :rubric_bands, coder: JSON
+    serialize :evaluation_steps, coder: JSON
+
+    validates :name, presence: true
+    validates :key, uniqueness: true, allow_nil: true
+
+    before_validation :generate_key
+    before_validation :normalize_rubric_bands
+    before_validation :set_defaults
+
+    def self.default_rubric_bands
+      DEFAULT_RUBRIC_BANDS.map(&:dup)
+    end
+
+    def self.default_rubric_text
+      rubric_text_for(default_rubric_bands)
+    end
+
+    def self.rubric_text_for(bands)
+      Array(bands).sort_by { |b| -(b["stars"] || 0) }.map do |band|
+        stars = band["stars"].to_i
+        label = stars == 1 ? "1 star" : "#{stars} stars"
+        "#{label}: #{band["description"]}"
+      end.join("\n\n")
+    end
+
+    def self.normalize_rubric_bands(raw_bands)
+      bands = raw_bands.is_a?(Hash) ? raw_bands.values : Array(raw_bands)
+      band_map = bands.each_with_object({}) do |band, acc|
+        next unless band.respond_to?(:to_h)
+
+        normalized = band.to_h.stringify_keys.slice("stars", "description")
+        stars = normalized["stars"].to_i
+        next unless (1..5).cover?(stars)
+
+        acc[stars] = {
+          "stars" => stars,
+          "description" => normalized["description"].to_s.strip
+        }
+      end
+
+      default_rubric_bands.map do |default_band|
+        stars = default_band["stars"]
+        band = band_map[stars]
+        {
+          "stars" => stars,
+          "description" => band && band["description"].present? ? band["description"] : default_band["description"]
+        }
+      end
+    end
+
+    def rubric_bands_for_form
+      self.class.normalize_rubric_bands(rubric_bands)
+    end
+
+    def display_rubric_text
+      self.class.rubric_text_for(rubric_bands_for_form)
+    end
+
+    def as_json(options = {})
+      {
+        id: id, name: name, key: key, instruction: instruction,
+        evaluation_steps: evaluation_steps, rubric_bands: rubric_bands,
+        created_at: created_at, updated_at: updated_at
+      }
+    end
+
+    private
+
+    def generate_key
+      self.key ||= name.parameterize if name.present?
+    end
+
+    def set_defaults
+      self.evaluation_steps ||= []
+      self.rubric_bands = self.class.default_rubric_bands if rubric_bands.blank?
+    end
+
+    def normalize_rubric_bands
+      self.rubric_bands = self.class.normalize_rubric_bands(rubric_bands) if rubric_bands.present?
+    end
+  end
+end

data/app/models/completion_kit/model.rb

@@ -0,0 +1,13 @@
+module CompletionKit
+  class Model < ApplicationRecord
+    STATUSES = %w[active retired failed].freeze
+
+    validates :provider, presence: true
+    validates :model_id, presence: true, uniqueness: { scope: :provider }
+    validates :status, presence: true, inclusion: { in: STATUSES }
+
+    scope :active, -> { where(status: "active") }
+    scope :for_generation, -> { active.where(supports_generation: true) }
+    scope :for_judging, -> { active.where(supports_judging: true) }
+  end
+end

data/app/models/completion_kit/prompt.rb

@@ -0,0 +1,99 @@
+module CompletionKit
+  class Prompt < ApplicationRecord
+    has_many :runs, dependent: :destroy
+    has_many :responses, through: :runs
+
+    validates :name, presence: true
+    validates :template, presence: true
+    validates :llm_model, presence: true
+    validates :family_key, presence: true
+    validates :version_number, presence: true, numericality: { only_integer: true, greater_than: 0 }
+
+    before_validation :assign_family_key, on: :create
+    before_validation :assign_version_number, on: :create
+    before_validation :set_defaults
+
+    scope :current_versions, -> { where(current: true).order(created_at: :desc) }
+
+    def self.available_models(provider: nil)
+      ApiConfig.available_models(provider: provider)
+    end
+
+    def self.current_for(identifier)
+      current_versions.find_by(family_key: identifier) ||
+        current_versions.find_by(name: identifier) ||
+        current_versions.find { |p| p.slug == identifier.to_s } ||
+        raise(ActiveRecord::RecordNotFound)
+    end
+
+    def slug
+      name.to_s.downcase.strip.gsub(/[^a-z0-9]+/, "-").gsub(/\A-|-\z/, "")
+    end
+
+    def variables
+      CsvProcessor.extract_variables(self)
+    end
+
+    def version_label
+      "v#{version_number}"
+    end
+
+    def display_name
+      "#{name} — #{version_label}"
+    end
+
+    def family_versions
+      self.class.where(family_key: family_key).order(version_number: :desc, created_at: :desc)
+    end
+
+    def clone_as_new_version(overrides = {})
+      self.class.create!(
+        {
+          name: name,
+          description: description,
+          template: template,
+          llm_model: llm_model,
+          family_key: family_key,
+          version_number: next_version_number,
+          current: false,
+          published_at: nil
+        }.merge(overrides.compact)
+      )
+    end
+
+    def publish!
+      transaction do
+        self.class.where(family_key: family_key).where.not(id: id).update_all(current: false)
+        reload
+        update!(current: true, published_at: Time.current)
+      end
+    end
+
+    def as_json(options = {})
+      {
+        id: id, name: name, description: description, template: template,
+        llm_model: llm_model, family_key: family_key, version_number: version_number,
+        current: current, created_at: created_at, updated_at: updated_at
+      }
+    end
+
+    private
+
+    def assign_family_key
+      self.family_key ||= SecureRandom.uuid
+    end
+
+    def assign_version_number
+      self.version_number ||= next_version_number
+    end
+
+    def next_version_number
+      self.class.where(family_key: family_key).maximum(:version_number).to_i + 1
+    end
+
+    def set_defaults
+      self.current = true if current.nil?
+      self.published_at ||= Time.current if current?
+    end
+  end
+end

data/app/models/completion_kit/provider_credential.rb

@@ -0,0 +1,114 @@
+module CompletionKit
+  class ProviderCredential < ApplicationRecord
+    include Turbo::Broadcastable
+    PROVIDERS = %w[openai anthropic ollama openrouter].freeze
+    PROVIDER_LABELS = {
+      "openai" => "OpenAI",
+      "anthropic" => "Anthropic",
+      "ollama" => "Ollama / local endpoint",
+      "openrouter" => "OpenRouter"
+    }.freeze
+
+    encrypts :api_key
+
+    def as_json(options = {})
+      {
+        id: id, provider: provider, api_endpoint: api_endpoint,
+        created_at: created_at, updated_at: updated_at
+      }
+    end
+
+    def display_provider
+      PROVIDER_LABELS[provider] || provider.titleize
+    end
+
+    validates :provider, presence: true, inclusion: { in: PROVIDERS }, uniqueness: true
+
+    after_save :enqueue_discovery
+
+    def config_hash
+      {
+        provider: provider,
+        api_key: api_key,
+        api_endpoint: api_endpoint
+      }.compact
+    end
+
+    def available_models
+      LlmClient.for_provider(provider, config_hash).available_models
+    rescue StandardError
+      []
+    end
+
+    def configured?
+      LlmClient.for_provider(provider, config_hash).configured?
+    rescue StandardError
+      false
+    end
+
+    def prompt_count
+      model_ids = Model.where(provider: provider).pluck(:model_id)
+      return 0 if model_ids.empty?
+      Prompt.where(llm_model: model_ids, current: true).count
+    end
+
+    def judge_count
+      model_ids = Model.where(provider: provider).pluck(:model_id)
+      return 0 if model_ids.empty?
+      Run.where(judge_model: model_ids).count
+    end
+
+    def last_used_at
+      model_ids = Model.where(provider: provider).pluck(:model_id)
+      return nil if model_ids.empty?
+      prompt_scope = Prompt.where(llm_model: model_ids).select(:id)
+      Run.where("prompt_id IN (:prompts) OR judge_model IN (:models)",
+                prompts: prompt_scope, models: model_ids)
+         .where.not(status: "pending")
+         .maximum(:created_at)
+    end
+
+    def broadcast_discovery_progress
+      broadcast_replace_to(
+        "completion_kit_provider_#{id}",
+        target: "discovery_status_#{id}",
+        html: render_partial("completion_kit/provider_credentials/discovery_status", provider_credential: self)
+      )
+    end
+
+    def broadcast_discovery_complete
+      broadcast_discovery_progress
+      broadcast_model_dropdowns
+    end
+
+    private
+
+    def enqueue_discovery
+      update_columns(discovery_status: "discovering", discovery_current: 0, discovery_total: 0)
+      ModelDiscoveryJob.perform_later(id)
+    end
+
+    def broadcast_model_dropdowns
+      helper = ApplicationController.helpers
+      gen_html = helper.ck_model_options_html(:generation)
+      judge_html = '<option value="">None</option>' + helper.ck_model_options_html(:judging)
+
+      Turbo::StreamsChannel.broadcast_action_to(
+        "completion_kit_provider_#{id}",
+        action: :replace,
+        target: "prompt_llm_model",
+        html: "<select name=\"prompt[llm_model]\" id=\"prompt_llm_model\" class=\"ck-input\">#{gen_html}</select>"
+      )
+      Turbo::StreamsChannel.broadcast_action_to(
+        "completion_kit_provider_#{id}",
+        action: :replace,
+        target: "run_judge_model",
+        html: "<select name=\"run[judge_model]\" id=\"run_judge_model\" class=\"ck-input\">#{judge_html}</select>"
+      )
+    end
+
+    def render_partial(partial, locals)
+      CompletionKit::ApplicationController.render(partial: partial, locals: locals)
+    end
+  end
+end

data/app/models/completion_kit/response.rb

@@ -0,0 +1,30 @@
+module CompletionKit
+  class Response < ApplicationRecord
+    belongs_to :run
+    has_many :reviews, dependent: :destroy
+
+    delegate :prompt, to: :run
+
+    validates :response_text, presence: true
+
+    def as_json(options = {})
+      {
+        id: id, run_id: run_id, input_data: input_data,
+        response_text: response_text, expected_output: expected_output,
+        created_at: created_at, score: score, reviewed: reviewed?,
+        reviews: reviews.map(&:as_json)
+      }
+    end
+
+    def score
+      scores = reviews.select { |r| r.ai_score.present? }.map { |r| r.ai_score.to_f }
+      return nil if scores.empty?
+
+      (scores.sum / scores.length).round(2)
+    end
+
+    def reviewed?
+      reviews.any? { |r| r.ai_score.present? }
+    end
+  end
+end

data/app/models/completion_kit/review.rb

@@ -0,0 +1,28 @@
+module CompletionKit
+  class Review < ApplicationRecord
+    STATUSES = %w[pending evaluated failed].freeze
+
+    belongs_to :response
+    belongs_to :metric, optional: true
+
+    validates :metric_name, presence: true
+    validates :status, inclusion: { in: STATUSES }
+    validates :ai_score, numericality: { greater_than_or_equal_to: 1, less_than_or_equal_to: 5 }, allow_nil: true
+
+    before_validation :set_default_status
+
+    def as_json(options = {})
+      {
+        id: id, response_id: response_id, metric_id: metric_id,
+        metric_name: metric_name, ai_score: ai_score,
+        ai_feedback: ai_feedback, status: status
+      }
+    end
+
+    private
+
+    def set_default_status
+      self.status ||= "pending"
+    end
+  end
+end

data/app/models/completion_kit/run.rb

@@ -0,0 +1,253 @@
+module CompletionKit
+  class Run < ApplicationRecord
+    include Turbo::Broadcastable
+
+    STATUSES = %w[pending generating judging completed failed].freeze
+
+    belongs_to :prompt
+    belongs_to :dataset, optional: true
+    has_many :responses, dependent: :destroy
+    has_many :run_metrics, -> { order(:position) }, dependent: :destroy
+    has_many :metrics, through: :run_metrics
+    has_many :suggestions, dependent: :destroy
+
+    validates :name, presence: true
+    validates :status, inclusion: { in: STATUSES }
+
+    before_validation :set_default_status, on: :create
+    before_validation :set_auto_name, on: :create
+
+    def judge_configured?
+      judge_model.present? && metrics.any? && ApiConfig.valid_for_model?(judge_model)
+    end
+
+    def avg_score
+      all_reviews = responses.flat_map(&:reviews)
+      scores = all_reviews.map(&:ai_score).compact.map(&:to_f)
+      return nil if scores.empty?
+
+      (scores.sum / scores.length).round(2)
+    end
+
+    def metric_averages
+      all_reviews = responses.flat_map(&:reviews).select { |r| r.ai_score.present? }
+      all_reviews.group_by(&:metric_name).map do |name, reviews|
+        scores = reviews.map { |r| r.ai_score.to_f }
+        { name: name, avg: (scores.sum / scores.length).round(1) }
+      end
+    end
+
+    def generate_responses!
+      rows = if dataset
+               CsvProcessor.process_self(self)
+             else
+               [{}]
+             end
+
+      if rows.empty?
+        errors.add(:base, "Dataset has no rows")
+        return false
+      end
+
+      client = LlmClient.for_model(prompt.llm_model, ApiConfig.for_model(prompt.llm_model))
+
+      unless client.configured?
+        msg = "LLM API not configured: #{client.configuration_errors.join(', ')}"
+        errors.add(:base, msg)
+        update_columns(status: "failed", error_message: msg) if persisted?
+        return false
+      end
+
+      update!(status: "generating", progress_current: 0, progress_total: rows.length, error_message: nil)
+      responses.destroy_all
+      broadcast_ui
+      broadcast_clear_responses
+
+      rows.each_with_index do |row, index|
+        input = row.empty? ? nil : row.to_json
+        rendered = CsvProcessor.apply_variables(prompt, row)
+        response_text = client.generate_completion(rendered, model: prompt.llm_model, temperature: temperature)
+
+        resp = responses.create!(
+          input_data: input,
+          response_text: response_text,
+          expected_output: row["expected_output"]
+        )
+
+        update_columns(progress_current: index + 1)
+        broadcast_progress
+        broadcast_response(resp)
+      end
+
+      if judge_configured?
+        judge_responses!
+      else
+        update!(status: "completed")
+        broadcast_ui
+      end
+
+      true
+    rescue Faraday::Error => e
+      update_columns(status: "failed", error_message: e.message)
+      errors.add(:base, e.message)
+      broadcast_ui
+      false
+    rescue StandardError => e
+      update_columns(status: "failed", error_message: e.message) if persisted?
+      errors.add(:base, e.message)
+      broadcast_ui if persisted?
+      false
+    end
+
+    def judge_responses!
+      total_evaluations = responses.count * metrics.count
+      update!(status: "judging", progress_current: 0, progress_total: total_evaluations, error_message: nil)
+      broadcast_ui
+
+      judge = JudgeService.new(ApiConfig.for_model(judge_model).merge(judge_model: judge_model))
+      evaluation_count = 0
+
+      responses.find_each do |response|
+        metrics.each do |metric|
+          evaluation = judge.evaluate(
+            response.response_text,
+            response.expected_output,
+            prompt.template,
+            criteria: metric.respond_to?(:instruction) ? metric.instruction.to_s : "",
+            evaluation_steps: metric.respond_to?(:evaluation_steps) ? metric.evaluation_steps : nil,
+            rubric_text: metric.respond_to?(:display_rubric_text) ? metric.display_rubric_text : nil,
+            input_data: response.input_data
+          )
+
+          response.reviews.find_or_initialize_by(metric_id: metric.id).tap do |review|
+            review.assign_attributes(
+              metric_name: metric.name,
+              instruction: metric.respond_to?(:instruction) ? metric.instruction.to_s : "",
+              status: "evaluated",
+              ai_score: evaluation[:score],
+              ai_feedback: evaluation[:feedback]
+            )
+            review.save!
+          end
+
+          evaluation_count += 1
+          update_columns(progress_current: evaluation_count)
+          broadcast_progress
+        end
+
+        broadcast_response_update(response)
+      end
+
+      update!(status: "completed")
+      broadcast_ui
+      true
+    rescue Faraday::Error => e
+      update_columns(status: "failed", error_message: e.message)
+      errors.add(:base, e.message)
+      broadcast_ui
+      false
+    rescue StandardError => e
+      update_columns(status: "failed", error_message: e.message) if persisted?
+      errors.add(:base, e.message)
+      broadcast_ui if persisted?
+      false
+    end
+
+    def as_json(options = {})
+      {
+        id: id, name: name, status: status, prompt_id: prompt_id,
+        dataset_id: dataset_id, judge_model: judge_model, temperature: temperature,
+        created_at: created_at, updated_at: updated_at,
+        responses_count: responses.count, avg_score: avg_score,
+        progress_current: progress_current, progress_total: progress_total,
+        error_message: error_message, metric_ids: metric_ids
+      }
+    end
+
+    private
+
+    def broadcast_ui
+      broadcast_progress
+      broadcast_status_header
+      broadcast_actions
+      broadcast_sort_toolbar
+    end
+
+    def render_engine_partial(partial, locals)
+      CompletionKit::ApplicationController.render(
+        partial: partial,
+        locals: locals
+      )
+    end
+
+    def broadcast_progress
+      reload
+      broadcast_replace_to(
+        "completion_kit_run_#{id}",
+        target: "run_progress",
+        html: render_engine_partial("completion_kit/runs/progress", run: self)
+      )
+      broadcast_status_header
+    end
+
+    def broadcast_status_header
+      broadcast_replace_to(
+        "completion_kit_run_#{id}",
+        target: "run_status_header",
+        html: render_engine_partial("completion_kit/runs/status_header", run: self)
+      )
+    end
+
+    def broadcast_actions
+      broadcast_replace_to(
+        "completion_kit_run_#{id}",
+        target: "run_actions",
+        html: render_engine_partial("completion_kit/runs/actions", run: self)
+      )
+    end
+
+    def broadcast_sort_toolbar
+      broadcast_replace_to(
+        "completion_kit_run_#{id}",
+        target: "run_sort_toolbar",
+        html: render_engine_partial("completion_kit/runs/sort_toolbar", run: self)
+      )
+    end
+
+    def broadcast_clear_responses
+      broadcast_replace_to(
+        "completion_kit_run_#{id}",
+        target: "run_responses",
+        html: '<div id="run_responses"></div>'
+      )
+    end
+
+    def broadcast_response(response)
+      broadcast_append_to(
+        "completion_kit_run_#{id}",
+        target: "run_responses",
+        html: render_engine_partial("completion_kit/runs/response_row", run: self, response: response, index: responses.where("id <= ?", response.id).count)
+      )
+    end
+
+    def broadcast_response_update(response)
+      broadcast_replace_to(
+        "completion_kit_run_#{id}",
+        target: "response_#{response.id}",
+        html: render_engine_partial("completion_kit/runs/response_row", run: self, response: response, index: responses.where("id <= ?", response.id).count)
+      )
+    end
+
+    def set_default_status
+      self.status ||= "pending"
+    end
+
+    def set_auto_name
+      return if name.present?
+      return unless prompt.present?
+
+      count = Run.where(prompt_id: prompt_id).count + 1
+      self.name = "#{prompt.name} — v#{prompt.version_number} ##{count}"
+    end
+  end
+end

data/app/models/completion_kit/run_metric.rb

@@ -0,0 +1,6 @@
+module CompletionKit
+  class RunMetric < ApplicationRecord
+    belongs_to :run
+    belongs_to :metric
+  end
+end

data/app/models/completion_kit/suggestion.rb

@@ -0,0 +1,8 @@
+module CompletionKit
+  class Suggestion < ApplicationRecord
+    belongs_to :run
+    belongs_to :prompt
+
+    validates :suggested_template, presence: true
+  end
+end