prompt_engine 1.0.0

data/app/controllers/prompt_engine/versions_controller.rb

@@ -0,0 +1,90 @@
+module PromptEngine
+  class VersionsController < ApplicationController
+    layout "prompt_engine/admin"
+
+    before_action :set_prompt
+    before_action :set_version, only: [ :show, :restore ]
+    before_action :set_compare_versions, only: [ :compare ]
+
+    def index
+      @versions = @prompt.versions
+    end
+
+    def show
+    end
+
+    def compare
+      if @version_a && @version_b
+        @changes = calculate_changes(@version_a, @version_b)
+      else
+        redirect_to prompt_versions_path(@prompt), alert: "Please select two versions to compare"
+      end
+    end
+
+    def restore
+      ActiveRecord::Base.transaction do
+        @prompt.update!(
+          content: @version.content,
+          system_message: @version.system_message,
+          model: @version.model,
+          temperature: @version.temperature,
+          max_tokens: @version.max_tokens
+        )
+      end
+      redirect_to @prompt, notice: "Prompt restored to version #{@version.version_number}"
+    rescue => e
+      redirect_to prompt_versions_path(@prompt), alert: "Failed to restore version: #{e.message}"
+    end
+
+    private
+
+    def set_prompt
+      @prompt = Prompt.find(params[:prompt_id])
+    end
+
+    def set_version
+      @version = @prompt.versions.find(params[:id])
+    end
+
+    def set_compare_versions
+      if params[:version_a_id].present? && params[:version_b_id].present?
+        @version_a = @prompt.versions.find_by(id: params[:version_a_id])
+        @version_b = @prompt.versions.find_by(id: params[:version_b_id])
+      elsif params[:id].present?
+        @version_b = @prompt.versions.find(params[:id])
+        @version_a = @prompt.versions.where("version_number < ?", @version_b.version_number).order(version_number: :desc).first
+        @version_a ||= @version_b
+      end
+    end
+
+    def calculate_changes(version_a, version_b)
+      {
+        content: {
+          old: version_a.content,
+          new: version_b.content,
+          changed: version_a.content != version_b.content
+        },
+        system_message: {
+          old: version_a.system_message,
+          new: version_b.system_message,
+          changed: version_a.system_message != version_b.system_message
+        },
+        model: {
+          old: version_a.model,
+          new: version_b.model,
+          changed: version_a.model != version_b.model
+        },
+        temperature: {
+          old: version_a.temperature,
+          new: version_b.temperature,
+          changed: version_a.temperature != version_b.temperature
+        },
+        max_tokens: {
+          old: version_a.max_tokens,
+          new: version_b.max_tokens,
+          changed: version_a.max_tokens != version_b.max_tokens
+        }
+      }
+    end
+  end
+end

data/app/helpers/prompt_engine/application_helper.rb

@@ -0,0 +1,4 @@
+module PromptEngine
+  module ApplicationHelper
+  end
+end

data/app/jobs/prompt_engine/application_job.rb

@@ -0,0 +1,4 @@
+module PromptEngine
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/mailers/prompt_engine/application_mailer.rb

@@ -0,0 +1,6 @@
+module PromptEngine
+  class ApplicationMailer < ActionMailer::Base
+    default from: "from@example.com"
+    layout "mailer"
+  end
+end

data/app/models/prompt_engine/application_record.rb

@@ -0,0 +1,5 @@
+module PromptEngine
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/prompt_engine/eval_result.rb

@@ -0,0 +1,19 @@
+module PromptEngine
+  class EvalResult < ApplicationRecord
+    belongs_to :eval_run
+    belongs_to :test_case
+
+    scope :passed, -> { where(passed: true) }
+    scope :failed, -> { where(passed: false) }
+    scope :by_execution_time, -> { order(:execution_time_ms) }
+
+    def execution_time_seconds
+      return nil unless execution_time_ms
+      execution_time_ms / 1000.0
+    end
+
+    def status
+      passed? ? "passed" : "failed"
+    end
+  end
+end

data/app/models/prompt_engine/eval_run.rb

@@ -0,0 +1,40 @@
+module PromptEngine
+  class EvalRun < ApplicationRecord
+    belongs_to :eval_set
+    belongs_to :prompt_version
+    has_many :eval_results, dependent: :destroy
+
+    enum :status, { pending: 0, running: 1, completed: 2, failed: 3 }
+
+    scope :recent, -> { order(created_at: :desc) }
+    scope :by_status, ->(status) { where(status: status) }
+    scope :successful, -> { completed.where("passed_count > 0") }
+
+    def success_rate
+      return 0 if total_count.zero?
+      (passed_count.to_f / total_count * 100).round(1)
+    end
+
+    def duration
+      return nil unless started_at && completed_at
+      completed_at - started_at
+    end
+
+    def duration_in_words
+      return "Not started" unless started_at
+      return "Running" if running?
+      return "Failed" if failed? && !completed_at
+
+      seconds = duration
+      return nil unless seconds
+
+      if seconds < 60
+        "#{seconds.round} seconds"
+      elsif seconds < 3600
+        "#{(seconds / 60).round} minutes"
+      else
+        "#{(seconds / 3600).round(1)} hours"
+      end
+    end
+  end
+end

data/app/models/prompt_engine/eval_set.rb

@@ -0,0 +1,97 @@
+module PromptEngine
+  class EvalSet < ApplicationRecord
+    belongs_to :prompt
+    has_many :test_cases, dependent: :destroy
+    has_many :eval_runs, dependent: :destroy
+
+    GRADER_TYPES = {
+      exact_match: "Exact Match",
+      regex: "Regular Expression",
+      contains: "Contains Text",
+      json_schema: "JSON Match (Exact)"
+    }.freeze
+
+    validates :name, presence: true
+    validates :name, uniqueness: { scope: :prompt_id }
+    validates :grader_type, presence: true, inclusion: { in: GRADER_TYPES.keys.map(&:to_s) }
+
+    validate :validate_grader_config
+
+    scope :by_name, -> { order(:name) }
+    scope :with_test_cases, -> { includes(:test_cases) }
+
+    def latest_run
+      eval_runs.recent.first
+    end
+
+    def average_success_rate
+      runs_with_results = eval_runs.completed.where("total_count > 0")
+      return 0 if runs_with_results.empty?
+
+      total_passed = runs_with_results.sum(:passed_count)
+      total_count = runs_with_results.sum(:total_count)
+
+      return 0 if total_count.zero?
+      (total_passed.to_f / total_count * 100).round(1)
+    end
+
+    def ready_to_run?
+      test_cases.any?
+    end
+
+    def grader_type_display
+      GRADER_TYPES[grader_type.to_sym] || grader_type.humanize
+    end
+
+    def requires_grader_config?
+      %w[regex json_schema].include?(grader_type)
+    end
+
+    private
+
+    def validate_grader_config
+      return unless requires_grader_config?
+
+      case grader_type
+      when "regex"
+        validate_regex_config
+      when "json_schema"
+        validate_json_schema_config
+      end
+    end
+
+    def validate_regex_config
+      pattern = grader_config["pattern"]
+
+      if pattern.blank?
+        errors.add(:grader_config, "regex pattern is required")
+        return
+      end
+
+      begin
+        Regexp.new(pattern)
+      rescue RegexpError => e
+        errors.add(:grader_config, "invalid regex pattern: #{e.message}")
+      end
+    end
+
+    def validate_json_schema_config
+      schema = grader_config["schema"]
+
+      if schema.blank?
+        errors.add(:grader_config, "JSON schema is required")
+        return
+      end
+
+      begin
+        JSON.parse(schema.to_json) if schema.is_a?(Hash)
+        # Basic schema validation - check for required fields
+        unless schema.is_a?(Hash) && schema["type"].present?
+          errors.add(:grader_config, "JSON schema must include a 'type' field")
+        end
+      rescue JSON::ParserError => e
+        errors.add(:grader_config, "invalid JSON schema: #{e.message}")
+      end
+    end
+  end
+end

data/app/models/prompt_engine/parameter.rb

@@ -0,0 +1,126 @@
+module PromptEngine
+  class Parameter < ApplicationRecord
+    self.table_name = "prompt_engine_parameters"
+
+    # Parameter types that can be used
+    TYPES = %w[string integer decimal boolean datetime date array json].freeze
+
+    belongs_to :prompt, class_name: "PromptEngine::Prompt"
+
+    validates :name, presence: true,
+              uniqueness: { scope: :prompt_id },
+              format: { with: /\A[a-zA-Z_][a-zA-Z0-9_]*\z/, message: "must start with a letter or underscore and contain only letters, numbers, and underscores" }
+    validates :parameter_type, presence: true, inclusion: { in: TYPES }
+    validates :required, inclusion: { in: [ true, false ] }
+
+    scope :required, -> { where(required: true) }
+    scope :optional, -> { where(required: false) }
+    scope :ordered, -> { order(position: :asc, created_at: :asc) }
+
+    before_validation :set_defaults
+
+    # Convert the parameter value based on its type
+    def cast_value(value)
+      return default_value if value.blank? && !required?
+
+      case parameter_type
+      when "integer"
+        value.to_i
+      when "decimal"
+        value.to_f
+      when "boolean"
+        ActiveModel::Type::Boolean.new.cast(value)
+      when "datetime"
+        DateTime.parse(value.to_s) rescue nil
+      when "date"
+        Date.parse(value.to_s) rescue nil
+      when "array"
+        value.is_a?(Array) ? value : value.to_s.split(",").map(&:strip)
+      when "json"
+        value.is_a?(String) ? JSON.parse(value) : value rescue {}
+      else
+        value.to_s
+      end
+    end
+
+    # Validate a value against this parameter's rules
+    def validate_value(value)
+      errors = []
+
+      if required? && value.blank?
+        errors << "#{name} is required"
+      end
+
+      if validation_rules.present?
+        # Apply custom validation rules
+        if validation_rules["min_length"] && value.to_s.length < validation_rules["min_length"]
+          errors << "#{name} must be at least #{validation_rules['min_length']} characters"
+        end
+
+        if validation_rules["max_length"] && value.to_s.length > validation_rules["max_length"]
+          errors << "#{name} must be at most #{validation_rules['max_length']} characters"
+        end
+
+        if validation_rules["pattern"] && !value.to_s.match?(Regexp.new(validation_rules["pattern"]))
+          errors << "#{name} must match pattern: #{validation_rules['pattern']}"
+        end
+
+        if validation_rules["min"] && cast_value(value) < validation_rules["min"]
+          errors << "#{name} must be at least #{validation_rules['min']}"
+        end
+
+        if validation_rules["max"] && cast_value(value) > validation_rules["max"]
+          errors << "#{name} must be at most #{validation_rules['max']}"
+        end
+      end
+
+      errors
+    end
+
+    # Generate form input attributes for this parameter
+    def form_input_options
+      options = {
+        label: name.humanize,
+        required: required?,
+        placeholder: example_value,
+        hint: description
+      }
+
+      case parameter_type
+      when "integer", "decimal"
+        options[:type] = "number"
+        options[:step] = parameter_type == "decimal" ? "0.01" : "1"
+        options[:min] = validation_rules["min"] if validation_rules&.dig("min")
+        options[:max] = validation_rules["max"] if validation_rules&.dig("max")
+      when "boolean"
+        options[:type] = "checkbox"
+      when "datetime"
+        options[:type] = "datetime-local"
+      when "date"
+        options[:type] = "date"
+      when "array"
+        options[:type] = "text"
+        options[:hint] = "#{description} (comma-separated values)"
+      when "json"
+        options[:type] = "textarea"
+        options[:hint] = "#{description} (JSON format)"
+      else
+        options[:type] = "text"
+        options[:minlength] = validation_rules["min_length"] if validation_rules&.dig("min_length")
+        options[:maxlength] = validation_rules["max_length"] if validation_rules&.dig("max_length")
+        options[:pattern] = validation_rules["pattern"] if validation_rules&.dig("pattern")
+      end
+
+      options[:value] = default_value if default_value.present?
+
+      options
+    end
+
+    private
+
+    def set_defaults
+      self.parameter_type ||= "string"
+      self.required = true if required.nil?
+    end
+  end
+end

data/app/models/prompt_engine/parameter_parser.rb

@@ -0,0 +1,39 @@
+module PromptEngine
+  class ParameterParser
+    attr_reader :content
+
+    def initialize(content)
+      @content = content
+    end
+
+    def extract_parameters
+      return [] if content.blank?
+
+      parameter_names = content.scan(/\{\{([^}]+)\}\}/).flatten.map(&:strip).uniq
+
+      parameter_names.map do |name|
+        {
+          name: name,
+          placeholder: "{{#{name}}}",
+          required: true
+        }
+      end
+    end
+
+    def replace_parameters(parameters = {})
+      result = content.dup
+
+      return result if parameters.nil?
+
+      parameters.each do |key, value|
+        result.gsub!("{{#{key}}}", value.to_s)
+      end
+
+      result
+    end
+
+    def has_parameters?
+      extract_parameters.any?
+    end
+  end
+end

data/app/models/prompt_engine/playground_run_result.rb

@@ -0,0 +1,20 @@
+module PromptEngine
+  class PlaygroundRunResult < ApplicationRecord
+    self.table_name = "prompt_engine_playground_run_results"
+
+    belongs_to :prompt_version, class_name: "PromptEngine::PromptVersion"
+
+    validates :provider, presence: true
+    validates :model, presence: true
+    validates :rendered_prompt, presence: true
+    validates :response, presence: true
+    validates :execution_time, presence: true, numericality: { greater_than_or_equal_to: 0 }
+    validates :token_count, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
+
+    serialize :parameters, coder: JSON
+
+    scope :recent, -> { order(created_at: :desc) }
+    scope :by_provider, ->(provider) { where(provider: provider) }
+    scope :successful, -> { where.not(response: nil) }
+  end
+end

data/app/models/prompt_engine/prompt.rb

@@ -0,0 +1,192 @@
+module PromptEngine
+  class Prompt < ApplicationRecord
+    self.table_name = "prompt_engine_prompts"
+
+    has_many :versions, -> { order(version_number: :desc) },
+             class_name: "PromptEngine::PromptVersion",
+             dependent: :destroy
+    has_many :parameters, -> { ordered },
+             class_name: "PromptEngine::Parameter",
+             dependent: :destroy
+    has_many :eval_sets,
+             class_name: "PromptEngine::EvalSet",
+             dependent: :destroy
+
+    attr_accessor :change_summary
+
+    validates :name, presence: true, uniqueness: { scope: :status }
+    validates :content, presence: true
+
+    accepts_nested_attributes_for :parameters, allow_destroy: true
+
+    enum :status, {
+      draft: "draft",
+      active: "active",
+      archived: "archived"
+    }, default: "draft"
+
+    scope :active, -> { where(status: "active") }
+    scope :by_name, -> { order(:name) }
+
+    after_create :create_initial_version
+    after_create :sync_parameters!
+    after_update :create_version_if_changed
+    after_update :sync_parameters!, if: :saved_change_to_content?
+    before_save :clean_orphaned_parameters
+
+    VERSIONED_ATTRIBUTES = %w[content system_message model temperature max_tokens metadata].freeze
+
+    def current_version
+      versions.first
+    end
+
+    def version_count
+      versions_count
+    end
+
+    def restore_version!(version_number)
+      version = versions.find_by!(version_number: version_number)
+      version.restore!
+    end
+
+    def version_at(version_number)
+      versions.find_by(version_number: version_number)
+    end
+
+    def versioned_attributes_changed?
+      # This method is for checking if versioned attributes have changed before save
+      (changed & VERSIONED_ATTRIBUTES).any?
+    end
+
+    # Parameter management methods
+    def detect_variables
+      # Don't cache the detector as content can change
+      PromptEngine::VariableDetector.new(content).variable_names
+    end
+
+    def sync_parameters!
+      detected_vars = detect_variables
+      existing_names = parameters.pluck(:name)
+
+      # Add new parameters
+      new_vars = detected_vars - existing_names
+      if new_vars.any?
+        # Get max position once, before the loop
+        max_position = parameters.maximum(:position) || 0
+        detector = PromptEngine::VariableDetector.new(content)
+
+        new_vars.each_with_index do |var_name, index|
+          var_info = detector.extract_variables.find { |v| v[:name] == var_name }
+
+          # Skip if parameter already exists (race condition protection)
+          next if parameters.exists?(name: var_name)
+
+          parameters.create!(
+            name: var_name,
+            parameter_type: var_info[:type],
+            required: var_info[:required],
+            position: max_position + index + 1
+          )
+        end
+      end
+
+      # Remove parameters that no longer exist
+      removed_vars = existing_names - detected_vars
+      parameters.where(name: removed_vars).destroy_all if removed_vars.any?
+
+      true
+    end
+
+    def render_with_params(provided_params = {})
+              detector = PromptEngine::VariableDetector.new(content)
+
+      # Validate all required parameters are provided
+      validation = validate_parameters(provided_params)
+      return { error: validation[:errors].join(", ") } unless validation[:valid]
+
+      # Cast parameters to their correct types, including defaults
+      casted_params = {}
+      parameters.each do |param|
+        value = provided_params[param.name] || provided_params[param.name.to_sym]
+        # Let cast_value handle the default value logic
+        casted_params[param.name] = param.cast_value(value)
+      end
+
+      # Also include any parameters not defined in the database but present in the template
+      detected_vars = detect_variables
+      detected_vars.each do |var_name|
+        unless casted_params.key?(var_name)
+          value = provided_params[var_name] || provided_params[var_name.to_sym]
+          casted_params[var_name] = value.to_s if value.present?
+        end
+      end
+
+      {
+        content: detector.render(casted_params),
+        system_message: system_message,
+        model: model,
+        temperature: temperature,
+        max_tokens: max_tokens,
+        parameters_used: casted_params
+      }
+    end
+
+    def validate_parameters(provided_params = {})
+      errors = []
+
+      parameters.each do |param|
+        value = provided_params[param.name] || provided_params[param.name.to_sym]
+        # Use default value if not provided and parameter is optional
+        if value.blank? && !param.required? && param.default_value.present?
+          value = param.default_value
+        end
+        param_errors = param.validate_value(value)
+        errors.concat(param_errors)
+      end
+
+      {
+        valid: errors.empty?,
+        errors: errors
+      }
+    end
+
+    private
+
+    def create_initial_version
+      versions.create!(
+        content: content,
+        system_message: system_message,
+        model: model,
+        temperature: temperature,
+        max_tokens: max_tokens,
+        metadata: metadata,
+        change_description: "Initial version"
+      )
+    end
+
+    def create_version_if_changed
+      # Check saved_changes in the after_update callback
+      return unless (saved_changes.keys & VERSIONED_ATTRIBUTES).any?
+
+      versions.create!(
+        content: content,
+        system_message: system_message,
+        model: model,
+        temperature: temperature,
+        max_tokens: max_tokens,
+        metadata: metadata,
+        change_description: "Updated: #{(saved_changes.keys & VERSIONED_ATTRIBUTES).join(', ')}"
+      )
+    end
+
+    def clean_orphaned_parameters
+      return unless content_changed?
+
+      # Mark parameters for destruction if their names are not in the content
+      detected_vars = detect_variables
+      parameters.each do |param|
+        param.mark_for_destruction unless detected_vars.include?(param.name)
+      end
+    end
+  end
+end