prompt_engine 1.0.0

data/app/models/prompt_engine/prompt_version.rb

@@ -0,0 +1,72 @@
+module PromptEngine
+  class PromptVersion < ApplicationRecord
+    self.table_name = "prompt_engine_prompt_versions"
+
+    belongs_to :prompt, class_name: "PromptEngine::Prompt", counter_cache: :versions_count
+    has_many :playground_run_results, class_name: "PromptEngine::PlaygroundRunResult", dependent: :destroy
+    has_many :eval_runs, class_name: "PromptEngine::EvalRun", dependent: :destroy
+
+    validates :version_number, presence: true,
+              numericality: { greater_than: 0 },
+              uniqueness: { scope: :prompt_id }
+    validates :content, presence: true
+
+    before_validation :set_version_number, on: :create
+    validate :ensure_immutability, on: :update
+
+    scope :latest, -> { order(version_number: :desc) }
+    scope :chronological, -> { order(created_at: :asc) }
+
+    def restore!
+      # Update the prompt attributes
+      prompt.update!(to_prompt_attributes)
+
+      # Check if a version was created (attributes changed)
+      latest_version = prompt.versions.first
+
+      if latest_version.created_at > 1.second.ago
+        # A new version was just created, update its description
+        latest_version.update_column(:change_description, "Restored from version #{version_number}")
+      else
+        # No version was created (no changes), create one manually
+        prompt.versions.create!(
+          to_prompt_attributes.merge(
+            change_description: "Restored from version #{version_number}"
+          )
+        )
+      end
+    end
+
+    def to_prompt_attributes
+      {
+        content: content,
+        system_message: system_message,
+        model: model,
+        temperature: temperature,
+        max_tokens: max_tokens,
+        metadata: metadata
+      }
+    end
+
+    private
+
+    def set_version_number
+      return if version_number.present?
+      return unless prompt
+
+      max_version = prompt.versions.maximum(:version_number) || 0
+      self.version_number = max_version + 1
+    end
+
+    def ensure_immutability
+      immutable_attributes = %w[content system_message model temperature max_tokens]
+      changed_immutable = (changed & immutable_attributes)
+
+      if changed_immutable.any?
+        changed_immutable.each do |attr|
+          errors.add(attr, "cannot be changed after creation")
+        end
+      end
+    end
+  end
+end

data/app/models/prompt_engine/setting.rb

@@ -0,0 +1,45 @@
+module PromptEngine
+  class Setting < ApplicationRecord
+    self.table_name = "prompt_engine_settings"
+
+    # Rails automatically encrypts these attributes
+    encrypts :openai_api_key
+    encrypts :anthropic_api_key
+
+    # Singleton pattern - only one settings record should exist
+    def self.instance
+      first_or_create!
+    end
+
+    # Check if API keys are configured
+    def openai_configured?
+      openai_api_key.present?
+    end
+
+    def anthropic_configured?
+      anthropic_api_key.present?
+    end
+
+    # Get masked API key for display (show only first and last 3 characters)
+    def masked_openai_api_key
+      mask_api_key(openai_api_key)
+    end
+
+    def masked_anthropic_api_key
+      mask_api_key(anthropic_api_key)
+    end
+
+    private
+
+    def mask_api_key(key)
+      return nil if key.blank?
+      return "*****" if key.length <= 6
+
+      # Show first 3 characters, then ..., then last 3 characters
+      # e.g., "sk-abc123xyz789" becomes "sk-...789"
+      first_part = key[0..2]  # First 3 characters
+      last_part = key[-3..]   # Last 3 characters
+      "#{first_part}...#{last_part}"
+    end
+  end
+end

data/app/models/prompt_engine/test_case.rb

@@ -0,0 +1,29 @@
+module PromptEngine
+  class TestCase < ApplicationRecord
+    belongs_to :eval_set
+    has_many :eval_results, dependent: :destroy
+
+    validates :input_variables, presence: true
+    validates :expected_output, presence: true
+
+    scope :by_description, -> { order(:description) }
+
+    def display_name
+      description.presence || "Test case ##{id}"
+    end
+
+    def passed_count
+      eval_results.where(passed: true).count
+    end
+
+    def failed_count
+      eval_results.where(passed: false).count
+    end
+
+    def success_rate
+      total = eval_results.count
+      return 0 if total.zero?
+      (passed_count.to_f / total * 100).round(1)
+    end
+  end
+end

data/app/services/prompt_engine/evaluation_runner.rb

@@ -0,0 +1,258 @@
+# Client is autoloaded by Rails
+
+module PromptEngine
+  class EvaluationRunner
+    def initialize(eval_run)
+      @eval_run = eval_run
+      @eval_set = eval_run.eval_set
+      @prompt_version = eval_run.prompt_version
+      @prompt = @prompt_version.prompt
+      @client = OpenAiEvalsClient.new
+    end
+
+    def execute
+      @eval_run.update!(status: :running, started_at: Time.current)
+
+      # Step 1: Create or get OpenAI eval configuration
+      ensure_openai_eval_exists
+
+      # Step 2: Create test data file in JSONL format
+      file_id = upload_test_data
+      @eval_run.update!(openai_file_id: file_id)
+
+      # Step 3: Create eval run on OpenAI
+      openai_run = create_openai_run(file_id)
+      @eval_run.update!(
+        openai_run_id: openai_run["id"],
+        report_url: openai_run["report_url"]
+      )
+
+      # Step 4: Poll for results
+      poll_for_results
+
+    rescue => e
+      @eval_run.update!(status: :failed, error_message: e.message)
+      raise
+    end
+
+    private
+
+    def ensure_openai_eval_exists
+      return if @eval_set.openai_eval_id.present?
+
+      # Build schema properties dynamically based on prompt parameters
+      schema_properties = {}
+      required_fields = []
+
+      # Add properties for each parameter in the prompt
+      @prompt.parameters.each do |param|
+        schema_properties[param.name] = { type: parameter_type_to_json_schema(param.parameter_type) }
+        required_fields << param.name if param.required?
+      end
+
+      # Always include expected_output
+      schema_properties["expected_output"] = { type: "string" }
+      required_fields << "expected_output"
+
+      # Create eval configuration on OpenAI
+      eval_config = @client.create_eval(
+        name: "#{@prompt.name} - #{@eval_set.name}",
+        data_source_config: {
+          type: "custom",
+          item_schema: {
+            type: "object",
+            properties: schema_properties,
+            required: required_fields
+          },
+          include_sample_schema: true
+        },
+        testing_criteria: build_testing_criteria
+      )
+
+      @eval_set.update!(openai_eval_id: eval_config["id"])
+    end
+
+    def upload_test_data
+      # Create temporary JSONL file
+      file_path = Rails.root.join("tmp", "eval_#{@eval_run.id}.jsonl")
+
+      File.open(file_path, "w") do |file|
+        @eval_set.test_cases.each do |test_case|
+          # Flatten the structure - put input variables directly on item
+          item_data = test_case.input_variables.dup
+          item_data["expected_output"] = test_case.expected_output
+
+          line = { item: item_data }
+          file.puts(line.to_json)
+        end
+      end
+
+      # Upload to OpenAI
+      response = @client.upload_file(file_path)
+
+      # Clean up
+      File.delete(file_path)
+
+      response["id"]
+    end
+
+    def create_openai_run(file_id)
+      # Build message template with prompt content
+      messages_template = [
+        {
+          role: "system",
+          content: @prompt_version.system_message || ""
+        },
+        {
+          role: "user",
+          content: build_templated_content
+        }
+      ]
+
+      @client.create_run(
+        eval_id: @eval_set.openai_eval_id,
+        name: "Run at #{Time.current}",
+        data_source: {
+          type: "completions",
+          model: @prompt_version.model || "gpt-4",
+          input_messages: {
+            type: "template",
+            template: messages_template
+          },
+          source: {
+            type: "file_id",
+            id: file_id
+          }
+        }
+      )
+    end
+
+    def build_templated_content
+      # Convert our {{variable}} syntax to OpenAI's template syntax
+      content = @prompt_version.content.dup
+
+      # Replace {{variable}} with {{ item.variable }} (flat structure)
+      content.gsub(/\{\{(\w+)\}\}/) do |match|
+        variable_name = $1
+        "{{ item.#{variable_name} }}"
+      end
+    end
+
+    def poll_for_results
+      max_attempts = 60  # 5 minutes with 5 second intervals
+      attempts = 0
+
+      loop do
+        attempts += 1
+
+        run_status = @client.get_run(
+          eval_id: @eval_set.openai_eval_id,
+          run_id: @eval_run.openai_run_id
+        )
+
+        case run_status["status"]
+        when "completed"
+          process_results(run_status)
+          break
+        when "failed", "canceled"
+          @eval_run.update!(
+            status: :failed,
+            error_message: run_status["error"] || "Eval run #{run_status["status"]}"
+          )
+          break
+        else
+          # Still running
+          if attempts >= max_attempts
+            @eval_run.update!(
+              status: :failed,
+              error_message: "Timeout waiting for eval results"
+            )
+            break
+          end
+
+          sleep 5
+        end
+      end
+    end
+
+    def process_results(run_status)
+      # Extract counts from OpenAI response
+      result_counts = run_status["result_counts"] || {}
+
+      @eval_run.update!(
+        status: :completed,
+        completed_at: Time.current,
+        total_count: result_counts["total"] || 0,
+        passed_count: result_counts["passed"] || 0,
+        failed_count: result_counts["failed"] || 0
+      )
+
+      # Note: Individual test results would need to be fetched separately
+      # For MVP, we just store the aggregate counts
+    end
+
+    def build_testing_criteria
+      case @eval_set.grader_type
+      when "exact_match"
+        [ {
+          type: "string_check",
+          name: "Exact match",
+          input: "{{ sample.output_text }}",
+          operation: "eq",
+          reference: "{{ item.expected_output }}"
+        } ]
+      when "regex"
+        [ {
+          type: "string_check",
+          name: "Regex match",
+          input: "{{ sample.output_text }}",
+          operation: "regex",
+          reference: @eval_set.grader_config["pattern"]
+        } ]
+      when "contains"
+        [ {
+          type: "string_check",
+          name: "Contains text",
+          input: "{{ sample.output_text }}",
+          operation: "contains",
+          reference: "{{ item.expected_output }}"
+        } ]
+      when "json_schema"
+        # OpenAI doesn't support json_schema_check directly
+        # For MVP, we'll just verify the output is valid JSON format
+        # and matches the expected output exactly
+        [ {
+          type: "string_check",
+          name: "JSON format validation",
+          input: "{{ sample.output_text }}",
+          operation: "eq",
+          reference: "{{ item.expected_output }}"
+        } ]
+      else
+        # Default to exact match
+        [ {
+          type: "string_check",
+          name: "Exact match",
+          input: "{{ sample.output_text }}",
+          operation: "eq",
+          reference: "{{ item.expected_output }}"
+        } ]
+      end
+    end
+
+    def parameter_type_to_json_schema(type)
+      case type
+      when "integer"
+        "integer"
+      when "decimal"
+        "number"
+      when "boolean"
+        "boolean"
+      when "array", "json"
+        "array"
+      else
+        "string"
+      end
+    end
+  end
+end

data/app/services/prompt_engine/playground_executor.rb

@@ -0,0 +1,124 @@
+module PromptEngine
+  class PlaygroundExecutor
+    attr_reader :prompt, :provider, :api_key, :parameters
+
+    MODELS = {
+      "anthropic" => "claude-3-5-sonnet-20241022",
+      "openai" => "gpt-4o"
+    }.freeze
+
+    def initialize(prompt:, provider:, api_key:, parameters: {})
+      @prompt = prompt
+      @provider = provider
+      @api_key = api_key
+      @parameters = parameters || {}
+    end
+
+    def execute
+      validate_inputs!
+
+      start_time = Time.current
+
+      # Replace parameters in prompt content
+      parser = ParameterParser.new(prompt.content)
+      processed_content = parser.replace_parameters(parameters)
+
+      # Configure RubyLLM with the appropriate API key
+      configure_ruby_llm
+
+      # Create chat instance with the model
+      chat = RubyLLM.chat(model: MODELS[provider])
+
+      # Apply temperature if specified
+      if prompt.temperature.present?
+        chat = chat.with_temperature(prompt.temperature)
+      end
+
+      # Apply system message if present
+      if prompt.system_message.present?
+        chat = chat.with_instructions(prompt.system_message)
+      end
+
+      # Execute the prompt
+      # Note: max_tokens may need to be passed differently depending on RubyLLM version
+      response = chat.ask(processed_content)
+
+      execution_time = (Time.current - start_time).round(3)
+
+      # Handle response based on its structure
+      response_content = if response.respond_to?(:content)
+                          response.content
+      elsif response.is_a?(String)
+                          response
+      else
+                          response.to_s
+      end
+
+      # Try to get token count if available
+      token_count = if response.respond_to?(:input_tokens) && response.respond_to?(:output_tokens)
+                      (response.input_tokens || 0) + (response.output_tokens || 0)
+      else
+                      0 # Default if token information isn't available
+      end
+
+      {
+        response: response_content,
+        execution_time: execution_time,
+        token_count: token_count,
+        model: MODELS[provider],
+        provider: provider
+      }
+    rescue StandardError => e
+      handle_error(e)
+    end
+
+    private
+
+    def validate_inputs!
+      raise ArgumentError, "Provider is required" if provider.blank?
+      raise ArgumentError, "API key is required" if api_key.blank?
+      raise ArgumentError, "Invalid provider" unless MODELS.key?(provider)
+    end
+
+    def configure_ruby_llm
+      require "ruby_llm"
+
+      RubyLLM.configure do |config|
+        case provider
+        when "anthropic"
+          config.anthropic_api_key = api_key
+        when "openai"
+          config.openai_api_key = api_key
+        end
+      end
+    end
+
+    def handle_error(error)
+      # Re-raise ArgumentError as-is for validation errors
+      raise error if error.is_a?(ArgumentError)
+
+      # Check for specific error types first
+      case error
+      when Net::HTTPUnauthorized
+        raise "Invalid API key"
+      when Net::HTTPTooManyRequests
+        raise "Rate limit exceeded. Please try again later."
+      when Net::HTTPError
+        raise "Network error. Please check your connection and try again."
+      else
+        # Then check error message patterns
+        error_message = error.message.to_s
+        case error_message
+        when /unauthorized/i
+          raise "Invalid API key"
+        when /rate limit/i
+          raise "Rate limit exceeded. Please try again later."
+        when /network/i
+          raise "Network error. Please check your connection and try again."
+        else
+          raise "An error occurred: #{error.message}"
+        end
+      end
+    end
+  end
+end

data/app/services/prompt_engine/variable_detector.rb

@@ -0,0 +1,97 @@
+module PromptEngine
+  class VariableDetector
+    # Regex pattern to match {{variable_name}} syntax
+    # Allows letters, numbers, underscores, and dots for nested variables
+    VARIABLE_PATTERN = /\{\{([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*)\}\}/
+
+    attr_reader :content
+
+    def initialize(content)
+      @content = content.to_s
+    end
+
+    # Extract all variables from the content
+    def extract_variables
+      variables = []
+
+      content.scan(VARIABLE_PATTERN) do |match|
+        variable_name = match.first
+        variables << {
+          name: variable_name,
+          placeholder: "{{#{variable_name}}}",
+          position: Regexp.last_match.offset(0),
+          type: infer_type(variable_name),
+          required: true
+        }
+      end
+
+      # Remove duplicates while preserving order
+      variables.uniq { |v| v[:name] }
+    end
+
+    # Get just the variable names as an array
+    def variable_names
+      extract_variables.map { |v| v[:name] }
+    end
+
+    # Check if content contains variables
+    def has_variables?
+      content.match?(VARIABLE_PATTERN)
+    end
+
+    # Count unique variables
+    def variable_count
+      variable_names.count
+    end
+
+    # Replace variables with provided values
+    def render(variables = {})
+      rendered = content.dup
+
+      variables.each do |key, value|
+        # Support both string and symbol keys
+        placeholder = "{{#{key}}}"
+        rendered.gsub!(placeholder, value.to_s)
+      end
+
+      rendered
+    end
+
+    # Validate that all required variables are provided
+    def validate_variables(provided_variables = {})
+      missing = []
+      stringified_keys = provided_variables.stringify_keys
+
+      variable_names.each do |var_name|
+        unless stringified_keys.key?(var_name)
+          missing << var_name
+        end
+      end
+
+      {
+        valid: missing.empty?,
+        missing_variables: missing
+      }
+    end
+
+    private
+
+    # Attempt to infer type from variable name
+    def infer_type(variable_name)
+      case variable_name.downcase
+      when /(_id|_count|_number|_qty|_quantity)$/
+        "integer"
+      when /(_at|_date|_time)$/
+        "datetime"
+      when /(_price|_amount|_cost|_total)$/
+        "decimal"
+      when /(is_|has_|can_|should_)/
+        "boolean"
+      when /(_list|_array|_items)$/
+        "array"
+      else
+        "string"
+      end
+    end
+  end
+end

data/app/views/layouts/prompt_engine/admin.html.erb

@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>PromptEngine Admin</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= stylesheet_link_tag "prompt_engine/application", "data-turbo-track": "reload" %>
+  </head>
+
+  <body class="prompt-engine-admin">
+    <div class="admin-layout">
+      <aside class="admin-layout__sidebar">
+        <div class="admin-sidebar">
+          <div class="admin-sidebar__header">
+            <h1 class="admin-sidebar__title">PromptEngine</h1>
+            <span class="admin-sidebar__version">v<%= PromptEngine::VERSION %></span>
+          </div>
+
+          <nav class="admin-sidebar__nav">
+            <ul class="admin-nav">
+              <li class="admin-nav__item">
+                <%= link_to "Dashboard", root_path, 
+                    class: "admin-nav__link #{'admin-nav__link--active' if controller_name == 'dashboard'}" %>
+              </li>
+              <li class="admin-nav__item">
+                <%= link_to "Prompts", prompts_path, 
+                    class: "admin-nav__link #{'admin-nav__link--active' if %w[prompts versions playground].include?(controller_name)}" %>
+              </li>
+              <li class="admin-nav__item">
+                <%= link_to "Evaluations", evaluations_path, 
+                    class: "admin-nav__link #{'admin-nav__link--active' if %w[eval_sets eval_runs test_cases].include?(controller_name)}" %>
+              </li>
+              <li class="admin-nav__item">
+                <%= link_to "Test Runs", playground_run_results_path, 
+                    class: "admin-nav__link #{'admin-nav__link--active' if controller_name == 'playground_run_results'}" %>
+              </li>
+              <li class="admin-nav__item">
+                <%= link_to "Settings", edit_settings_path, 
+                    class: "admin-nav__link #{'admin-nav__link--active' if controller_name == 'settings'}" %>
+              </li>
+            </ul>
+          </nav>
+        </div>
+      </aside>
+
+      <main class="admin-layout__main">
+        <div class="admin-main">
+          <% if flash.any? %>
+            <div class="admin-notifications">
+              <% flash.each do |type, message| %>
+                <div class="admin-notification admin-notification--<%= type %>">
+                  <%= message %>
+                </div>
+              <% end %>
+            </div>
+          <% end %>
+
+          <%= yield %>
+        </div>
+      </main>
+    </div>
+  </body>
+</html>