prompt_engine 1.0.0

data/app/controllers/prompt_engine/eval_sets_controller.rb

@@ -0,0 +1,200 @@
+module PromptEngine
+  class EvalSetsController < ApplicationController
+    layout "prompt_engine/admin"
+
+    before_action :set_prompt
+    before_action :set_eval_set, only: [ :show, :edit, :update, :destroy, :run, :compare, :metrics ]
+
+    def index
+      @eval_sets = @prompt.eval_sets
+    end
+
+    def show
+      @test_cases = @eval_set.test_cases
+      @recent_runs = @eval_set.eval_runs.order(created_at: :desc).limit(5)
+    end
+
+    def new
+      @eval_set = @prompt.eval_sets.build
+    end
+
+    def create
+      @eval_set = @prompt.eval_sets.build(eval_set_params)
+
+      if @eval_set.save
+        redirect_to prompt_eval_set_path(@prompt, @eval_set), notice: "Evaluation set was successfully created."
+      else
+        flash.now[:alert] = "Please fix the errors below."
+        render :new
+      end
+    end
+
+    def edit
+    end
+
+    def update
+      if @eval_set.update(eval_set_params)
+        redirect_to prompt_eval_set_path(@prompt, @eval_set), notice: "Evaluation set was successfully updated."
+      else
+        flash.now[:alert] = "Please fix the errors below."
+        render :edit
+      end
+    end
+
+    def destroy
+      @eval_set.destroy
+      redirect_to prompt_eval_sets_path(@prompt), notice: "Evaluation set was successfully deleted."
+    end
+
+    def run
+      # Check if API key is available
+      unless api_key_configured?
+        redirect_to prompt_eval_set_path(@prompt, @eval_set),
+          alert: "OpenAI API key not configured. Please configure it in Settings or contact your administrator."
+        return
+      end
+
+      # Create new eval run with current prompt version
+      @eval_run = @eval_set.eval_runs.create!(
+        prompt_version: @prompt.current_version
+      )
+
+      begin
+        # Run evaluation synchronously for MVP
+        PromptEngine::EvaluationRunner.new(@eval_run).execute
+        redirect_to prompt_eval_run_path(@prompt, @eval_run), notice: "Evaluation started successfully"
+      rescue PromptEngine::OpenAiEvalsClient::AuthenticationError => e
+        @eval_run.update!(status: :failed, error_message: e.message)
+        redirect_to prompt_eval_set_path(@prompt, @eval_set),
+          alert: "Authentication failed: Please check your OpenAI API key in Settings"
+      rescue PromptEngine::OpenAiEvalsClient::RateLimitError => e
+        @eval_run.update!(status: :failed, error_message: e.message)
+        redirect_to prompt_eval_set_path(@prompt, @eval_set), alert: "Rate limit exceeded: Please try again later"
+      rescue PromptEngine::OpenAiEvalsClient::APIError => e
+        @eval_run.update!(status: :failed, error_message: e.message)
+        redirect_to prompt_eval_set_path(@prompt, @eval_set), alert: "API error: #{e.message}"
+      rescue => e
+        @eval_run.update!(status: :failed, error_message: e.message)
+        Rails.logger.error "Evaluation error: #{e.class} - #{e.message}"
+        Rails.logger.error e.backtrace.join("\n")
+        redirect_to prompt_eval_set_path(@prompt, @eval_set), alert: "Evaluation failed: #{e.message}"
+      end
+    end
+
+    def compare
+      unless params[:run_ids].present? && params[:run_ids].is_a?(Array) && params[:run_ids].length == 2
+        redirect_to prompt_eval_set_path(@prompt, @eval_set),
+          alert: "Please select exactly two evaluation runs to compare."
+        return
+      end
+
+      @run1 = @eval_set.eval_runs.find(params[:run_ids][0])
+      @run2 = @eval_set.eval_runs.find(params[:run_ids][1])
+
+      # Ensure both runs are completed
+      unless @run1.status == "completed" && @run2.status == "completed"
+        redirect_to prompt_eval_set_path(@prompt, @eval_set),
+          alert: "Both evaluation runs must be completed to compare them."
+        return
+      end
+
+      # Calculate comparison metrics
+      @run1_success_rate = (@run1.total_count > 0) ? (@run1.passed_count.to_f / @run1.total_count * 100) : 0
+      @run2_success_rate = (@run2.total_count > 0) ? (@run2.passed_count.to_f / @run2.total_count * 100) : 0
+      @success_rate_diff = @run2_success_rate - @run1_success_rate
+    rescue ActiveRecord::RecordNotFound
+      redirect_to prompt_eval_set_path(@prompt, @eval_set),
+        alert: "One or both evaluation runs could not be found."
+    end
+
+    def metrics
+      # Get all completed runs for this eval set
+      @eval_runs = @eval_set.eval_runs.where(status: "completed").order(created_at: :asc)
+
+      # Calculate metrics data for charts
+      if @eval_runs.any?
+        # Success rate trend data (for line chart)
+        @success_rate_trend = @eval_runs.map do |run|
+          {
+            date: run.created_at.strftime("%b %d, %Y %I:%M %p"),
+            rate: (run.total_count > 0) ? (run.passed_count.to_f / run.total_count * 100).round(2) : 0,
+            version: "v#{run.prompt_version.version_number}"
+          }
+        end
+
+        # Success rate by version (for bar chart)
+        version_stats = @eval_runs.group_by { |r| r.prompt_version.version_number }
+        @success_rate_by_version = version_stats.map do |version, runs|
+          total_passed = runs.sum(&:passed_count)
+          total_count = runs.sum(&:total_count)
+          {
+            version: "v#{version}",
+            rate: (total_count > 0) ? (total_passed.to_f / total_count * 100).round(2) : 0,
+            runs: runs.count
+          }
+        end.sort_by { |v| v[:version] }
+
+        # Test case statistics
+        @total_test_cases = @eval_set.test_cases.count
+        @total_runs = @eval_runs.count
+        @overall_pass_rate = begin
+          total_passed = @eval_runs.sum(&:passed_count)
+          total_tests = @eval_runs.sum(&:total_count)
+          (total_tests > 0) ? (total_passed.to_f / total_tests * 100).round(2) : 0
+        end
+
+        # Average duration trend
+        @duration_trend = @eval_runs.map do |run|
+          duration = if run.completed_at && run.started_at
+            (run.completed_at - run.started_at).to_i
+          else
+            nil
+          end
+          {
+            date: run.created_at.strftime("%b %d, %Y %I:%M %p"),
+            duration: duration,
+            version: "v#{run.prompt_version.version_number}"
+          }
+        end.compact
+
+        # Recent activity (last 10 runs)
+        @recent_activity = @eval_runs.last(10).reverse
+      else
+        @success_rate_trend = []
+        @success_rate_by_version = []
+        @total_test_cases = @eval_set.test_cases.count
+        @total_runs = 0
+        @overall_pass_rate = 0
+        @duration_trend = []
+        @recent_activity = []
+      end
+    end
+
+    protected
+
+    helper_method :api_key_configured?
+
+    private
+
+    def set_prompt
+      @prompt = Prompt.find(params[:prompt_id])
+    end
+
+    def set_eval_set
+      @eval_set = @prompt.eval_sets.find(params[:id])
+    end
+
+    def eval_set_params
+      params.require(:eval_set).permit(:name, :description, :grader_type, grader_config: {})
+    end
+
+    def api_key_configured?
+      # Check if OpenAI API key is available from Settings or Rails credentials
+      settings = PromptEngine::Setting.instance
+      settings.openai_configured? || Rails.application.credentials.dig(:openai, :api_key).present?
+    rescue ActiveRecord::RecordNotFound
+      # If settings record doesn't exist, check Rails credentials
+      Rails.application.credentials.dig(:openai, :api_key).present?
+    end
+  end
+end

data/app/controllers/prompt_engine/evaluations_controller.rb

@@ -0,0 +1,32 @@
+module PromptEngine
+  class EvaluationsController < ApplicationController
+    layout "prompt_engine/admin"
+
+    def index
+      @prompts_with_eval_sets = Prompt.joins(:eval_sets)
+        .includes(eval_sets: [ :eval_runs ])
+        .distinct
+        .order(:name)
+
+      # Calculate overall statistics
+      @total_eval_sets = EvalSet.count
+      @total_eval_runs = EvalRun.count
+      @total_test_cases = TestCase.count
+
+      # Get recent evaluation activity
+      @recent_runs = EvalRun.includes(eval_set: :prompt, prompt_version: :prompt)
+        .order(created_at: :desc)
+        .limit(10)
+
+      # Calculate overall pass rate
+      completed_runs = EvalRun.where(status: "completed")
+      if completed_runs.any?
+        total_passed = completed_runs.sum(:passed_count)
+        total_tests = completed_runs.sum(:total_count)
+        @overall_pass_rate = (total_tests > 0) ? (total_passed.to_f / total_tests * 100).round(2) : 0
+      else
+        @overall_pass_rate = 0
+      end
+    end
+  end
+end

data/app/controllers/prompt_engine/playground_controller.rb

@@ -0,0 +1,57 @@
+module PromptEngine
+  class PlaygroundController < ApplicationController
+    layout "prompt_engine/admin"
+    before_action :set_prompt
+
+    def show
+      @parameters = ParameterParser.new(@prompt.content).extract_parameters.map { |p| p[:name] }
+      @settings = Setting.instance
+    end
+
+    def execute
+      executor = PlaygroundExecutor.new(
+        prompt: @prompt,
+        provider: params[:provider],
+        api_key: params[:api_key],
+        parameters: params[:parameters]
+      )
+
+      begin
+        result = executor.execute
+        @response = result[:response]
+        @execution_time = result[:execution_time]
+        @token_count = result[:token_count]
+        @model = result[:model]
+        @provider = result[:provider]
+
+        # Store the rendered prompt for display
+        parser = ParameterParser.new(@prompt.content)
+        @rendered_prompt = parser.replace_parameters(params[:parameters])
+
+        # Save the playground run result
+        @prompt.current_version.playground_run_results.create!(
+          provider: @provider,
+          model: @model,
+          rendered_prompt: @rendered_prompt,
+          system_message: @prompt.system_message,
+          parameters: params[:parameters],
+          response: @response,
+          execution_time: @execution_time,
+          token_count: @token_count,
+          temperature: @prompt.temperature,
+          max_tokens: @prompt.max_tokens
+        )
+      rescue => e
+        @error = e.message
+      end
+
+      render :result
+    end
+
+    private
+
+    def set_prompt
+      @prompt = Prompt.find(params[:id])
+    end
+  end
+end

data/app/controllers/prompt_engine/playground_run_results_controller.rb

@@ -0,0 +1,41 @@
+module PromptEngine
+  class PlaygroundRunResultsController < ApplicationController
+    layout "prompt_engine/admin"
+    before_action :set_playground_run_result, only: [ :show ]
+    before_action :set_context, only: [ :index ]
+
+    def index
+      @playground_run_results = scope.recent.includes(prompt_version: :prompt)
+    end
+
+    def show
+    end
+
+    private
+
+    def set_playground_run_result
+      @playground_run_result = PlaygroundRunResult.find(params[:id])
+    end
+
+    def set_context
+      if params[:prompt_id]
+        @prompt = Prompt.find(params[:prompt_id])
+      elsif params[:version_id]
+        @prompt_version = PromptVersion.find(params[:version_id])
+        @prompt = @prompt_version.prompt
+      end
+    end
+
+    def scope
+      if params[:prompt_id]
+        # Get all playground run results for all versions of this prompt
+        PlaygroundRunResult.joins(:prompt_version).where(prompt_engine_prompt_versions: { prompt_id: params[:prompt_id] })
+      elsif params[:version_id]
+        # Get playground run results for a specific version
+        PromptVersion.find(params[:version_id]).playground_run_results
+      else
+        PlaygroundRunResult.all
+      end
+    end
+  end
+end

data/app/controllers/prompt_engine/prompts_controller.rb

@@ -0,0 +1,70 @@
+module PromptEngine
+  class PromptsController < ApplicationController
+    layout "prompt_engine/admin"
+      before_action :set_prompt, only: [ :show, :edit, :update, :destroy ]
+
+      def index
+        @prompts = PromptEngine::Prompt.by_name
+      end
+
+      def show
+        # Get recent test runs for this prompt across all versions
+        @recent_test_runs = PromptEngine::PlaygroundRunResult
+          .joins(:prompt_version)
+          .where(prompt_engine_prompt_versions: { prompt_id: @prompt.id })
+          .recent
+          .limit(5)
+          .includes(:prompt_version)
+
+        # Get evaluation data for this prompt
+        @eval_sets = @prompt.eval_sets.includes(:test_cases, :eval_runs)
+        @recent_eval_runs = PromptEngine::EvalRun
+          .joins(:eval_set)
+          .where(prompt_engine_eval_sets: { prompt_id: @prompt.id })
+          .order(created_at: :desc)
+          .limit(5)
+          .includes(:eval_set, :prompt_version)
+      end
+
+      def new
+        @prompt = PromptEngine::Prompt.new
+      end
+
+      def create
+        @prompt = PromptEngine::Prompt.new(prompt_params)
+
+        if @prompt.save
+          redirect_to prompt_path(@prompt), notice: "Prompt was successfully created."
+        else
+          render :new, status: :unprocessable_entity
+        end
+      end
+
+      def edit
+      end
+
+      def update
+        if @prompt.update(prompt_params)
+          redirect_to prompt_path(@prompt), notice: "Prompt was successfully updated."
+        else
+          render :edit, status: :unprocessable_entity
+        end
+      end
+
+      def destroy
+        @prompt.destroy
+        redirect_to prompts_path, notice: "Prompt was successfully deleted."
+      end
+
+      private
+
+      def set_prompt
+        @prompt = PromptEngine::Prompt.find(params[:id])
+      end
+
+      def prompt_params
+        params.require(:prompt).permit(:name, :description, :content, :system_message, :model, :temperature, :max_tokens, :status,
+          parameters_attributes: [ :id, :name, :description, :parameter_type, :required, :default_value, :_destroy ])
+      end
+  end
+end

data/app/controllers/prompt_engine/settings_controller.rb

@@ -0,0 +1,28 @@
+module PromptEngine
+  class SettingsController < ApplicationController
+    layout "prompt_engine/admin"
+    before_action :load_settings
+
+    def edit
+      # Just display the form
+    end
+
+    def update
+      if @settings.update(settings_params)
+        redirect_to edit_settings_path, notice: "Settings have been updated successfully."
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    private
+
+    def load_settings
+      @settings = Setting.instance
+    end
+
+    def settings_params
+      params.require(:setting).permit(:openai_api_key, :anthropic_api_key)
+    end
+  end
+end

data/app/controllers/prompt_engine/test_cases_controller.rb

@@ -0,0 +1,231 @@
+module PromptEngine
+  class TestCasesController < ApplicationController
+    layout "prompt_engine/admin"
+
+    before_action :set_prompt
+    before_action :set_eval_set
+    before_action :set_test_case, only: [ :edit, :update, :destroy ]
+
+    def new
+      @test_case = @eval_set.test_cases.build
+      # Pre-populate with prompt's parameters
+      @test_case.input_variables = @prompt.parameters.each_with_object({}) do |param, hash|
+        hash[param.name] = param.default_value
+      end
+    end
+
+    def create
+      @test_case = @eval_set.test_cases.build(test_case_params)
+
+      if @test_case.save
+        redirect_to prompt_eval_set_path(@prompt, @eval_set), notice: "Test case was successfully created."
+      else
+        flash.now[:alert] = "Please fix the errors below."
+        render :new
+      end
+    end
+
+    def edit
+    end
+
+    def update
+      if @test_case.update(test_case_params)
+        redirect_to prompt_eval_set_path(@prompt, @eval_set), notice: "Test case was successfully updated."
+      else
+        flash.now[:alert] = "Please fix the errors below."
+        render :edit
+      end
+    end
+
+    def destroy
+      @test_case.destroy
+      redirect_to prompt_eval_set_path(@prompt, @eval_set), notice: "Test case was successfully deleted."
+    end
+
+    def import
+      # Display the import form
+    end
+
+    def import_preview
+      unless params[:file].present?
+        redirect_to import_prompt_eval_set_test_cases_path(@prompt, @eval_set),
+          alert: "Please select a file to import."
+        return
+      end
+
+      @imported_data = []
+      @errors = []
+
+      begin
+        file_content = params[:file].read
+        file_type = detect_file_type(params[:file])
+
+        if file_type == :csv
+          parse_csv(file_content)
+        elsif file_type == :json
+          parse_json(file_content)
+        else
+          @errors << "Unsupported file format. Please upload a CSV or JSON file."
+        end
+      rescue => e
+        @errors << "Error reading file: #{e.message}"
+      end
+
+      if @errors.any?
+        flash.now[:alert] = @errors.join(", ")
+        render :import
+      else
+        # Store the imported data in session for the create action
+        session[:imported_test_cases] = @imported_data
+        render :import_preview
+      end
+    end
+
+    def import_create
+      imported_data = session[:imported_test_cases]
+
+      unless imported_data.present?
+        redirect_to import_prompt_eval_set_test_cases_path(@prompt, @eval_set),
+          alert: "No import data found. Please upload a file again."
+        return
+      end
+
+      success_count = 0
+      errors = []
+
+      imported_data.each_with_index do |data, index|
+        test_case = @eval_set.test_cases.build(
+          input_variables: data[:input_variables],
+          expected_output: data[:expected_output],
+          description: data[:description]
+        )
+
+        if test_case.save
+          success_count += 1
+        else
+          errors << "Row #{index + 1}: #{test_case.errors.full_messages.join(", ")}"
+        end
+      end
+
+      # Clear the session data
+      session.delete(:imported_test_cases)
+
+      if errors.any?
+        flash[:alert] = "Import completed with errors: #{errors.join("; ")}"
+      else
+        flash[:notice] = "Successfully imported #{success_count} test cases."
+      end
+
+      redirect_to prompt_eval_set_path(@prompt, @eval_set)
+    end
+
+    private
+
+    def set_prompt
+      @prompt = Prompt.find(params[:prompt_id])
+    end
+
+    def set_eval_set
+      @eval_set = @prompt.eval_sets.find(params[:eval_set_id])
+    end
+
+    def set_test_case
+      @test_case = @eval_set.test_cases.find(params[:id])
+    end
+
+    def test_case_params
+      params.require(:test_case).permit(:description, :expected_output, input_variables: {})
+    end
+
+    def detect_file_type(file)
+      filename = file.original_filename.downcase
+
+      if filename.ends_with?(".csv")
+        :csv
+      elsif filename.ends_with?(".json")
+        :json
+      else
+        :unknown
+      end
+    end
+
+    def parse_csv(content)
+      require "csv"
+
+      # Get prompt parameters for column validation
+      expected_params = @prompt.parameters.pluck(:name)
+
+      CSV.parse(content, headers: true) do |row|
+        # Extract input variables from prompt parameter columns
+        input_variables = {}
+
+        expected_params.each do |param_name|
+          if row.headers.include?(param_name)
+            input_variables[param_name] = row[param_name]
+          else
+            @errors << "Missing required column: #{param_name}"
+            return
+          end
+        end
+
+        # Check for expected_output column
+        unless row.headers.include?("expected_output")
+          @errors << "Missing required column: expected_output"
+          return
+        end
+
+        # Add to imported data
+        @imported_data << {
+          input_variables: input_variables,
+          expected_output: row["expected_output"],
+          description: row["description"] # Optional column
+        }
+      end
+    rescue CSV::MalformedCSVError => e
+      @errors << "Invalid CSV format: #{e.message}"
+    end
+
+    def parse_json(content)
+      data = JSON.parse(content)
+
+      unless data.is_a?(Array)
+        @errors << "JSON must be an array of objects"
+        return
+      end
+
+      expected_params = @prompt.parameters.pluck(:name)
+
+      data.each_with_index do |item, index|
+        unless item.is_a?(Hash)
+          @errors << "Item #{index + 1} must be an object"
+          next
+        end
+
+        unless item["input_variables"].is_a?(Hash)
+          @errors << "Item #{index + 1}: input_variables must be an object"
+          next
+        end
+
+        unless item["expected_output"].present?
+          @errors << "Item #{index + 1}: expected_output is required"
+          next
+        end
+
+        # Validate that all required parameters are present
+        missing_params = expected_params - item["input_variables"].keys
+        if missing_params.any?
+          @errors << "Item #{index + 1}: missing required parameters: #{missing_params.join(", ")}"
+          next
+        end
+
+        @imported_data << {
+          input_variables: item["input_variables"],
+          expected_output: item["expected_output"],
+          description: item["description"]
+        }
+      end
+    rescue JSON::ParserError => e
+      @errors << "Invalid JSON format: #{e.message}"
+    end
+  end
+end