prompt_engine 1.0.0

data/lib/prompt_engine/engine.rb

@@ -0,0 +1,25 @@
+module PromptEngine
+  class Engine < ::Rails::Engine
+    isolate_namespace PromptEngine
+
+    config.generators do |g|
+      g.test_framework :rspec
+      g.fixture_replacement :factory_bot
+      g.factory_bot dir: "spec/factories"
+    end
+
+    # Ensure services and clients directories are in the autoload paths
+    config.autoload_paths += %W[#{config.root}/app/services]
+    config.autoload_paths += %W[#{config.root}/app/clients]
+
+    # Ensure engine's migrations are available to the host app
+    # This is the standard Rails engine pattern
+    initializer :append_migrations do |app|
+      unless app.root.to_s.match?(root.to_s)
+        config.paths["db/migrate"].expanded.each do |expanded_path|
+          app.config.paths["db/migrate"] << expanded_path
+        end
+      end
+    end
+  end
+end

data/lib/prompt_engine/version.rb

@@ -0,0 +1,3 @@
+module PromptEngine
+  VERSION = "1.0.0"
+end

data/lib/prompt_engine.rb

@@ -0,0 +1,33 @@
+require "prompt_engine/version"
+require "prompt_engine/engine"
+
+module PromptEngine
+  class << self
+    # Renders a prompt by name with the given variables
+    # @param prompt_name [String, Symbol] The name of the prompt to render
+    # @param variables [Hash] Variables to interpolate into the prompt
+    # @return [Hash] The rendered prompt with content and system_message
+    def render(prompt_name, variables: {})
+      prompt = Prompt.active.find_by!(name: prompt_name.to_s)
+
+      {
+        content: interpolate_variables(prompt.content, variables),
+        system_message: interpolate_variables(prompt.system_message, variables),
+        model: prompt.model,
+        temperature: prompt.temperature,
+        max_tokens: prompt.max_tokens
+      }
+    end
+
+    private
+
+    def interpolate_variables(text, variables)
+      return text if text.blank? || variables.empty?
+
+      text.gsub(/\{\{(\w+)\}\}/) do |match|
+        key = $1.to_sym
+        variables.fetch(key) { match }
+      end
+    end
+  end
+end

data/lib/tasks/active_prompt_tasks.rake

@@ -0,0 +1,32 @@
+# desc "Explaining what the task does"
+# task :prompt_engine do
+#   # Task goes here
+# end
+
+namespace :prompt_engine do
+  desc "Setup dummy app for development and testing"
+  task setup: :environment do
+    puts "Setting up PromptEngine dummy app..."
+
+    dummy_root = File.expand_path("../../spec/dummy", __dir__)
+
+    Dir.chdir(dummy_root) do
+      # Remove existing migrations and schema
+      FileUtils.rm_rf(Dir.glob("db/migrate/*"))
+      FileUtils.rm_f("db/schema.rb")
+      FileUtils.rm_f("db/development.sqlite3")
+      FileUtils.rm_f("db/test.sqlite3")
+
+      # Install engine migrations
+      system("bundle exec rails prompt_engine:install:migrations")
+
+      # Create and migrate databases
+      system("bundle exec rails db:create")
+      system("bundle exec rails db:migrate")
+      system("RAILS_ENV=test bundle exec rails db:create")
+      system("RAILS_ENV=test bundle exec rails db:migrate")
+
+      puts "✅ PromptEngine dummy app setup complete!"
+    end
+  end
+end

data/lib/tasks/eval_demo.rake

@@ -0,0 +1,149 @@
+namespace :prompt_engine do
+  desc "Demo the evaluation workflow"
+  task eval_demo: :environment do
+    puts "\n=== ActivePrompt Evaluation Demo ==="
+    puts "This demo will walk through the complete evaluation workflow\n\n"
+
+    # Step 1: Create or find a prompt
+    prompt = ActivePrompt::Prompt.find_or_create_by(name: "demo_summarizer") do |p|
+      p.description = "Demo prompt for testing evaluations"
+      p.content = "Summarize this in one word: {{text}}"
+      p.system_message = "You are a concise summarizer. Always respond with exactly one word."
+      p.model = "gpt-3.5-turbo"
+      p.temperature = 0.0
+      p.max_tokens = 10
+      p.status = "active"
+    end
+    puts "✓ Created prompt: #{prompt.name}"
+
+    # Step 2: Create an evaluation set
+    eval_set = prompt.eval_sets.find_or_create_by(name: "Demo Eval Set") do |es|
+      es.description = "Testing one-word summaries"
+    end
+    puts "✓ Created eval set: #{eval_set.name}"
+
+    # Step 3: Create test cases
+    test_cases_data = [
+      {
+        input: { "text" => "The cat is sleeping peacefully on the warm windowsill" },
+        expected: "Cat",
+        description: "Animal summary"
+      },
+      {
+        input: { "text" => "It's raining heavily with thunder and lightning" },
+        expected: "Storm",
+        description: "Weather summary"
+      },
+      {
+        input: { "text" => "The delicious pizza has cheese, pepperoni, and mushrooms" },
+        expected: "Pizza",
+        description: "Food summary"
+      }
+    ]
+
+    test_cases_data.each do |tc_data|
+      test_case = eval_set.test_cases.find_or_create_by(
+        description: tc_data[:description]
+      ) do |tc|
+        tc.input_variables = tc_data[:input]
+        tc.expected_output = tc_data[:expected]
+      end
+      puts "✓ Created test case: #{test_case.description}"
+    end
+
+    # Step 4: Run evaluation
+    puts "\n--- Running Evaluation ---"
+
+    begin
+      # Check if API key is configured
+      if Rails.application.credentials.dig(:openai, :api_key).blank?
+        puts "⚠️  OpenAI API key not configured"
+        puts "   To configure: rails credentials:edit"
+        puts "   Add: openai:\\n  api_key: your-api-key"
+        puts "\n--- Using Mock Evaluation ---"
+
+        # Create a mock eval run
+        eval_run = eval_set.eval_runs.create!(
+          prompt_version: prompt.current_version
+        )
+
+        # Simulate evaluation
+        eval_run.update!(
+          status: "completed",
+          started_at: 2.seconds.ago,
+          completed_at: Time.current,
+          total_count: 3,
+          passed_count: 2,
+          failed_count: 1,
+          error_message: nil
+        )
+
+        puts "✓ Mock evaluation completed"
+      else
+        puts "✓ OpenAI API key found"
+
+        # Create real eval run
+        eval_run = eval_set.eval_runs.create!(
+          prompt_version: prompt.current_version
+        )
+
+        # Note: The actual OpenAI Evals API may not be available
+        # This will fall back to mock if not available
+        begin
+          ActivePrompt::EvaluationRunner.new(eval_run).execute
+          puts "✓ Evaluation submitted to OpenAI"
+        rescue ActivePrompt::OpenAIEvalsClient::NotFoundError => e
+          puts "⚠️  OpenAI Evals API not available on this account"
+          puts "   Using mock evaluation instead"
+
+          eval_run.update!(
+            status: "completed",
+            started_at: 2.seconds.ago,
+            completed_at: Time.current,
+            total_count: 3,
+            passed_count: 2,
+            failed_count: 1
+          )
+        end
+      end
+
+      # Step 5: Display results
+      puts "\n--- Evaluation Results ---"
+      eval_run.reload
+      puts "Status: #{eval_run.status}"
+      puts "Total tests: #{eval_run.total_count}"
+      puts "Passed: #{eval_run.passed_count}"
+      puts "Failed: #{eval_run.failed_count}"
+
+      if eval_run.total_count > 0
+        success_rate = (eval_run.passed_count.to_f / eval_run.total_count * 100).round(1)
+        puts "Success rate: #{success_rate}%"
+      end
+
+      if eval_run.error_message.present?
+        puts "Error: #{eval_run.error_message}"
+      end
+
+      # URLs to access the UI
+      puts "\n--- View in Browser ---"
+      puts "Eval Set: http://localhost:3000/prompt_engine/prompts/#{prompt.id}/eval_sets/#{eval_set.id}"
+      puts "Eval Run: http://localhost:3000/prompt_engine/prompts/#{prompt.id}/eval_runs/#{eval_run.id}"
+    rescue => e
+      puts "❌ Error: #{e.message}"
+      puts e.backtrace.first(5).join("\n")
+    end
+
+    puts "\n=== Demo Complete ==="
+  end
+
+  desc "Clean up eval demo data"
+  task clean_eval_demo: :environment do
+    prompt = ActivePrompt::Prompt.find_by(name: "demo_summarizer")
+    if prompt
+      prompt.destroy
+      puts "✓ Cleaned up demo data"
+    else
+      puts "No demo data to clean up"
+    end
+  end
+end

metadata

@@ -0,0 +1,293 @@
+--- !ruby/object:Gem::Specification
+name: prompt_engine
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Avi Flombaum
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby_llm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bcrypt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: factory_bot_rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.4'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.3'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.40'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.40'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.20'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.20'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: PromptEngine is a Rails mountable engine that provides a simple interface
+  for managing AI prompts, templates, and responses within Rails applications.
+email:
+- 4515+aviflombaum@users.noreply.github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/stylesheets/prompt_engine/application.css
+- app/assets/stylesheets/prompt_engine/buttons.css
+- app/assets/stylesheets/prompt_engine/cards.css
+- app/assets/stylesheets/prompt_engine/comparison.css
+- app/assets/stylesheets/prompt_engine/components/_test_runs.css
+- app/assets/stylesheets/prompt_engine/dashboard.css
+- app/assets/stylesheets/prompt_engine/evaluations.css
+- app/assets/stylesheets/prompt_engine/forms.css
+- app/assets/stylesheets/prompt_engine/foundation.css
+- app/assets/stylesheets/prompt_engine/layout.css
+- app/assets/stylesheets/prompt_engine/loading.css
+- app/assets/stylesheets/prompt_engine/notifications.css
+- app/assets/stylesheets/prompt_engine/overrides.css
+- app/assets/stylesheets/prompt_engine/prompts.css
+- app/assets/stylesheets/prompt_engine/sidebar.css
+- app/assets/stylesheets/prompt_engine/tables.css
+- app/assets/stylesheets/prompt_engine/utilities.css
+- app/assets/stylesheets/prompt_engine/versions.css
+- app/clients/prompt_engine/open_ai_evals_client.rb
+- app/controllers/prompt_engine/admin/base_controller.rb
+- app/controllers/prompt_engine/application_controller.rb
+- app/controllers/prompt_engine/dashboard_controller.rb
+- app/controllers/prompt_engine/eval_runs_controller.rb
+- app/controllers/prompt_engine/eval_sets_controller.rb
+- app/controllers/prompt_engine/evaluations_controller.rb
+- app/controllers/prompt_engine/playground_controller.rb
+- app/controllers/prompt_engine/playground_run_results_controller.rb
+- app/controllers/prompt_engine/prompts_controller.rb
+- app/controllers/prompt_engine/settings_controller.rb
+- app/controllers/prompt_engine/test_cases_controller.rb
+- app/controllers/prompt_engine/versions_controller.rb
+- app/helpers/prompt_engine/application_helper.rb
+- app/jobs/prompt_engine/application_job.rb
+- app/mailers/prompt_engine/application_mailer.rb
+- app/models/prompt_engine/application_record.rb
+- app/models/prompt_engine/eval_result.rb
+- app/models/prompt_engine/eval_run.rb
+- app/models/prompt_engine/eval_set.rb
+- app/models/prompt_engine/parameter.rb
+- app/models/prompt_engine/parameter_parser.rb
+- app/models/prompt_engine/playground_run_result.rb
+- app/models/prompt_engine/prompt.rb
+- app/models/prompt_engine/prompt_version.rb
+- app/models/prompt_engine/setting.rb
+- app/models/prompt_engine/test_case.rb
+- app/services/prompt_engine/evaluation_runner.rb
+- app/services/prompt_engine/playground_executor.rb
+- app/services/prompt_engine/variable_detector.rb
+- app/views/layouts/prompt_engine/admin.html.erb
+- app/views/layouts/prompt_engine/application.html.erb
+- app/views/prompt_engine/dashboard/index.html.erb
+- app/views/prompt_engine/eval_runs/show.html.erb
+- app/views/prompt_engine/eval_sets/compare.html.erb
+- app/views/prompt_engine/eval_sets/edit.html.erb
+- app/views/prompt_engine/eval_sets/index.html.erb
+- app/views/prompt_engine/eval_sets/metrics.html.erb
+- app/views/prompt_engine/eval_sets/new.html.erb
+- app/views/prompt_engine/eval_sets/show.html.erb
+- app/views/prompt_engine/evaluations/index.html.erb
+- app/views/prompt_engine/playground/result.html.erb
+- app/views/prompt_engine/playground/show.html.erb
+- app/views/prompt_engine/playground_run_results/index.html.erb
+- app/views/prompt_engine/playground_run_results/show.html.erb
+- app/views/prompt_engine/prompts/_form.html.erb
+- app/views/prompt_engine/prompts/edit.html.erb
+- app/views/prompt_engine/prompts/index.html.erb
+- app/views/prompt_engine/prompts/new.html.erb
+- app/views/prompt_engine/prompts/show.html.erb
+- app/views/prompt_engine/settings/edit.html.erb
+- app/views/prompt_engine/shared/_form_errors.html.erb
+- app/views/prompt_engine/test_cases/edit.html.erb
+- app/views/prompt_engine/test_cases/import.html.erb
+- app/views/prompt_engine/test_cases/import_preview.html.erb
+- app/views/prompt_engine/test_cases/new.html.erb
+- app/views/prompt_engine/versions/_version_card.html.erb
+- app/views/prompt_engine/versions/compare.html.erb
+- app/views/prompt_engine/versions/index.html.erb
+- app/views/prompt_engine/versions/show.html.erb
+- config/routes.rb
+- db/migrate/20250124000001_create_eval_tables.rb
+- db/migrate/20250124000002_add_open_ai_fields_to_evals.rb
+- db/migrate/20250125000001_add_grader_fields_to_eval_sets.rb
+- db/migrate/20250723161909_create_prompts.rb
+- db/migrate/20250723184757_create_prompt_engine_versions.rb
+- db/migrate/20250723203838_create_prompt_engine_parameters.rb
+- db/migrate/20250724160623_create_prompt_engine_playground_run_results.rb
+- db/migrate/20250724165118_create_prompt_engine_settings.rb
+- lib/prompt_engine.rb
+- lib/prompt_engine/engine.rb
+- lib/prompt_engine/version.rb
+- lib/tasks/active_prompt_tasks.rake
+- lib/tasks/eval_demo.rake
+homepage: https://github.com/aviflombaum/prompt_engine
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/aviflombaum/prompt_engine
+  source_code_uri: https://github.com/aviflombaum/prompt_engine
+  changelog_uri: https://github.com/aviflombaum/prompt_engine/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Rails mountable engine for AI prompt management
+test_files: []