minitest-promptfoo 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1687874b77d0e965b186cf7f488579e65c9b318f9c24c0a888f9871aa20d30ff
+  data.tar.gz: 37315fe836966e9ebae13d72012140e3abd8a06c09456e492d68655896312e8b
+SHA512:
+  metadata.gz: 0a53f83579b00a493e0bf824a8a6cf784c13aefc4a5ef401efc3a59e5632178f7213380855093e6a7de319c6cbde0e4113d58bdf38dae04e3a1b0ff9e63b9f5c
+  data.tar.gz: f269bc6f6127042ea83118a262b794ddb288995533178290ce2dcc00b197548767bbef1397f2760bb65abb38ad0a8771932085ad63668e9b37a1ecfb5764dbc2

data/.ruby-version

@@ -0,0 +1 @@
+3.4.3

data/CHANGELOG.md

@@ -0,0 +1,26 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- Initial release of minitest-promptfoo
+- Core `Minitest::Promptfoo::Test` class for prompt testing
+- Configuration system for promptfoo executable path
+- Support for multiple providers
+- Assertion DSL: `includes`, `matches`, `equals`, `json_includes`, `javascript`, `rubric`
+- Rails integration with automatic prompt file discovery
+- Support for both .ptmpl and .liquid prompt formats
+- Pre-rendering support for template conflicts
+- Debug mode with `DEBUG_PROMPT_TEST` environment variable
+- Verbose mode for detailed failure messages
+- Comprehensive README with examples
+- Basic test coverage
+
+## [0.1.0] - TBD
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Chris Waters
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,326 @@
+# Minitest::Promptfoo
+
+A thin Minitest wrapper around [promptfoo](https://www.promptfoo.dev/) that brings prompt testing to Ruby projects. Test your LLM prompts with a familiar Minitest-like DSL, supporting multiple providers and assertion types.
+
+## Why Test Your Prompts?
+
+LLM outputs are non-deterministic, but that doesn't mean you can't test them. With minitest-promptfoo, you can:
+
+- Ensure prompts produce expected types of responses
+- Validate JSON structure in responses
+- Use LLM-as-judge for qualitative evaluation
+- Test against multiple providers simultaneously
+- Catch prompt regressions before they hit production
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'minitest-promptfoo'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install minitest-promptfoo
+```
+
+### Promptfoo Setup
+
+You'll need promptfoo installed. You can either:
+
+1. Install it locally via npm:
+```bash
+npm install -D promptfoo
+```
+
+2. Or use npx (no installation required):
+```bash
+# The gem will automatically fall back to `npx promptfoo`
+```
+
+## Basic Usage
+
+### Plain Ruby Projects
+
+```ruby
+require 'minitest/autorun'
+require 'minitest/promptfoo'
+
+class GreetingPromptTest < Minitest::Promptfoo::Test
+  # Set provider(s) for all tests in this class
+  self.providers = "openai:gpt-4o-mini"
+
+  def prompt_path
+    "prompts/greeting.ptmpl"  # Or .liquid
+  end
+
+  def test_generates_professional_greeting
+    assert_prompt(vars: { name: "Alice" }) do |response|
+      response.includes("Hello Alice")
+      response.matches(/[A-Z]/)  # Starts with capital
+      response.rubric("Response is professional and courteous")
+    end
+  end
+
+  def test_validates_json_structure
+    assert_prompt(vars: { format: "json" }) do |response|
+      response.json_includes(key: "greeting", value: "Hello")
+      response.json_includes(key: "sentiment", value: "positive")
+    end
+  end
+end
+```
+
+### Rails Projects
+
+In Rails, the gem automatically discovers prompt files based on test file paths:
+
+```ruby
+# test/services/greeting_service_test.rb
+class GreetingServiceTest < Minitest::Promptfoo::RailsTest
+  self.providers = "openai:gpt-4o-mini"
+
+  # Automatically finds app/services/greeting_service.ptmpl
+  # No need to define prompt_path!
+
+  def test_greeting_is_friendly
+    assert_prompt(vars: { name: "Bob" }) do |response|
+      response.includes("Hello Bob")
+      response.rubric("Greeting is warm and welcoming", threshold: 0.7)
+    end
+  end
+end
+```
+
+## Configuration
+
+Configure the gem in your test helper or setup file:
+
+```ruby
+# test/test_helper.rb
+require 'minitest/promptfoo'
+
+Minitest::Promptfoo.configure do |config|
+  # Optional: specify custom promptfoo executable path
+  config.promptfoo_executable = "./node_modules/.bin/promptfoo"
+
+  # Optional: set root path for resolving prompt files
+  config.root_path = Rails.root # or Dir.pwd
+end
+```
+
+## Assertion Types
+
+### String Matching
+
+```ruby
+assert_prompt(vars: { topic: "weather" }) do |response|
+  # Contains substring
+  response.includes("sunny")
+
+  # Matches regex
+  response.matches(/\d+°[CF]/)
+
+  # Exact equality
+  response.equals("It's a beautiful day!")
+end
+```
+
+### JSON Validation
+
+```ruby
+assert_prompt(vars: { query: "status" }) do |response|
+  response.json_includes(key: "status", value: "success")
+  response.json_includes(key: "code", value: 200)
+end
+```
+
+### Custom JavaScript
+
+```ruby
+assert_prompt(vars: { count: 5 }) do |response|
+  response.javascript("parseInt(output) > 3")
+  response.javascript("output.split(' ').length <= 10")
+end
+```
+
+### LLM-as-Judge
+
+```ruby
+assert_prompt(vars: { tone: "professional" }) do |response|
+  response.rubric("Response is professional and courteous")
+  response.rubric("Uses business-appropriate language", threshold: 0.8)
+end
+```
+
+## Multiple Providers
+
+Test your prompt across multiple providers:
+
+```ruby
+class MultiProviderTest < Minitest::Promptfoo::Test
+  self.providers = [
+    "openai:gpt-4o-mini",
+    "openai:chat:anthropic:claude-3-7-sonnet",
+    "openai:chat:google:gemini-2.0-flash"
+  ]
+
+  def prompt_path
+    "prompts/greeting.ptmpl"
+  end
+
+  def test_works_across_providers
+    assert_prompt(vars: { name: "Alice" }) do |response|
+      response.includes("Alice")
+    end
+  end
+end
+```
+
+## Provider Configuration
+
+Pass custom configuration to providers:
+
+```ruby
+def test_json_response_format
+  json_provider = {
+    id: "openai:gpt-4o-mini",
+    config: {
+      response_format: { type: "json_object" },
+      temperature: 0.7
+    }
+  }
+
+  assert_prompt(vars: { input: "data" }, providers: json_provider) do |response|
+    response.json_includes(key: "result", value: "success")
+  end
+end
+```
+
+## Prompt File Formats
+
+### Promptfoo Templates (.ptmpl)
+
+Use double-brace syntax for variables:
+
+```
+You are a helpful assistant.
+
+Greet the user named {{name}} in a {{tone}} manner.
+```
+
+### Liquid Templates (.liquid)
+
+Standard Liquid syntax (converted internally):
+
+```
+You are a helpful assistant.
+
+Greet the user named {name} in a {tone} manner.
+```
+
+## Pre-rendering Templates
+
+If your prompt contains syntax that conflicts with promptfoo's templating (like analyzing Liquid code), pre-render it:
+
+```ruby
+def test_liquid_code_analysis
+  assert_prompt(
+    vars: { code: "{{user.name | upcase}}" },
+    pre_render: true
+  ) do |response|
+    response.includes("variable interpolation")
+  end
+end
+```
+
+## Debugging
+
+Enable debug output to see the generated promptfoo config:
+
+```bash
+DEBUG_PROMPT_TEST=1 bundle exec rake test
+```
+
+Or enable verbose mode for detailed failure messages:
+
+```ruby
+assert_prompt(vars: { name: "Alice" }, verbose: true) do |response|
+  response.rubric("Be friendly")
+end
+```
+
+## Real-World Example
+
+```ruby
+class CustomerSupportPromptTest < Minitest::Promptfoo::Test
+  self.providers = "openai:gpt-4o-mini"
+
+  def prompt_path
+    "prompts/customer_support.ptmpl"
+  end
+
+  def test_handles_refund_request_professionally
+    assert_prompt(vars: {
+      issue: "item arrived damaged",
+      customer_name: "Jane Doe"
+    }) do |response|
+      response.includes("Jane")
+      response.rubric("Acknowledges the issue empathetically")
+      response.rubric("Offers clear next steps")
+      response.rubric("Maintains professional tone")
+      response.matches(/refund|replacement/i)
+    end
+  end
+
+  def test_escalates_complex_issues
+    assert_prompt(vars: {
+      issue: "legal complaint about data breach",
+      customer_name: "John Smith"
+    }) do |response|
+      response.rubric("Recognizes this requires escalation")
+      response.rubric("Does not make promises outside of AI's authority")
+      response.includes("escalate")
+    end
+  end
+end
+```
+
+## Differences from ActiveSupport::TestCase
+
+When using `Minitest::Promptfoo::Test` (non-Rails), note these differences:
+
+- No fixtures or setup helpers from Rails
+- Must explicitly define `prompt_path`
+- No automatic database transaction rollbacks
+- Uses plain Minitest assertions
+
+For Rails projects, use `Minitest::Promptfoo::RailsTest` to get all Rails testing features plus automatic prompt discovery.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/christhesoul/minitest-promptfoo.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Credits
+
+Built with love on top of:
+- [promptfoo](https://www.promptfoo.dev/) - The excellent prompt testing framework
+- [minitest](https://github.com/minitest/minitest) - Ruby's favorite testing library

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "minitest/test_task"
+
+Minitest::TestTask.create
+
+require "standard/rake"
+
+task default: %i[test standard]

data/examples/greeting.ptmpl

@@ -0,0 +1,3 @@
+You are a helpful assistant.
+
+Greet the user named {{name}} in a {{tone}} manner.

data/examples/simple_prompt_test.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# Example usage of minitest-promptfoo
+#
+# To run this example:
+# 1. Create a prompt file at examples/greeting.ptmpl
+# 2. Run: ruby examples/simple_prompt_test.rb
+
+require "bundler/setup"
+require "minitest/autorun"
+require "minitest/promptfoo"
+
+class SimplePromptTest < Minitest::Promptfoo::Test
+  # Use the echo provider for testing (doesn't call actual LLMs)
+  self.providers = "echo"
+
+  def prompt_path
+    File.join(__dir__, "greeting.ptmpl")
+  end
+
+  def test_prompt_includes_name
+    assert_prompt(vars: {name: "Alice", tone: "friendly"}) do |response|
+      response.includes("Alice")
+      response.includes("friendly")
+    end
+  end
+
+  def test_prompt_with_different_tone
+    assert_prompt(vars: {name: "Bob", tone: "professional"}) do |response|
+      response.includes("Bob")
+      response.includes("professional")
+    end
+  end
+end

data/lib/minitest/promptfoo/assertion_builder.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Minitest
+  module Promptfoo
+    # DSL for building promptfoo assertions in a minitest-like style
+    #
+    # Example:
+    #   builder = AssertionBuilder.new
+    #   builder.includes("Hello")
+    #   builder.matches(/\d+/)
+    #   builder.rubric("Response is professional")
+    #   builder.to_promptfoo_assertions
+    class AssertionBuilder
+      def initialize
+        @assertions = []
+      end
+
+      # String inclusion check
+      def includes(text)
+        @assertions << {
+          "type" => "contains",
+          "value" => text
+        }
+      end
+
+      # Regex pattern matching
+      def matches(pattern)
+        @assertions << {
+          "type" => "regex",
+          "value" => pattern.source
+        }
+      end
+
+      # Exact equality check
+      def equals(expected)
+        @assertions << {
+          "type" => "equals",
+          "value" => expected
+        }
+      end
+
+      # JSON structure validation using JavaScript
+      def json_includes(key:, value:)
+        @assertions << {
+          "type" => "is-json"
+        }
+        # Handle both string output (needs parsing) and object output (already parsed)
+        @assertions << {
+          "type" => "javascript",
+          "value" => "(typeof output === 'string' ? JSON.parse(output) : output)[#{key.inspect}] === #{value.to_json}"
+        }
+      end
+
+      # Custom JavaScript assertion
+      def javascript(js_code)
+        @assertions << {
+          "type" => "javascript",
+          "value" => js_code
+        }
+      end
+
+      # LLM-as-judge rubric evaluation
+      def rubric(criteria, threshold: 0.5)
+        @assertions << {
+          "type" => "llm-rubric",
+          "value" => criteria,
+          "threshold" => threshold
+        }
+      end
+
+      # Convert to promptfoo assertion format
+      def to_promptfoo_assertions
+        @assertions
+      end
+    end
+  end
+end

data/lib/minitest/promptfoo/configuration.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Promptfoo
+    class Configuration
+      attr_accessor :promptfoo_executable, :root_path
+
+      def initialize
+        @promptfoo_executable = nil
+        @root_path = Dir.pwd
+      end
+
+      # Resolves the promptfoo executable path
+      # Priority: configured path > npx promptfoo
+      def resolve_executable
+        return promptfoo_executable if promptfoo_executable && executable_exists?(promptfoo_executable)
+
+        # Try local node_modules
+        local_bin = File.join(root_path, "node_modules", ".bin", "promptfoo")
+        return local_bin if executable_exists?(local_bin)
+
+        # Fall back to npx
+        "npx promptfoo"
+      end
+
+      private
+
+      def executable_exists?(path)
+        File.exist?(path) && File.executable?(path)
+      end
+    end
+
+    class << self
+      attr_writer :configuration
+
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield(configuration)
+      end
+
+      def reset_configuration!
+        @configuration = Configuration.new
+      end
+    end
+  end
+end

data/lib/minitest/promptfoo/failure_formatter.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Minitest
+  module Promptfoo
+    # Formats promptfoo test failures into human-readable error messages
+    class FailureFormatter
+      def initialize(verbose: false)
+        @verbose = verbose
+      end
+
+      # Main entry point: formats a complete failure message from promptfoo results
+      def format_results(passing_providers, failing_providers)
+        msg = "Prompt evaluation results:\n"
+
+        passing_providers.each do |provider_id|
+          msg += "  ✓ #{provider_id}\n"
+        end
+
+        failing_providers.each do |failure|
+          msg += "  ✗ #{failure[:id]}\n"
+        end
+
+        msg += "\n"
+
+        failing_providers.each do |failure|
+          msg += format_provider_failure(failure[:id], failure[:result])
+          msg += "\n"
+        end
+
+        unless @verbose
+          msg += "💡 Tip: Add `verbose: true` to assert_prompt for detailed debugging output\n"
+        end
+
+        msg
+      end
+
+      private
+
+      def format_provider_failure(provider_id, provider_result)
+        output_text = provider_result.dig("response", "output") || provider_result.dig("output")
+        error = provider_result.dig("error") || provider_result.dig("response", "error")
+        grading_result = provider_result.dig("gradingResult") || {}
+        component_results = grading_result.dig("componentResults") || []
+
+        msg = "#{provider_id} FAILED:\n\n"
+
+        msg += format_api_error(error) if error&.length&.positive?
+        msg += format_response_output(output_text, error)
+
+        assertion_failures = extract_assertion_failures(component_results)
+        msg += format_assertion_failures(assertion_failures, output_text) if assertion_failures.any?
+
+        msg += format_verbose_output(provider_result) if @verbose
+
+        msg
+      end
+
+      def format_api_error(error)
+        "API Error:\n  #{error}\n\n"
+      end
+
+      def format_response_output(output_text, error)
+        if output_text && output_text.to_s.length > 0
+          formatted_output = output_text.is_a?(String) ? output_text : JSON.pretty_generate(output_text)
+          "Response:\n  #{formatted_output.gsub("\n", "\n  ")}\n\n"
+        elsif !error || error.length == 0
+          "No response received from provider\n\n"
+        else
+          ""
+        end
+      end
+
+      def format_assertion_failures(assertion_failures, output_text)
+        msg = "Failures:\n"
+
+        # If JSON parsing failed, only show that error (other failures are consequences)
+        json_parse_failure = assertion_failures.find { |f| f[:type] == "is-json" }
+
+        if json_parse_failure
+          msg += format_assertion_failure(json_parse_failure, output_text)
+        else
+          assertion_failures.each do |failure|
+            msg += format_assertion_failure(failure, output_text)
+          end
+        end
+
+        msg
+      end
+
+      def format_verbose_output(provider_result)
+        "\nRaw Provider Result (verbose mode):\n" \
+          "  #{JSON.pretty_generate(provider_result).gsub("\n", "\n  ")}\n"
+      end
+
+      def extract_assertion_failures(component_results)
+        component_results.select { |result| !result.dig("pass") }.map do |result|
+          {
+            type: result.dig("assertion", "type"),
+            value: result.dig("assertion", "value"),
+            threshold: result.dig("assertion", "threshold"),
+            score: result.dig("score"),
+            reason: result.dig("reason"),
+            named_scores: result.dig("namedScores")
+          }
+        end
+      end
+
+      def format_assertion_failure(failure, output_text)
+        case failure[:type]
+        when "llm-rubric"
+          format_rubric_failure(failure)
+        when "contains"
+          "  ✗ includes(#{failure[:value].inspect}) - not found in response\n"
+        when "regex"
+          "  ✗ matches(/#{failure[:value]}/) - pattern not found\n"
+        when "equals"
+          "  ✗ equals(#{failure[:value].inspect}) - response does not match\n"
+        when "javascript"
+          format_javascript_failure(failure, output_text)
+        when "is-json"
+          format_invalid_json_failure(failure, output_text)
+        else
+          "  ✗ #{failure[:type]} assertion failed\n"
+        end
+      end
+
+      def format_javascript_failure(failure, output_text)
+        js_code = failure[:value].to_s
+
+        if json_assertion?(js_code)
+          parsed = parse_json_assertion(js_code)
+          if parsed
+            key = parsed[:key]
+            expected = parsed[:expected]
+            actual_value = extract_json_value(output_text, key.to_s)
+            msg = "  ✗ json_includes(key: #{key.inspect})\n"
+            msg += "    Expected: #{expected.inspect}\n"
+            msg += "    Actual: #{actual_value.inspect}\n"
+            return msg
+          end
+        end
+
+        "  ✗ javascript assertion failed\n"
+      end
+
+      def format_invalid_json_failure(failure, output_text)
+        msg = "  ✗ response is not valid JSON\n"
+
+        if output_text && output_text.to_s.length > 0
+          text = output_text.is_a?(String) ? output_text : JSON.pretty_generate(output_text)
+          snippet = (text.length > 100) ? "#{text[0..100]}..." : text
+          msg += "    Output: #{snippet.inspect}\n"
+        end
+
+        msg
+      end
+
+      def format_rubric_failure(failure)
+        score = failure[:score] || 0
+        threshold = failure[:threshold] || 0.5
+
+        msg = "  ✗ rubric (score: #{score.round(2)}/#{threshold})\n"
+        if score >= threshold
+          msg += "      Note: Score meets threshold but one or more criteria failed\n"
+          msg += "      Promptfoo requires ALL criteria to pass, not just the aggregate score\n"
+        end
+
+        if @verbose
+          criteria = failure[:value]
+          reason = failure[:reason]
+
+          if criteria && criteria.to_s.length > 0
+            msg += "\n    Rubric criteria:\n"
+            criteria.split("\n").each do |line|
+              msg += "      #{line}\n" if line.strip.length > 0
+            end
+          end
+
+          if reason && reason.to_s.length > 0
+            msg += "\n    Judge feedback:\n"
+            reason.split("\n").each do |line|
+              msg += "      #{line}\n"
+            end
+          end
+
+          msg += "\n"
+        end
+
+        msg
+      end
+
+      # JSON assertion helpers
+
+      def json_assertion?(js_code)
+        js_code.to_s.match?(/JSON\.parse\(output\)\[/)
+      end
+
+      def parse_json_assertion(js_code)
+        match = js_code.match(/JSON\.parse\(output\)\[(['"])(.+?)\1\]\s*===\s*(.+)/)
+        return nil unless match
+
+        key = match[2]
+        expected_json = match[3]
+
+        expected_value = begin
+          JSON.parse(expected_json)
+        rescue JSON::ParserError
+          expected_json
+        end
+
+        {key: key, expected: expected_value}
+      end
+
+      def extract_json_value(output_text, key)
+        return nil unless output_text && output_text.to_s.length > 0
+
+        parsed = output_text.is_a?(String) ? JSON.parse(output_text) : output_text
+        parsed[key]
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/minitest/promptfoo/promptfoo_runner.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "open3"
+require "json"
+
+module Minitest
+  module Promptfoo
+    # Handles execution of the promptfoo CLI and parsing of results
+    class PromptfooRunner
+      class ExecutionError < StandardError; end
+
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      # Executes promptfoo CLI with the given config and options
+      # Returns a hash with :success, :stdout, :stderr keys
+      def execute(config_path, working_dir, pre_render: false, show_output: false)
+        env_vars = build_env_vars(pre_render: pre_render)
+        cmd = build_command(config_path)
+
+        if show_output
+          execute_with_output(env_vars, cmd, working_dir)
+        else
+          execute_silently(env_vars, cmd, working_dir)
+        end
+      end
+
+      # Parses promptfoo JSON output file
+      def parse_output(output_path)
+        return {} unless File.exist?(output_path)
+
+        JSON.parse(File.read(output_path))
+      rescue JSON::ParserError => e
+        raise ExecutionError, "Failed to parse promptfoo output: #{e.message}"
+      end
+
+      private
+
+      def build_env_vars(pre_render:)
+        pre_render ? {"PROMPTFOO_DISABLE_TEMPLATING" => "true"} : {}
+      end
+
+      def build_command(config_path)
+        base_cmd = @configuration.resolve_executable
+        args = ["eval", "-c", config_path, "--no-cache"]
+
+        if base_cmd.start_with?("npx")
+          base_cmd.split + args
+        else
+          [base_cmd] + args
+        end
+      end
+
+      def execute_with_output(env_vars, cmd, working_dir)
+        success = system(env_vars, *cmd, chdir: working_dir)
+        {success: success, stdout: "", stderr: ""}
+      end
+
+      def execute_silently(env_vars, cmd, working_dir)
+        stdout, stderr, status = Open3.capture3(env_vars, *cmd, chdir: working_dir)
+        {success: status.success?, stdout: stdout, stderr: stderr}
+      end
+    end
+  end
+end

data/lib/minitest/promptfoo/rails.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Promptfoo
+    # Rails integration for automatic prompt file discovery
+    #
+    # Automatically discovers .ptmpl or .liquid prompt files based on Rails conventions:
+    #   app/services/foo/bar.ptmpl → test/services/foo/bar_test.rb
+    #
+    # Usage:
+    #   class MyPromptTest < Minitest::Promptfoo::RailsTest
+    #     # No need to define prompt_path, it's auto-discovered!
+    #
+    #     test "generates greeting" do
+    #       assert_prompt(vars: { name: "Alice" }) do |response|
+    #         response.includes("Hello Alice")
+    #       end
+    #     end
+    #   end
+    module Rails
+      def self.included(base)
+        base.class_eval do
+          # Override prompt_path to use Rails convention-based discovery
+          def prompt_path
+            @prompt_path ||= resolve_prompt_path_rails
+          end
+
+          private
+
+          def resolve_prompt_path_rails
+            test_file_path = method(name).source_location[0]
+            test_dir = File.dirname(test_file_path)
+            test_basename = File.basename(test_file_path, "_test.rb")
+
+            app_dir = test_dir.gsub(%r{^(.*/)?test/}, '\1app/')
+
+            [".ptmpl", ".liquid"].each do |ext|
+              candidate = File.join(app_dir, "#{test_basename}#{ext}")
+              return candidate if File.exist?(candidate)
+            end
+
+            raise PromptNotFoundError, "Could not find prompt file for #{test_file_path}"
+          end
+        end
+      end
+    end
+
+    # Convenience class that combines Test + Rails integration
+    # Inherits from ActiveSupport::TestCase if available, otherwise Minitest::Test
+    if defined?(ActiveSupport::TestCase)
+      class RailsTest < ActiveSupport::TestCase
+        include Minitest::Promptfoo::Rails
+
+        # Borrow all the assertion methods from Test
+        # but keep ActiveSupport::TestCase as the base
+        include Minitest::Promptfoo::Test.instance_methods(false).each_with_object(Module.new) { |m, mod|
+          mod.define_method(m, Minitest::Promptfoo::Test.instance_method(m))
+        }
+
+        # Include class methods
+        class << self
+          attr_accessor :_providers
+
+          def providers
+            @_providers || "echo"
+          end
+
+          def providers=(value)
+            @_providers = value
+          end
+
+          def inherited(subclass)
+            super
+            subclass._providers = _providers
+          end
+        end
+      end
+    else
+      # Fallback if ActiveSupport isn't available
+      class RailsTest < Test
+        include Rails
+      end
+    end
+  end
+end

data/lib/minitest/promptfoo/test.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "tmpdir"
+require "minitest/test"
+require_relative "assertion_builder"
+require_relative "failure_formatter"
+require_relative "promptfoo_runner"
+
+module Minitest
+  module Promptfoo
+    # Base class for testing LLM prompts using promptfoo.
+    #
+    # Recommended Usage (Minitest-like DSL):
+    #   class MyPromptTest < Minitest::Promptfoo::Test
+    #     # Set provider(s) for ALL tests in this class (DRY!)
+    #     # Providers can be strings or hashes with config (see promptfoo docs)
+    #     self.providers = [
+    #       "openai:gpt-4o-mini",  # Simple string format
+    #       {
+    #         id: "openai:chat:anthropic:claude-3-7-sonnet",
+    #         config: { response_format: { type: "json_object" } }  # With config
+    #       }
+    #     ]
+    #
+    #     def prompt_path
+    #       "prompts/greeting.ptmpl"  # Or .liquid
+    #     end
+    #
+    #     test "generates professional greeting" do
+    #       assert_prompt(vars: { name: "Alice" }) do |response|
+    #         response.includes("Hello Alice")
+    #         response.matches(/[A-Z]/)  # Starts with capital letter
+    #         response.rubric("Response is professional and courteous")
+    #       end
+    #     end
+    #   end
+    class Test < Minitest::Test
+      class PromptNotFoundError < StandardError; end
+      class EvaluationError < StandardError; end
+
+      # Class-level configuration
+      class << self
+        def debug?
+          ENV["DEBUG_PROMPT_TEST"] == "1"
+        end
+
+        def providers
+          @providers || "echo"
+        end
+
+        attr_writer :providers
+
+        def inherited(subclass)
+          super
+          subclass.providers = providers if defined?(@providers)
+        end
+      end
+
+      def prompt_path
+        raise NotImplementedError, "#{self.class}#prompt_path must be implemented"
+      end
+
+      def prompt_content
+        @prompt_content ||= begin
+          path = prompt_path
+          raise PromptNotFoundError, "Prompt file not found: #{path}" unless File.exist?(path)
+          File.read(path, encoding: "UTF-8")
+        end
+      end
+
+      # Minitest-like DSL for prompt testing
+      #
+      # Example:
+      #   assert_prompt(vars: { input: "test" }) do |response|
+      #     response.includes("expected text")
+      #     response.matches(/\d{3}-\d{4}/)
+      #     response.rubric("Response is professional and courteous")
+      #   end
+      def assert_prompt(vars:, providers: nil, verbose: false, pre_render: false, &block)
+        builder = AssertionBuilder.new
+        yield(builder)
+
+        output = evaluate_prompt(
+          prompt_text: prompt_content,
+          vars: vars,
+          providers: providers,
+          assertions: builder.to_promptfoo_assertions,
+          verbose: verbose,
+          pre_render: pre_render
+        )
+
+        # Real assertion: verify promptfoo produced results
+        assert(output.any?, "Promptfoo evaluation produced no output")
+
+        output
+      end
+
+      def evaluate_prompt(prompt_text:, vars:, providers: nil, assertions: [], pre_render: false, verbose: false, show_output: false)
+        Dir.mktmpdir do |tmpdir|
+          config_path = File.join(tmpdir, "promptfooconfig.yaml")
+          output_path = File.join(tmpdir, "output.json")
+
+          # Convert single-brace {var} syntax to double-brace {{var}} for promptfoo
+          promptfoo_text = prompt_text.gsub(/(?<!\{)\{(\w+)\}(?!\})/, '{{\1}}')
+
+          if pre_render
+            vars.each do |key, value|
+              promptfoo_text = promptfoo_text.gsub("{{#{key}}}", value.to_s)
+            end
+            config_vars = {}
+          else
+            config_vars = vars
+          end
+
+          # Use provided provider(s) or fall back to class-level default
+          providers_array = wrap_array(providers || self.class.providers)
+
+          config = build_promptfoo_config(
+            prompt: promptfoo_text,
+            vars: config_vars,
+            providers: providers_array,
+            assertions: assertions,
+            output_path: output_path
+          )
+
+          config_yaml = YAML.dump(config)
+          File.write(config_path, config_yaml)
+
+          debug("Promptfoo Config", config_yaml)
+
+          runner = PromptfooRunner.new(Minitest::Promptfoo.configuration)
+          result = runner.execute(config_path, tmpdir, show_output: show_output, pre_render: pre_render)
+
+          debug("Promptfoo Result", result.inspect)
+
+          output = runner.parse_output(output_path)
+
+          unless result[:success] || output.any?
+            raise EvaluationError, <<~ERROR
+              promptfoo evaluation failed
+              STDOUT: #{result[:stdout]}
+              STDERR: #{result[:stderr]}
+            ERROR
+          end
+
+          check_provider_failures(output, providers_array, verbose: verbose) if assertions.any?
+
+          output
+        end
+      end
+
+      private
+
+      def check_provider_failures(output, providers, verbose: false)
+        results = output.dig("results", "results") || []
+        passing_providers = []
+        failing_providers = []
+
+        results.each do |provider_result|
+          provider_id = provider_result.dig("provider", "id")
+          success = provider_result.dig("success")
+
+          if success
+            passing_providers << provider_id
+          else
+            failing_providers << {
+              id: provider_id,
+              result: provider_result
+            }
+          end
+        end
+
+        if failing_providers.any?
+          formatter = FailureFormatter.new(verbose: verbose)
+          error_msg = formatter.format_results(passing_providers, failing_providers)
+          flunk(error_msg)
+        end
+      end
+
+      def build_promptfoo_config(prompt:, vars:, providers:, assertions:, output_path:)
+        normalized_providers = providers.map do |provider|
+          case provider
+          when String
+            provider
+          when Hash
+            deep_stringify_keys(provider)
+          end
+        end
+
+        {
+          "prompts" => [prompt],
+          "providers" => normalized_providers,
+          "tests" => [
+            {
+              "vars" => vars.transform_keys(&:to_s),
+              "assert" => assertions
+            }
+          ],
+          "outputPath" => output_path
+        }
+      end
+
+      def debug(title, content)
+        return unless self.class.debug?
+
+        warn "\n=== #{title} ==="
+        warn content
+        warn "=" * (title.length + 8)
+        warn ""
+      end
+
+      # Simple array wrapper (replaces ActiveSupport's Array.wrap)
+      def wrap_array(object)
+        case object
+        when nil then []
+        when Array then object
+        else [object]
+        end
+      end
+
+      # Simple deep stringify keys (replaces ActiveSupport method)
+      def deep_stringify_keys(hash)
+        hash.each_with_object({}) do |(key, value), result|
+          result[key.to_s] = stringify_value(value)
+        end
+      end
+
+      def stringify_value(value)
+        case value
+        when Hash then deep_stringify_keys(value)
+        when Array then value.map { |v| stringify_value(v) }
+        else value
+        end
+      end
+    end
+  end
+end

data/lib/minitest/promptfoo/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Promptfoo
+    VERSION = "0.1.0"
+  end
+end

data/lib/minitest/promptfoo.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "promptfoo/version"
+require_relative "promptfoo/configuration"
+require_relative "promptfoo/test"
+
+# Auto-load Rails integration if Rails is detected
+if defined?(Rails)
+  require_relative "promptfoo/rails"
+end
+
+module Minitest
+  module Promptfoo
+    class Error < StandardError; end
+  end
+end

data/sig/minitest/promptfoo.rbs

@@ -0,0 +1,6 @@
+module Minitest
+  module Promptfoo
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: minitest-promptfoo
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Waters
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.35.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.35.1
+description: A thin Minitest wrapper around promptfoo that brings prompt testing to
+  Ruby projects. Test LLM prompts with a familiar Minitest-like DSL, supporting multiple
+  providers and assertion types.
+email:
+- chris.waters@shopify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".ruby-version"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/greeting.ptmpl
+- examples/simple_prompt_test.rb
+- lib/minitest/promptfoo.rb
+- lib/minitest/promptfoo/assertion_builder.rb
+- lib/minitest/promptfoo/configuration.rb
+- lib/minitest/promptfoo/failure_formatter.rb
+- lib/minitest/promptfoo/promptfoo_runner.rb
+- lib/minitest/promptfoo/rails.rb
+- lib/minitest/promptfoo/test.rb
+- lib/minitest/promptfoo/version.rb
+- sig/minitest/promptfoo.rbs
+homepage: https://github.com/christhesoul/minitest-promptfoo
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/christhesoul/minitest-promptfoo
+  source_code_uri: https://github.com/christhesoul/minitest-promptfoo
+  changelog_uri: https://github.com/christhesoul/minitest-promptfoo/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Minitest integration for promptfoo - test your LLM prompts with confidence
+test_files: []