minitest-promptfoo 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c46190c19ff929aa46aa5b39b6857af623f0ca6ede8187a58220c5e8e5ea0fe2
-  data.tar.gz: fc4d8fec213be486de2398a9fc050d3a683d3b610012bf4107f2bd09e34bea7d
+  metadata.gz: 897a922e922eea260395f08b5f40433b548893a7db1a2b6e0c645596d142eb48
+  data.tar.gz: 8e272e24fd278722bde4a250f6d785413a2df9855a82362e266cca63cc7be3f5
 SHA512:
-  metadata.gz: 8e59555be654b7ab5aa40e1f08747a4d71989a25c162cf57a6ba8ee4cbf4c4365c7f3d9844c61937fb1d82b445bdedc9982e61ae515eb8ac87f5b212771c5c18
-  data.tar.gz: a8002f9d6f25419abcbb0f89b687246d15c2b89ada13219fed14b11fcf13cb271ce3e694f173a6472d5d0ae38100dc73a3c1fd11e56092c62862cbb43cf67fbe
+  metadata.gz: 15348b1b8f62175e2b33e2982442bbad5b78cc8ecb79c9ebaf8dc17d85631a1cd05a46e9ead0da63b64e35cf1d03d7c762618f318bb992e84f1c8e86d8e0b6dc
+  data.tar.gz: 5c4e0dfd6f9f4343a8985964b0dee89fab2b65306ee768b56a713fc30970ecfd1ef45f95964f0df9c7a85bacc111db49b3d7757ea45d5997c2155095cc9b22fa

data/CHANGELOG.md

@@ -28,3 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [0.1.1] - Spirit in the Night
 
 - Fixed bug causing unbound method call in Rails apps
+
+## [0.1.2] - Growin' Up
+
+- Fixed more bugs relating to Rails implementation

data/README.md

@@ -85,7 +85,7 @@ In Rails, the gem automatically discovers prompt files based on test file paths:
 
 ```ruby
 # test/services/greeting_service_test.rb
-class GreetingServiceTest < Minitest::Promptfoo::RailsTest
+class GreetingServiceTest < Minitest::Promptfoo::PromptTest
   self.providers = "openai:gpt-4o-mini"
 
   # Automatically finds app/services/greeting_service.ptmpl
@@ -303,7 +303,7 @@ When using `Minitest::Promptfoo::Test` (non-Rails), note these differences:
 - No automatic database transaction rollbacks
 - Uses plain Minitest assertions
 
-For Rails projects, use `Minitest::Promptfoo::RailsTest` to get all Rails testing features plus automatic prompt discovery.
+For Rails projects, use `Minitest::Promptfoo::PromptTest` to get all Rails testing features plus automatic prompt discovery.
 
 ## Development
 

data/lib/minitest/promptfoo/rails.rb

@@ -8,7 +8,7 @@ module Minitest
     #   app/services/foo/bar.ptmpl → test/services/foo/bar_test.rb
     #
     # Usage:
-    #   class MyPromptTest < Minitest::Promptfoo::RailsTest
+    #   class MyPromptTest < Minitest::Promptfoo::PromptTest
     #     # No need to define prompt_path, it's auto-discovered!
     #
     #     test "generates greeting" do
@@ -39,7 +39,7 @@ module Minitest
               return candidate if File.exist?(candidate)
             end
 
-            raise PromptNotFoundError, "Could not find prompt file for #{test_file_path}"
+            raise Minitest::Promptfoo::TestMethods::PromptNotFoundError, "Could not find prompt file for #{test_file_path}"
           end
         end
       end
@@ -47,39 +47,19 @@ module Minitest
 
     # Convenience class that combines Test + Rails integration
     # Inherits from ActiveSupport::TestCase if available, otherwise Minitest::Test
-    if defined?(ActiveSupport::TestCase)
-      class RailsTest < ActiveSupport::TestCase
-        include Minitest::Promptfoo::Rails
-
-        # Copy instance methods from Test
-        Minitest::Promptfoo::Test.instance_methods(false).each do |method_name|
-          define_method(method_name) do |*args, **kwargs, &block|
-            # Delegate to Test's implementation
-            Minitest::Promptfoo::Test.instance_method(method_name).bind_call(self, *args, **kwargs, &block)
-          end
-        end
-
-        # Include class methods
-        class << self
-          def debug?
-            ENV["DEBUG_PROMPT_TEST"] == "1"
-          end
-
-          def providers
-            @providers || "echo"
-          end
-
-          attr_writer :providers
-
-          def inherited(subclass)
-            super
-            subclass.providers = providers if defined?(@providers)
+    if defined?(ActiveSupport)
+      # Defer class definition until Rails test framework is fully loaded
+      ActiveSupport.on_load(:active_support_test_case) do
+        unless Minitest::Promptfoo.const_defined?(:PromptTest)
+          class PromptTest < ActiveSupport::TestCase
+            include Minitest::Promptfoo::TestMethods
+            include Minitest::Promptfoo::Rails
           end
         end
       end
     else
       # Fallback if ActiveSupport isn't available
-      class RailsTest < Test
+      class PromptTest < Test
         include Rails
       end
     end

data/lib/minitest/promptfoo/test.rb

@@ -1,11 +1,10 @@
 # frozen_string_literal: true
 
-require "yaml"
-require "tmpdir"
 require "minitest/test"
 require_relative "assertion_builder"
 require_relative "failure_formatter"
 require_relative "promptfoo_runner"
+require_relative "test_methods"
 
 module Minitest
   module Promptfoo
@@ -36,203 +35,7 @@ module Minitest
     #     end
     #   end
     class Test < Minitest::Test
-      class PromptNotFoundError < StandardError; end
-      class EvaluationError < StandardError; end
-
-      # Class-level configuration
-      class << self
-        def debug?
-          ENV["DEBUG_PROMPT_TEST"] == "1"
-        end
-
-        def providers
-          @providers || "echo"
-        end
-
-        attr_writer :providers
-
-        def inherited(subclass)
-          super
-          subclass.providers = providers if defined?(@providers)
-        end
-      end
-
-      def prompt_path
-        raise NotImplementedError, "#{self.class}#prompt_path must be implemented"
-      end
-
-      def prompt_content
-        @prompt_content ||= begin
-          path = prompt_path
-          raise PromptNotFoundError, "Prompt file not found: #{path}" unless File.exist?(path)
-          File.read(path, encoding: "UTF-8")
-        end
-      end
-
-      # Minitest-like DSL for prompt testing
-      #
-      # Example:
-      #   assert_prompt(vars: { input: "test" }) do |response|
-      #     response.includes("expected text")
-      #     response.matches(/\d{3}-\d{4}/)
-      #     response.rubric("Response is professional and courteous")
-      #   end
-      def assert_prompt(vars:, providers: nil, verbose: false, pre_render: false, &block)
-        builder = AssertionBuilder.new
-        yield(builder)
-
-        output = evaluate_prompt(
-          prompt_text: prompt_content,
-          vars: vars,
-          providers: providers,
-          assertions: builder.to_promptfoo_assertions,
-          verbose: verbose,
-          pre_render: pre_render
-        )
-
-        # Real assertion: verify promptfoo produced results
-        assert(output.any?, "Promptfoo evaluation produced no output")
-
-        output
-      end
-
-      def evaluate_prompt(prompt_text:, vars:, providers: nil, assertions: [], pre_render: false, verbose: false, show_output: false)
-        Dir.mktmpdir do |tmpdir|
-          config_path = File.join(tmpdir, "promptfooconfig.yaml")
-          output_path = File.join(tmpdir, "output.json")
-
-          # Convert single-brace {var} syntax to double-brace {{var}} for promptfoo
-          promptfoo_text = prompt_text.gsub(/(?<!\{)\{(\w+)\}(?!\})/, '{{\1}}')
-
-          if pre_render
-            vars.each do |key, value|
-              promptfoo_text = promptfoo_text.gsub("{{#{key}}}", value.to_s)
-            end
-            config_vars = {}
-          else
-            config_vars = vars
-          end
-
-          # Use provided provider(s) or fall back to class-level default
-          providers_array = wrap_array(providers || self.class.providers)
-
-          config = build_promptfoo_config(
-            prompt: promptfoo_text,
-            vars: config_vars,
-            providers: providers_array,
-            assertions: assertions,
-            output_path: output_path
-          )
-
-          config_yaml = YAML.dump(config)
-          File.write(config_path, config_yaml)
-
-          debug("Promptfoo Config", config_yaml)
-
-          runner = PromptfooRunner.new(Minitest::Promptfoo.configuration)
-          result = runner.execute(config_path, tmpdir, show_output: show_output, pre_render: pre_render)
-
-          debug("Promptfoo Result", result.inspect)
-
-          output = runner.parse_output(output_path)
-
-          unless result[:success] || output.any?
-            raise EvaluationError, <<~ERROR
-              promptfoo evaluation failed
-              STDOUT: #{result[:stdout]}
-              STDERR: #{result[:stderr]}
-            ERROR
-          end
-
-          check_provider_failures(output, providers_array, verbose: verbose) if assertions.any?
-
-          output
-        end
-      end
-
-      private
-
-      def check_provider_failures(output, providers, verbose: false)
-        results = output.dig("results", "results") || []
-        passing_providers = []
-        failing_providers = []
-
-        results.each do |provider_result|
-          provider_id = provider_result.dig("provider", "id")
-          success = provider_result.dig("success")
-
-          if success
-            passing_providers << provider_id
-          else
-            failing_providers << {
-              id: provider_id,
-              result: provider_result
-            }
-          end
-        end
-
-        if failing_providers.any?
-          formatter = FailureFormatter.new(verbose: verbose)
-          error_msg = formatter.format_results(passing_providers, failing_providers)
-          flunk(error_msg)
-        end
-      end
-
-      def build_promptfoo_config(prompt:, vars:, providers:, assertions:, output_path:)
-        normalized_providers = providers.map do |provider|
-          case provider
-          when String
-            provider
-          when Hash
-            deep_stringify_keys(provider)
-          end
-        end
-
-        {
-          "prompts" => [prompt],
-          "providers" => normalized_providers,
-          "tests" => [
-            {
-              "vars" => vars.transform_keys(&:to_s),
-              "assert" => assertions
-            }
-          ],
-          "outputPath" => output_path
-        }
-      end
-
-      def debug(title, content)
-        return unless self.class.debug?
-
-        warn "\n=== #{title} ==="
-        warn content
-        warn "=" * (title.length + 8)
-        warn ""
-      end
-
-      # Simple array wrapper (replaces ActiveSupport's Array.wrap)
-      def wrap_array(object)
-        case object
-        when nil then []
-        when Array then object
-        else [object]
-        end
-      end
-
-      # Simple deep stringify keys (replaces ActiveSupport method)
-      def deep_stringify_keys(hash)
-        hash.each_with_object({}) do |(key, value), result|
-          result[key.to_s] = stringify_value(value)
-        end
-      end
-
-      def stringify_value(value)
-        case value
-        when Hash then deep_stringify_keys(value)
-        when Array then value.map { |v| stringify_value(v) }
-        else value
-        end
-      end
+      include TestMethods
     end
   end
 end

data/lib/minitest/promptfoo/test_methods.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "tmpdir"
+
+module Minitest
+  module Promptfoo
+    # Shared behavior for prompt testing that can be included in any test class
+    module TestMethods
+      class PromptNotFoundError < StandardError; end
+      class EvaluationError < StandardError; end
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def debug?
+          ENV["DEBUG_PROMPT_TEST"] == "1"
+        end
+
+        def providers
+          @providers || "echo"
+        end
+
+        attr_writer :providers
+
+        def inherited(subclass)
+          super
+          subclass.providers = providers if defined?(@providers)
+        end
+      end
+
+      def prompt_path
+        raise NotImplementedError, "#{self.class}#prompt_path must be implemented"
+      end
+
+      def prompt_content
+        @prompt_content ||= begin
+          path = prompt_path
+          raise PromptNotFoundError, "Prompt file not found: #{path}" unless File.exist?(path)
+          File.read(path, encoding: "UTF-8")
+        end
+      end
+
+      def assert_prompt(vars:, providers: nil, verbose: false, pre_render: false, &block)
+        builder = AssertionBuilder.new
+        yield(builder)
+
+        output = evaluate_prompt(
+          prompt_text: prompt_content,
+          vars: vars,
+          providers: providers,
+          assertions: builder.to_promptfoo_assertions,
+          verbose: verbose,
+          pre_render: pre_render
+        )
+
+        # Real assertion: verify promptfoo produced results
+        assert(output.any?, "Promptfoo evaluation produced no output")
+
+        output
+      end
+
+      def evaluate_prompt(prompt_text:, vars:, providers: nil, assertions: [], pre_render: false, verbose: false, show_output: false)
+        Dir.mktmpdir do |tmpdir|
+          config_path = File.join(tmpdir, "promptfooconfig.yaml")
+          output_path = File.join(tmpdir, "output.json")
+
+          # Convert single-brace {var} syntax to double-brace {{var}} for promptfoo
+          promptfoo_text = prompt_text.gsub(/(?<!\{)\{(\w+)\}(?!\})/, '{{\1}}')
+
+          if pre_render
+            vars.each do |key, value|
+              promptfoo_text = promptfoo_text.gsub("{{#{key}}}", value.to_s)
+            end
+            config_vars = {}
+          else
+            config_vars = vars
+          end
+
+          # Use provided provider(s) or fall back to class-level default
+          providers_array = wrap_array(providers || self.class.providers)
+
+          config = build_promptfoo_config(
+            prompt: promptfoo_text,
+            vars: config_vars,
+            providers: providers_array,
+            assertions: assertions,
+            output_path: output_path
+          )
+
+          config_yaml = YAML.dump(config)
+          File.write(config_path, config_yaml)
+
+          debug("Promptfoo Config", config_yaml)
+
+          runner = PromptfooRunner.new(Minitest::Promptfoo.configuration)
+          result = runner.execute(config_path, tmpdir, show_output: show_output, pre_render: pre_render)
+
+          debug("Promptfoo Result", result.inspect)
+
+          output = runner.parse_output(output_path)
+
+          unless result[:success] || output.any?
+            raise EvaluationError, <<~ERROR
+              promptfoo evaluation failed
+              STDOUT: #{result[:stdout]}
+              STDERR: #{result[:stderr]}
+            ERROR
+          end
+
+          check_provider_failures(output, providers_array, verbose: verbose) if assertions.any?
+
+          output
+        end
+      end
+
+      private
+
+      def check_provider_failures(output, providers, verbose: false)
+        results = output.dig("results", "results") || []
+        passing_providers = []
+        failing_providers = []
+
+        results.each do |provider_result|
+          provider_id = provider_result.dig("provider", "id")
+          success = provider_result.dig("success")
+
+          if success
+            passing_providers << provider_id
+          else
+            failing_providers << {
+              id: provider_id,
+              result: provider_result
+            }
+          end
+        end
+
+        if failing_providers.any?
+          formatter = FailureFormatter.new(verbose: verbose)
+          error_msg = formatter.format_results(passing_providers, failing_providers)
+          flunk(error_msg)
+        end
+      end
+
+      def build_promptfoo_config(prompt:, vars:, providers:, assertions:, output_path:)
+        normalized_providers = providers.map do |provider|
+          case provider
+          when String
+            provider
+          when Hash
+            deep_stringify_keys(provider)
+          end
+        end
+
+        {
+          "prompts" => [prompt],
+          "providers" => normalized_providers,
+          "tests" => [
+            {
+              "vars" => vars.transform_keys(&:to_s),
+              "assert" => assertions
+            }
+          ],
+          "outputPath" => output_path
+        }
+      end
+
+      def debug(title, content)
+        return unless self.class.debug?
+
+        warn "\n=== #{title} ==="
+        warn content
+        warn "=" * (title.length + 8)
+        warn ""
+      end
+
+      # Simple array wrapper (replaces ActiveSupport's Array.wrap)
+      def wrap_array(object)
+        case object
+        when nil then []
+        when Array then object
+        else [object]
+        end
+      end
+
+      # Simple deep stringify keys (replaces ActiveSupport method)
+      def deep_stringify_keys(hash)
+        hash.each_with_object({}) do |(key, value), result|
+          result[key.to_s] = stringify_value(value)
+        end
+      end
+
+      def stringify_value(value)
+        case value
+        when Hash then deep_stringify_keys(value)
+        when Array then value.map { |v| stringify_value(v) }
+        else value
+        end
+      end
+    end
+  end
+end

data/lib/minitest/promptfoo/version.rb

@@ -2,6 +2,6 @@
 
 module Minitest
   module Promptfoo
-    VERSION = "0.1.1"
+    VERSION = "0.1.3"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: minitest-promptfoo
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Chris Waters
@@ -74,6 +74,7 @@ files:
 - lib/minitest/promptfoo/promptfoo_runner.rb
 - lib/minitest/promptfoo/rails.rb
 - lib/minitest/promptfoo/test.rb
+- lib/minitest/promptfoo/test_methods.rb
 - lib/minitest/promptfoo/version.rb
 - sig/minitest/promptfoo.rbs
 homepage: https://github.com/christhesoul/minitest-promptfoo