minitest-promptfoo 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 897a922e922eea260395f08b5f40433b548893a7db1a2b6e0c645596d142eb48
-  data.tar.gz: 8e272e24fd278722bde4a250f6d785413a2df9855a82362e266cca63cc7be3f5
+  metadata.gz: adc15656589f6ac5ef5c617ff36f9461de0d8cea9aaee9c5dcb1f93a3846657e
+  data.tar.gz: a64e466f50c5388e03419b3dec3a38ada32d87507b60c7377579c0c1135a96ee
 SHA512:
-  metadata.gz: 15348b1b8f62175e2b33e2982442bbad5b78cc8ecb79c9ebaf8dc17d85631a1cd05a46e9ead0da63b64e35cf1d03d7c762618f318bb992e84f1c8e86d8e0b6dc
-  data.tar.gz: 5c4e0dfd6f9f4343a8985964b0dee89fab2b65306ee768b56a713fc30970ecfd1ef45f95964f0df9c7a85bacc111db49b3d7757ea45d5997c2155095cc9b22fa
+  metadata.gz: a11d7f0afa408a7e551068976bbfb9c29ffce7f0935402cd3830660a4d0a75da8a344619ed47c51672f2a90cccb044baf1939e044289f548854af0e0d2b3ba97
+  data.tar.gz: 678f4a3dd170d6660f373ac45fe268f37bf7bf8c2a82352722137f6df04037325644d93fc4b36d4388593e61c505cea742ab5120f1343a5905b0f76042dddb0b

data/CHANGELOG.md

@@ -2,13 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.1.0] - Blinded by the Light
 
-## [Unreleased]
+Initial release of minitest-promptfoo:
 
-### Added
-- Initial release of minitest-promptfoo
 - Core `Minitest::Promptfoo::Test` class for prompt testing
 - Configuration system for promptfoo executable path
 - Support for multiple providers
@@ -21,10 +18,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Comprehensive README with examples
 - Basic test coverage
 
-## [0.1.0] - Blinded by the Light
-
-- Initial release
-
 ## [0.1.1] - Spirit in the Night
 
 - Fixed bug causing unbound method call in Rails apps
@@ -32,3 +25,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [0.1.2] - Growin' Up
 
 - Fixed more bugs relating to Rails implementation
+
+## [0.1.3] - Does This Bus Stop at 82nd Street?
+
+- Renamed `Minitest::Promptfoo::RailsTest` to `Minitest::Promptfoo::PromptTest` for clearer semantics
+
+## [0.1.4] - For You
+
+- `force_json!` method to handle JSON responses wrapped in markdown code fences (e.g., ` ```json `)
+  - Automatically strips backticks before parsing JSON
+  - Skips `is-json` validation when enabled (since raw output isn't valid JSON)
+  - Particularly useful for Anthropic and other providers that ignore `response_format` settings

data/CLAUDE.md

@@ -0,0 +1,15 @@
+This is a Ruby gem that provides a thin Ruby wrapper around the Promptfoo prompt testing framework.
+
+The aim of this project is to provide a Minitest aesthetic to testing prompts that live in a repo.
+
+Our aim is not to recreate Promptfoo in Ruby, but rather to harness the power of Promptfoo while keeping the joy of writing Ruby.
+
+Your tasks will probably involve providing a Ruby to interface to Promptfoo features. This generally involves
+coercing code within a Minitest-style assertion into some YAML that we then provide when shelling out to the
+Promptfoo command line tool.
+
+Always bias towards simple and beautiful Ruby code.
+
+Resources:
+- Promptfoo repo: https://github.com/promptfoo/promptfoo
+- Promptfoo docs: https://www.promptfoo.dev/docs/intro/

data/README.md

@@ -143,6 +143,24 @@ assert_prompt(vars: { query: "status" }) do |response|
 end
 ```
 
+#### Handling Markdown-Wrapped JSON
+
+Some providers (looking at you, Anthropic) wrap JSON responses in markdown code fences like ` ```json `. Use `force_json!` to strip these before parsing:
+
+```ruby
+assert_prompt(vars: { query: "status" }) do |response|
+  response.force_json!  # Strips ```json and ``` wrappers
+
+  response.json_includes(key: "status", value: "success")
+  response.json_includes(key: "exclusive_to_app", value: true)
+end
+```
+
+When `force_json!` is called:
+- Markdown code fences are automatically stripped before JSON parsing
+- The `is-json` validation is skipped (since the raw output isn't valid JSON)
+- All subsequent `json_includes` assertions handle the stripping automatically
+
 ### Custom JavaScript
 
 ```ruby

data/lib/minitest/promptfoo/assertion_builder.rb

@@ -15,6 +15,7 @@ module Minitest
     class AssertionBuilder
       def initialize
         @assertions = []
+        @force_json = false
       end
 
       # String inclusion check
@@ -43,13 +44,24 @@ module Minitest
 
       # JSON structure validation using JavaScript
       def json_includes(key:, value:)
-        @assertions << {
-          "type" => "is-json"
-        }
+        # Only validate is-json if we're not forcing JSON (since force_json means output may have markdown fences)
+        unless @force_json
+          @assertions << {
+            "type" => "is-json"
+          }
+        end
+
+        # Build the parsing logic - strip markdown fences if force_json is enabled
+        parse_logic = if @force_json
+          "JSON.parse(output.replace(/^```(?:json)?\\n?|\\n?```$/g, '').trim())"
+        else
+          "JSON.parse(output)"
+        end
+
         # Handle both string output (needs parsing) and object output (already parsed)
         @assertions << {
           "type" => "javascript",
-          "value" => "(typeof output === 'string' ? JSON.parse(output) : output)[#{key.inspect}] === #{value.to_json}"
+          "value" => "(typeof output === 'string' ? #{parse_logic} : output)[#{key.inspect}] === #{value.to_json}"
         }
       end
 
@@ -70,6 +82,16 @@ module Minitest
         }
       end
 
+      # Force JSON parsing by stripping markdown code fences
+      def force_json!
+        @force_json = true
+      end
+
+      # Check if force_json was called
+      def force_json?
+        @force_json
+      end
+
       # Convert to promptfoo assertion format
       def to_promptfoo_assertions
         @assertions

data/lib/minitest/promptfoo/test_methods.rb

@@ -53,7 +53,8 @@ module Minitest
           providers: providers,
           assertions: builder.to_promptfoo_assertions,
           verbose: verbose,
-          pre_render: pre_render
+          pre_render: pre_render,
+          force_json: builder.force_json?
         )
 
         # Real assertion: verify promptfoo produced results
@@ -62,7 +63,7 @@ module Minitest
         output
       end
 
-      def evaluate_prompt(prompt_text:, vars:, providers: nil, assertions: [], pre_render: false, verbose: false, show_output: false)
+      def evaluate_prompt(prompt_text:, vars:, providers: nil, assertions: [], pre_render: false, verbose: false, show_output: false, force_json: false)
         Dir.mktmpdir do |tmpdir|
           config_path = File.join(tmpdir, "promptfooconfig.yaml")
           output_path = File.join(tmpdir, "output.json")
@@ -87,7 +88,8 @@ module Minitest
             vars: config_vars,
             providers: providers_array,
             assertions: assertions,
-            output_path: output_path
+            output_path: output_path,
+            force_json: force_json
           )
 
           config_yaml = YAML.dump(config)
@@ -144,14 +146,21 @@ module Minitest
         end
       end
 
-      def build_promptfoo_config(prompt:, vars:, providers:, assertions:, output_path:)
+      def build_promptfoo_config(prompt:, vars:, providers:, assertions:, output_path:, force_json: false)
         normalized_providers = providers.map do |provider|
-          case provider
+          provider_config = case provider
           when String
-            provider
+            { "id" => provider }
           when Hash
             deep_stringify_keys(provider)
           end
+
+          if force_json
+            provider_config["config"] ||= {}
+            provider_config["config"]["transformResponse"] = strip_markdown_fences_js
+          end
+
+          provider_config
         end
 
         {
@@ -199,6 +208,11 @@ module Minitest
         else value
         end
       end
+
+      # JavaScript function to strip markdown code fences from JSON responses
+      def strip_markdown_fences_js
+        "json.output.replace(/^```(?:json)?\\n?|\\n?```$/g, '').trim()"
+      end
     end
   end
 end

data/lib/minitest/promptfoo/version.rb

@@ -2,6 +2,6 @@
 
 module Minitest
   module Promptfoo
-    VERSION = "0.1.3"
+    VERSION = "0.1.4"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: minitest-promptfoo
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Chris Waters
@@ -62,6 +62,7 @@ extra_rdoc_files: []
 files:
 - ".ruby-version"
 - CHANGELOG.md
+- CLAUDE.md
 - LICENSE.txt
 - README.md
 - Rakefile