elasticsearch-test-runner 0.17.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce6c6e7adcbf6ba2d8f592f776d51bb1903b8ae2da6591981413fda1e44ce6e7
-  data.tar.gz: e3d92c1626835a8c5eec26b85b1050c35b02e62c9f9e5e30abd64add7c0afe45
+  metadata.gz: 897b6c6fd42ff59d4a4a03ad00fd86ada140707095ded6dbdc16e24b1d1c7e57
+  data.tar.gz: 680d531cb3c7f06aa35b693c91d0b61f34e8a1aac5e5ba5fd89421b60a8589c8
 SHA512:
-  metadata.gz: be441d1a5d468c450cd282f650134f4bd333d91b4c1bd373affb73c0f451283b0fe2a92f6e91cd9688e8b26e5e698edcd319b0c7c7759bb5ddb5e1ad3e0e1237
-  data.tar.gz: 24850372db7b732513f58005e838044a8e637890b220fcd5a1f363beacff1bafe8747f5bf8af6c92ba9a388b85fabf815682b6ea025ff8acf9508b91a9a5e198
+  metadata.gz: 898b06aad5b5f1b4def588adccb9099e5618570884654650ab2704c471573b4fda55c3aa7fa3d4254cc9913c325f85d30b7160fecfe5e7f23fbff913370a972c
+  data.tar.gz: f8c9e058b5e78e69eac3ae80f3519572bd0166e4c1b3ddf785b3502e1cd8cff1824fd85d861254e195652dc399e8d7d116a80b3ca3de7cc06c5f1f3acf85b50e

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [0.18.0] - 2026-03-05
+
+- Lots of improvements when displaying errors. Fixes missing exceptions during particular failures. The display for errors and failures is now more detailed and comfortable to read.
+- Fixes printing a success when there's no response object, for instance when catching an expected exception.
+- The code for catching exceptions was refactored and made more reliable, including showing better information as previously mentioned.
+
 ## [0.17.0] - 2026-03-04
 
 - Better debugging detection, check for `ENV['DEBUG']` to be equal to `true`, not just if it's set.

data/lib/elasticsearch/tests/code_runner.rb

@@ -60,11 +60,19 @@ module Elasticsearch
         print_debug_message(method.to_sym, params) if debug?
         @response
       rescue StandardError => e
-        raise e unless expected_exception?(catchable, e)
-
-        puts "Catchable: #{e}\nResponse: #{@response}\n" if debug?
+        # Raise if it's an actual error:
+        unless expected_exception?(catchable, e)
+          print_failure(action, @response)
+          raise e
+        end
+        # Show success if we caught an expected exception:
+        print_success
+        print_debug_catchable(e) if debug?
       end
 
+      # The keyword `catch` is used in the yaml tests to catch exceptions. This code looks at the
+      # expected error and the response error to compare them and check that we got the expected
+      # exception.
       def expected_exception?(error_type, e)
         return false if error_type.nil?
 
@@ -86,14 +94,14 @@ module Elasticsearch
         when 'forbidden'
           e.is_a?(Elastic::Transport::Transport::Errors::Forbidden)
         when /error parsing field/, /illegal_argument_exception/
-          e.message =~ /\[400\]/ ||
+          e.message.match?(/\[400\]/) ||
             e.is_a?(Elastic::Transport::Transport::Errors::BadRequest)
         when /NullPointerException/
-          e.message =~ /\[400\]/
+          e.message.match?(/\[400\]/)
         when /status_exception/
-          e.message =~ /\[409\]/
+          e.message.match?(/\[409\]/)
         else
-          e.message =~ /#{error_type}/
+          e.message.match?(/#{error_type}/)
         end
       end
 
@@ -112,6 +120,8 @@ module Elasticsearch
           print_success
         else
           print_match_failure(action)
+          message = "Match failure\nExpected #{k}: #{v}.\nResult: #{result}"
+          raise ActionError.new(message, $test_file, action)
         end
       end
 
@@ -129,6 +139,8 @@ module Elasticsearch
           print_success
         else
           print_failure(action, @response)
+          message = "do_length failure\nExpected #{k}: #{v}.\nResult: #{result}"
+          raise ActionError.new(message, $test_file, action)
         end
       end
 
@@ -147,6 +159,8 @@ module Elasticsearch
           print_success
         else
           print_failure(action, @response)
+          message = "is_true failure\nResponse: #{response_value || @response}."
+          raise ActionError.new(message, $test_file, action)
         end
       end
 
@@ -156,6 +170,8 @@ module Elasticsearch
           print_success
         else
           print_failure(action, @response)
+          message = "is_false failure\nResponse: #{response_value || @response}."
+          raise ActionError.new(message, $test_file, action)
         end
       end
 
@@ -166,11 +182,14 @@ module Elasticsearch
       #
       def compare(action)
         operator, value = action.first
+
         result = search_in_response(value.keys.first)
         if result&.send(COMPARATORS[operator], value[value.keys.first])
           print_success
         else
           print_failure(action, @response)
+          message = "comparision failure\n#{result} #{COMPARATORS[operator]} #{value[value.keys.first]}"
+          raise ActionError.new(message, $test_file, action)
         end
       end
 

data/lib/elasticsearch/tests/printer.rb

@@ -30,8 +30,13 @@ module Elasticsearch
         if quiet?
           print '🟢 '
         else
-          status = boolean_response? ? @response : @response.status
-          puts "🟢 \e[33m#{@short_name}\e[0m - #{@action}  \e[32mpassed\e[0m [#{status}]"
+          msg = "🟢 \e[33m#{@short_name}\e[0m - #{@action} \e[32mpassed\e[0m"
+          if @response.nil?
+            puts msg
+          else
+            status = boolean_response? ? @response : @response.status
+            puts "#{msg} [#{status}]"
+          end
         end
       end
 
@@ -46,23 +51,24 @@ module Elasticsearch
           puts "🔴 \e[33m#{@short_name}\e[0m - #{@action}  \e[31mfailed\e[0m"
         end
         message = ["Expected result: #{action}"]
-        if defined?(ElasticsearchServerless) &&
-           response.is_a?(ElasticsearchServerless::API::Response) ||
-           defined?(Elasticsearch::API) && response.is_a?(Elasticsearch::API::Response)
-          message << 'Response:'
+        if response&.body
           message << response.body
         else
           message << response
         end
-        raise Elasticsearch::Tests::ActionError.new(response.body, @short_name, action)
       end
 
       def print_match_failure(action)
         keys = action['match'].keys.first
         value = action['match'].values.first
 
+        if quiet?
+          print '🔴 '
+        else
+          puts "🔴 \e[33m#{@short_name} (match)\e[0m - \e[32;9m#{keys}: #{value}\e[0m  \e[95m#{keys}: #{search_in_response(action['match'].keys.first)}\e[0m \e[31mfailed\e[0m"
+        end
         message = <<~MSG
-          🔴 #{@short_name} #{@title} failed
+          #{@short_name} #{@title} failed
           Expected: { #{keys}: #{value} }
           Actual  : { #{keys}: #{search_in_response(action['match'].keys.first)} }
           Response: #{@response}
@@ -79,14 +85,16 @@ module Elasticsearch
       end
 
       def self.display_errors(errors, logger)
+        puts
         print TTY::Box.frame("❌ Errors/Failures: #{errors.count}", width: BOX_WIDTH, style: { border: { fg: :red } })
-        errors.map do |error|
+
+        errors.each do |error|
+          print TTY::Box.frame("🧪 Test: #{error[:file]}", width: BOX_WIDTH * 0.6, style: { border: { fg: :red } })
           message = []
-          message << "🧪 Test: #{error[:file]}"
           message << "▶ Action: #{error[:action].first}" if error[:action]
-          message << "🔬 #{error.class} - #{error[:error].message}"
+          message << "🔬 #{error[:error].class} - #{error[:error].message}"
           message << error[:error].backtrace.join("$/\n") if ENV['DEBUG'] == 'true'
-          print TTY::Box.frame(message.join("\n"), width: BOX_WIDTH, style: { border: { fg: :red } })
+          puts message.join("\n")
           logger.error(message.join("\n"))
         end
       end
@@ -99,6 +107,7 @@ module Elasticsearch
                      #{summary}
                      #{duration}
         MSG
+        puts
         print TTY::Box.frame(message, width: BOX_WIDTH, title: { top_left: '[SUMMARY]' }, style: { border: { fg: :cyan } })
         logger.info duration
       end
@@ -132,6 +141,14 @@ module Elasticsearch
         end
       end
 
+      def print_debug_catchable(exception)
+        puts TTY::Box.frame(
+               "Catchable: #{exception}\nResponse: #{@response}\n",
+               width: BOX_WIDTH,
+               title: { top_left: '[DEBUG]' }
+             )
+      end
+
       private
 
       def boolean_response?

data/lib/elasticsearch/tests/test_runner.rb

@@ -86,8 +86,6 @@ module Elasticsearch
         test = Elasticsearch::Tests::Test.new(yaml, test_path, @client)
         test.execute
         @tests_count += test.count
-      rescue StandardError => e
-        raise e
       end
 
       def test_filename(file)

data/lib/elasticsearch/tests/version.rb

@@ -19,6 +19,6 @@
 
 module Elasticsearch
   module Tests
-    VERSION = '0.17.0'
+    VERSION = '0.18.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-test-runner
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.18.0
 platform: ruby
 authors:
 - Elastic Client Library Maintainers