bugsnag-maze-runner 9.13.0 → 9.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0faaa4d935a7db46b9c6bd717fb7a5679e30c4add8e9a8010ae33708ee9a9cca
-  data.tar.gz: afee3aba6ca6e9e6a917bd8c243c4e118426bfa568aa5f97d9b8d2f175c2568c
+  metadata.gz: eb842b24909ce9e347d04a2c37cdd108f22a5261faa41566188cb7eae9b7f8a0
+  data.tar.gz: 9d17559ba0c60a8707629dca8d9bf43a9d599a7f314750451c5a75beead78f5b
 SHA512:
-  metadata.gz: 680e8b3f617e71d67fd3a8cf5fa24303cbbfcb0ea853b63bb306d133a3c7bdcc323da256892b5dcd8367d8440f4000d393e74353591831ab26b3c3f565c2e265
-  data.tar.gz: 827d8aa95c7dc4b18ef253ae96e24e3ad7f3b0e61c0175474c86c35fa60ba9d47872e19fecfbb8167259c5b874349e5c1f470bd2c14e7aeebb0ac08e973153c0
+  metadata.gz: a44150cac8caac4e3e555857d7eb46749e86e56015b8a8d9404af4281b953b86c7dd55b5b54aabb8216ef2d993e91a1285f617986f75565bb486a18b376895f2
+  data.tar.gz: 04a683fc786c1ac1d0bebc4b0422002423545d9af567693eaa5d0e96a3b760952ed34f5ac9da7da3cb17d096c508dc236b657e16b5198db5fd2a34a6afcc6c15

data/bin/maze-runner

@@ -65,8 +65,6 @@ require_relative '../lib/maze/server'
 
 require_relative '../lib/maze/assertions/request_set_assertions'
 
-require_relative '../lib/maze/schemas/trace_schema'
-require_relative '../lib/maze/schemas/trace_validator'
 require_relative '../lib/maze/schemas/validator'
 
 require_relative '../lib/maze/store'
@@ -103,6 +101,25 @@ require_relative '../lib/utils/selenium_money_patch'
 # Encapsulates the MazeRunner entry point
 class MazeRunnerEntry
 
+  def read_options_file(filename)
+    return unless File.exist?(filename)
+
+    $logger.info "Reading command line options from #{filename}"
+    File.readlines(filename).each do |line|
+      line.strip!
+      next if line.empty? || line.start_with?('#')
+      @args << line
+    end
+  end
+
+  # Loads options from config files
+  def load_options_from_files
+    all_file = File.join('features', 'support', 'maze.all.cfg')
+    buildkite_file = File.join('features', 'support', 'maze.buildkite.cfg')
+    read_options_file(all_file)
+    read_options_file(buildkite_file) if ENV['BUILDKITE']
+  end
+
   # Removes Maze Runner specific args from the array, as these will cause Cucumber to error.
   def remove_maze_runner_args
     Maze::Option.constants.each do |opt|
@@ -134,6 +151,13 @@ class MazeRunnerEntry
       @args << "--expand"
     end
 
+    # Check if we've set ENV['MAZE_NO_FAIL_FAST'] to override the fail fast behaviour
+    # And remove the --fail-fast option if it's set
+    if ENV['MAZE_NO_FAIL_FAST'] && @args.include?('--fail-fast')
+      @args = @args - ['--fail-fast']
+      $logger.info 'Suppressing --fail-fast option as MAZE_NO_FAIL_FAST is set.'
+    end
+
     # Load internal steps and helper functions
     load_dir = File.expand_path(File.dirname(File.dirname(__FILE__))).freeze
     paths = Dir.glob("#{load_dir}/lib/features/**/*.rb")
@@ -187,6 +211,7 @@ class MazeRunnerEntry
     Maze::Option::Processor.populate Maze.config, options
 
     # Adjust CL options before calling down to Cucumber
+    load_options_from_files
     remove_maze_runner_args
     add_cucumber_args
 

data/lib/features/steps/trace_steps.rb

@@ -317,7 +317,6 @@ def assert_received_spans(list, min_received, max_received = nil)
 
   Maze.check.operator(max_received, :>=, received_count, "#{received_count} spans received") if max_received
 
-  Maze::Schemas::Validator.verify_against_schema(list, 'trace')
   Maze::Schemas::Validator.validate_payload_elements(list, 'trace')
 end
 

data/lib/features/support/internal_hooks.rb

@@ -233,7 +233,6 @@ end
 # specified for the specific endpoint
 After do |scenario|
   ['error', 'session', 'build', 'trace'].each do |endpoint|
-    Maze::Schemas::Validator.verify_against_schema(Maze::Server.list_for(endpoint), endpoint)
     Maze::Schemas::Validator.validate_payload_elements(Maze::Server.list_for(endpoint), endpoint)
   end
 end

data/lib/maze/configuration.rb

@@ -14,6 +14,7 @@ module Maze
       self.android_app_files_directory = nil
       self.span_timestamp_validation = true
       self.unmanaged_traces_mode = false
+      self.client_mode_validation = true
       @legacy_driver = false
     end
 
@@ -83,6 +84,33 @@ module Maze
     # Enables unmanaged trace mode.
     attr_accessor :unmanaged_traces_mode
 
+    # Custom validators to use for a given endpoint
+    attr_reader :custom_validators
+
+    # Consumes a block that will be triggered when a request is received for a specific endpoint
+    # This will prevent any existing default validation from triggering.
+    #
+    # @param endpoint [String] The endpoint to validate
+    # @param validator [Proc] The block to run when a request is received
+    def add_validator(endpoint, &validator)
+      @custom_validators ||= {}
+      @custom_validators[endpoint] = validator
+    end
+
+    # Whether default validation should be skipped for a given endpoint
+    attr_reader :skipped_validators
+
+    # Sets whether to skip default validation for a given endpoint
+    #
+    # @param endpoint [String] The endpoint to skip default validation for
+    def skip_default_validation(endpoint)
+      @skipped_validators ||= {}
+      @skipped_validators[endpoint] = true
+    end
+
+    # Sets whether validation should be run in client mode
+    attr_accessor :client_mode_validation
+
     #
     # General appium configuration
     #

data/lib/maze/repeaters/bugsnag_repeater.rb

@@ -20,7 +20,7 @@ module Maze
         url = case @request_type
               when :errors then 'https://notify.bugsnag.com/'
               when :sessions then 'https://sessions.bugsnag.com/'
-              when :traces then 'https://otlp.bugsnag.com/v1/traces'
+              when :traces then "https://#{Maze.config.bugsnag_repeater_api_key}.otlp.bugsnag.com/v1/traces"
               else return nil
               end
         URI.parse(url)

data/lib/maze/schemas/config_validator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../helper'
+require_relative 'validator_base'
+
+module Maze
+  module Schemas
+    class ConfigValidator < ValidatorBase
+
+      attr_accessor :success
+      attr_accessor :errors
+      attr_reader :headers
+      attr_reader :body
+
+      def initialize(request, validation_block)
+        super(request)
+        @validation_block = validation_block
+      end
+
+      def validate
+        @success = true
+        @validation_block.call(self)
+      rescue => exception
+        @success = false
+        @errors << "A #{exception.class} occurred while running validation: #{exception.message}, \n #{exception.backtrace.join("\n")}" 
+
+      end
+    end
+  end
+end

data/lib/maze/schemas/error_validator.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative '../helper'
+require_relative 'validator_base'
+
+module Maze
+  module Schemas
+
+    # Contains a set of pre-defined validations for ensuring errors are correct
+    class ErrorValidator < ValidatorBase
+    end
+  end
+end

data/lib/maze/schemas/trace_validator.rb

@@ -1,84 +1,50 @@
 # frozen_string_literal: true
 
 require_relative '../helper'
+require_relative 'trace_schema'
+require_relative 'validator_base'
 
 module Maze
   module Schemas
 
-    HEX_STRING_16 = '^[A-Fa-f0-9]{16}$'
-    HEX_STRING_32 = '^[A-Fa-f0-9]{32}$'
     SAMPLING_HEADER_ENTRY = '((1(.0)?|0(\.[0-9]+)?):[0-9]+)'
     SAMPLING_HEADER = "^#{SAMPLING_HEADER_ENTRY}(;#{SAMPLING_HEADER_ENTRY})*$"
-    HOUR_TOLERANCE = 60 * 60 * 1000 * 1000 * 1000 # 1 hour in nanoseconds
 
     # Contains a set of pre-defined validations for ensuring traces are correct
-    class TraceValidator
-
-      # Whether the trace passed the validation, one of true, false, or nil (not run)
-      #   @returns [Boolean|nil] Whether the validation was successful
-      attr_reader :success
-      attr_reader :errors
-
-      # Creates the validator
-      #
-      #   @param request [Hash] The trace request to validate
-      def initialize(request)
-        @headers = request[:request].header
-        @body = request[:body]
-        @success = nil
-        @errors = []
-      end
-
+    class TraceValidator < ValidatorBase
       # Runs the validation against the trace given
       def validate
+        # The tests are being run
         @success = true
-
+        verify_against_schema
         validate_headers
         regex_comparison('resourceSpans.0.scopeSpans.0.spans.0.spanId', HEX_STRING_16)
         regex_comparison('resourceSpans.0.scopeSpans.0.spans.0.traceId', HEX_STRING_32)
         element_int_in_range('resourceSpans.0.scopeSpans.0.spans.0.kind', 0..5)
         regex_comparison('resourceSpans.0.scopeSpans.0.spans.0.startTimeUnixNano', '^[0-9]+$')
         regex_comparison('resourceSpans.0.scopeSpans.0.spans.0.endTimeUnixNano', '^[0-9]+$')
-        element_contains('resourceSpans.0.resource.attributes', 'device.id')
-        each_element_contains('resourceSpans.0.scopeSpans.0.spans', 'attributes', 'bugsnag.sampling.p')
-        element_contains('resourceSpans.0.resource.attributes', 'deployment.environment')
-        element_contains('resourceSpans.0.resource.attributes', 'telemetry.sdk.name')
-        element_contains('resourceSpans.0.resource.attributes', 'telemetry.sdk.version')
+        each_span_element_contains('resourceSpans.0.scopeSpans.0.spans', 'attributes', 'bugsnag.sampling.p')
+        span_element_contains('resourceSpans.0.resource.attributes', 'deployment.environment')
+        span_element_contains('resourceSpans.0.resource.attributes', 'telemetry.sdk.name')
+        span_element_contains('resourceSpans.0.resource.attributes', 'telemetry.sdk.version')
         validate_timestamp('resourceSpans.0.scopeSpans.0.spans.0.startTimeUnixNano', HOUR_TOLERANCE)
         validate_timestamp('resourceSpans.0.scopeSpans.0.spans.0.endTimeUnixNano', HOUR_TOLERANCE)
         element_a_greater_or_equal_element_b(
           'resourceSpans.0.scopeSpans.0.spans.0.endTimeUnixNano',
           'resourceSpans.0.scopeSpans.0.spans.0.startTimeUnixNano'
         )
-      end
 
-      def validate_timestamp(path, tolerance)
-        return unless Maze.config.span_timestamp_validation
-        timestamp = Maze::Helper.read_key_path(@body, path)
-        unless timestamp.kind_of?(String)
-          @success = false
-          @errors << "Timestamp was expected to be a string, was '#{timestamp.class.name}'"
-          return
-        end
-        parsed_timestamp = timestamp.to_i
-        unless parsed_timestamp > 0
-          @success = false
-          @errors << "Timestamp was expected to be a positive integer, was '#{parsed_timestamp}'"
-          return
-        end
-        time_in_nanos = Time.now.to_i * 1000000000
-        unless (time_in_nanos - parsed_timestamp).abs < tolerance
-          @success = false
-          @errors << "Timestamp was expected to be within #{tolerance} nanoseconds of the current time (#{time_in_nanos}), was '#{parsed_timestamp}'"
+        if Maze.config.client_mode_validation
+          span_element_contains('resourceSpans.0.resource.attributes', 'device.id')
         end
       end
 
-      def validate_header(name)
-        value = @headers[name]
-        if value.nil? || value.size > 1
-          @errors << "Expected exactly one value for header #{name}, received #{value || 'nil'}"
-        else
-          yield value[0]
+      def verify_against_schema
+        if !@schema_errors.nil? && @schema_errors.size > 0
+          @success = false
+          @schema_errors.each do |error|
+            @errors << "#{JSONSchemer::Errors.pretty(error)}"
+          end
         end
       end
 
@@ -118,29 +84,7 @@ module Maze
         end
       end
 
-      def regex_comparison(path, regex)
-        element_value = Maze::Helper.read_key_path(@body, path)
-        expected = Regexp.new(regex)
-        unless expected.match(element_value)
-          @success = false
-          @errors << "Element '#{path}' was expected to match the regex '#{regex}', but was '#{element_value}'"
-        end
-      end
-
-      def element_int_in_range(path, range)
-        element_value = Maze::Helper.read_key_path(@body, path)
-        if element_value.nil? || !element_value.kind_of?(Integer)
-          @success = false
-          @errors << "Element '#{path}' was expected to be an integer, was '#{element_value}'"
-          return
-        end
-        unless range.include?(element_value)
-          @success = false
-          @errors << "Element '#{path}':'#{element_value}' was expected to be in the range '#{range}'"
-        end
-      end
-
-      def element_contains(path, key_value, value_type=nil, possible_values=nil)
+      def span_element_contains(path, key_value, value_type=nil, possible_values=nil)
         container = Maze::Helper.read_key_path(@body, path)
         if container.nil? || !container.kind_of?(Array)
           @success = false
@@ -161,7 +105,7 @@ module Maze
         end
       end
 
-      def each_element_contains(container_path, attribute_path, key_value)
+      def each_span_element_contains(container_path, attribute_path, key_value)
         container = Maze::Helper.read_key_path(@body, container_path)
         if container.nil? || !container.kind_of?(Array)
           @success = false
@@ -169,16 +113,7 @@ module Maze
           return
         end
         container.each_with_index do |_item, index|
-          element_contains("#{container_path}.#{index}.#{attribute_path}", key_value)
-        end
-      end
-
-      def element_a_greater_or_equal_element_b(path_a, path_b)
-        element_a = Maze::Helper.read_key_path(@body, path_a)
-        element_b = Maze::Helper.read_key_path(@body, path_b)
-        unless element_a && element_b && element_a >= element_b
-          @success = false
-          @errors << "Element '#{path_a}':'#{element_a}' was expected to be greater than or equal to '#{path_b}':'#{element_b}'"
+          span_element_contains("#{container_path}.#{index}.#{attribute_path}", key_value)
         end
       end
     end

data/lib/maze/schemas/validator.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+require_relative 'config_validator'
+require_relative 'error_validator'
+require_relative 'trace_validator'
+
 module Maze
   module Schemas
 
@@ -7,56 +11,42 @@ module Maze
     class Validator
 
       class << self
-
-        # Tests that payloads for a specific path have passed any schema checks implemented on receipt
-        # Throws an AssertionFailedError with a list of issues on failure
-        #
-        # @param list [Array] An array of received requests
-        # @param list_name [String] The name of the payload list for received requests
-        def verify_against_schema(list, list_name)
-          request_schema_results = list.all.map { |request| request[:schema_errors] }
-          passed = true
-          request_schema_results.each.with_index(1) do |schema_errors, index|
-            next if schema_errors.nil?
-            if schema_errors.size > 0
-              passed = false
-              $stdout.puts "\n"
-              $stdout.puts "\e[31m--- #{list_name} #{index} failed validation:\e[0m"
-              schema_errors.each do |error|
-                $stdout.puts "\e[31m#{JSONSchemer::Errors.pretty(error)}\e[0m"
-              end
-              $stdout.puts "\n"
-            end
-          end
-
-          unless passed
-            raise Test::Unit::AssertionFailedError.new 'The received payloads did not match the endpoint schema.  A full list of the errors can be found above'
-          end
-        end
-
         # Tests that payloads for a specific path pass any additional validation checks
         # Throws an AssertionFailedError with a list of issues on failure
         #
         # @param list [Array] An array of received requests
         # @param list_name [String] The name of the payload list for received requests
         def validate_payload_elements(list, list_name)
-          validator_class = case list_name
-          when 'trace', 'traces'
-            Maze::Schemas::TraceValidator
+          # Test to see if a custom validator exists for the list
+          custom_validator = Maze.config.custom_validators&.key?(list_name)
+
+          if Maze.config.skipped_validators && Maze.config.skipped_validators[list_name]
+            validator_class = false
           else
-            nil
+            validator_class = case list_name
+            when 'trace', 'traces'
+              Maze::Schemas::TraceValidator
+            when 'error', 'errors'
+              Maze::Schemas::ErrorValidator
+            else
+              nil
+            end
           end
 
-          if validator_class
-            validators = list.all.map do |request|
-              validator = validator_class.new(request)
-              validator.validate
-              validator
-            end
+          list_validators = list.all.map do |request|
+            payload_validators = []
+            payload_validators << Maze::Schemas::ConfigValidator.new(request, Maze.config.custom_validators[list_name]) if custom_validator
+            payload_validators << validator_class.new(request) if validator_class
+
+            payload_validators.each { |validator| validator.validate }
+            payload_validators
+          end
 
-            return if validators.all? { |validator| validator.success }
-            validators.each.with_index(1) do |validator, index|
+          failing = false
+          list_validators.each.with_index(1) do |validators, index|
+            validators.each do |validator|
               unless validator.success
+                failing = true
                 $stdout.puts "\n"
                 $stdout.puts "\e[31m--- #{list_name} #{index} failed validation with the following errors:\e[0m"
                 validator.errors.each do |error|
@@ -65,8 +55,8 @@ module Maze
                 $stdout.puts "\n"
               end
             end
-            raise Test::Unit::AssertionFailedError.new("One or more #{list_name} payloads failed validation.  A full list of the errors can be found above")
           end
+          raise Test::Unit::AssertionFailedError.new("One or more #{list_name} payloads failed validation.  A full list of the errors can be found above") if failing
         end
       end
     end

data/lib/maze/schemas/validator_base.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Maze
+  module Schemas
+    class ValidatorBase
+
+      HEX_STRING_16 = '^[A-Fa-f0-9]{16}$'
+      HEX_STRING_32 = '^[A-Fa-f0-9]{32}$'
+      HOUR_TOLERANCE = 60 * 60 * 1000 * 1000 * 1000 # 1 hour in nanoseconds
+
+      # Whether the payloads passed the validation, one of true, false, or nil (not run)
+      #   @returns [Boolean|nil] Whether the validation was successful
+      attr_reader :success
+
+      # An array of error messages if the validation failed
+      #  @returns [Array] The error messages
+      attr_reader :errors
+
+      # Creates the validator
+      #
+      #   @param request [Hash] The trace request to validate
+      def initialize(request)
+        @headers = request[:request].header
+        @body = request[:body]
+        @schema_errors = request[:schema_errors]
+        @success = nil
+        @errors = []
+      end
+
+      def validate
+        # By default the validation will pass
+        @success = true
+      end
+
+      def element_has_value(path, value)
+        element = Maze::Helper.read_key_path(@body, path)
+        if element.nil? || element != value
+          @success = false
+          @errors << "Element '#{path}' was expected to be '#{value}', was '#{element}'"
+        end
+      end
+
+      def element_exists(path)
+        element = Maze::Helper.read_key_path(@body, path)
+        if element.nil?
+          @success = false
+          @errors << "Element '#{path}' was not found"
+        end
+      end
+
+      def each_element_exists(paths)
+        if paths.kind_of?(Array)
+          paths.each {|path| element_exists(path)}
+        else
+          $logger.warn("each_element_exists was called with a non-array value: '#{paths}'. Use element_exists instead.")
+          element_exists(paths)
+        end
+
+      end
+
+      def each_element_contains(container_path, path)
+        containers = Maze::Helper.read_key_path(@body, container_path)
+        containers.each_with_index do |container, index|
+          element = Maze::Helper.read_key_path(container, path)
+          if element.nil?
+            @success = false
+            @errors << "Required #{container_path} element #{path} was not present at index #{index}"
+          end
+        end
+      end
+
+      def each_event_contains(path)
+        each_element_contains('events', path)
+      end
+
+      def each_element_contains_each(container_path, paths)
+        paths.each { |path| each_element_contains(container_path, path) }
+      end
+
+      def each_event_contains_each(paths)
+        paths.each { |path| each_event_contains(path) }
+      end
+
+      def regex_comparison(path, regex)
+        element_value = Maze::Helper.read_key_path(@body, path)
+        expected = Regexp.new(regex)
+        unless expected.match(element_value)
+          @success = false
+          @errors << "Element '#{path}' was expected to match the regex '#{regex}', but was '#{element_value}'"
+        end
+      end
+
+      def element_int_in_range(path, range)
+        element_value = Maze::Helper.read_key_path(@body, path)
+        if element_value.nil? || !element_value.kind_of?(Integer)
+          @success = false
+          @errors << "Element '#{path}' was expected to be an integer, was '#{element_value}'"
+          return
+        end
+        unless range.include?(element_value)
+          @success = false
+          @errors << "Element '#{path}':'#{element_value}' was expected to be in the range '#{range}'"
+        end
+      end
+
+      def element_a_greater_or_equal_element_b(path_a, path_b)
+        element_a = Maze::Helper.read_key_path(@body, path_a)
+        element_b = Maze::Helper.read_key_path(@body, path_b)
+        unless element_a && element_b && element_a >= element_b
+          @success = false
+          @errors << "Element '#{path_a}':'#{element_a}' was expected to be greater than or equal to '#{path_b}':'#{element_b}'"
+        end
+      end
+
+      def validate_timestamp(path, tolerance)
+        return unless Maze.config.span_timestamp_validation
+        timestamp = Maze::Helper.read_key_path(@body, path)
+        unless timestamp.kind_of?(String)
+          @success = false
+          @errors << "Timestamp was expected to be a string, was '#{timestamp.class.name}'"
+          return
+        end
+        parsed_timestamp = timestamp.to_i
+        unless parsed_timestamp > 0
+          @success = false
+          @errors << "Timestamp was expected to be a positive integer, was '#{parsed_timestamp}'"
+          return
+        end
+        time_in_nanos = Time.now.to_i * 1000000000
+        unless (time_in_nanos - parsed_timestamp).abs < tolerance
+          @success = false
+          @errors << "Timestamp was expected to be within #{tolerance} nanoseconds of the current time (#{time_in_nanos}), was '#{parsed_timestamp}'"
+        end
+      end
+
+      def validate_header(name)
+        begin
+          value = @headers[name]
+          if value.nil? || value.size > 1
+            @success = false
+            @errors << "Expected exactly one value for header #{name}, received #{value || 'nil'}"
+          else
+            yield value[0]
+          end
+        rescue => e
+          @success = false
+          @errors << "Error validating header #{name} with value #{value}: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/maze.rb

@@ -7,7 +7,7 @@ require_relative 'maze/timers'
 # Glues the various parts of MazeRunner together that need to be accessed globally,
 # providing an alternative to the proliferation of global variables or singletons.
 module Maze
-  VERSION = '9.13.0'
+  VERSION = '9.14.0'
 
   class << self
     attr_accessor :check, :driver, :internal_hooks, :mode, :start_time, :dynamic_retry, :public_address,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bugsnag-maze-runner
 version: !ruby/object:Gem::Version
-  version: 9.13.0
+  version: 9.14.0
 platform: ruby
 authors:
 - Steve Kirkland
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-14 00:00:00.000000000 Z
+date: 2024-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cucumber
@@ -440,9 +440,12 @@ files:
 - lib/maze/retry_handler.rb
 - lib/maze/runner.rb
 - lib/maze/schemas/OtelTraceSchema.json
+- lib/maze/schemas/config_validator.rb
+- lib/maze/schemas/error_validator.rb
 - lib/maze/schemas/trace_schema.rb
 - lib/maze/schemas/trace_validator.rb
 - lib/maze/schemas/validator.rb
+- lib/maze/schemas/validator_base.rb
 - lib/maze/server.rb
 - lib/maze/servlets/all_commands_servlet.rb
 - lib/maze/servlets/base_servlet.rb