bugsnag-maze-runner 6.27.0

data/lib/features/support/internal_hooks.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+require 'cucumber'
+require 'fileutils'
+require 'json'
+require 'securerandom'
+require 'selenium-webdriver'
+require 'uri'
+
+BeforeAll do
+
+  Maze.check = Maze::Checks::AssertCheck.new
+
+  # Infer mode of operation from config, one of:
+  # - Appium (using either remote or local devices)
+  # - Browser (Selenium with local or remote browsers)
+  # - Command (the software under test is invoked with a system call)
+  # TODO Consider making this a specific command line option defaulting to Appium
+  is_appium = [:bs, :sl, :bb, :local].include?(Maze.config.farm) && !Maze.config.app.nil?
+  is_browser = !Maze.config.browser.nil?
+  if is_appium
+    Maze.mode = :appium
+    Maze.internal_hooks = Maze::Hooks::AppiumHooks.new
+  elsif is_browser
+    Maze.mode = :browser
+    Maze.internal_hooks = Maze::Hooks::BrowserHooks.new
+  else
+    Maze.mode = :command
+    Maze.internal_hooks = Maze::Hooks::CommandHooks.new
+  end
+  $logger.info "Running in #{Maze.mode.to_s} mode"
+
+  # Clear out maze_output folder
+  maze_output = Dir.glob(File.join(Dir.pwd, 'maze_output', '*'))
+  if Maze.config.file_log && !maze_output.empty?
+    maze_output.each { |path| $logger.info "Clearing contents of #{path}" }
+    FileUtils.rm_rf(maze_output)
+  end
+
+  # Record the local server starting time
+  Maze.start_time = Time.now.strftime('%Y-%m-%d %H:%M:%S')
+
+  # Start document server, if asked for
+  Maze::DocumentServer.start unless Maze.config.document_server_root.nil?
+
+  # Start mock server
+  Maze::Server.start
+
+  # Invoke the internal hook for the mode of operation
+  Maze.internal_hooks.before_all
+
+  # Call any blocks registered by the client
+  Maze.hooks.call_before_all
+end
+
+# @param config The Cucumber config
+InstallPlugin do |config|
+  # Start Bugsnag
+  Maze::BugsnagConfig.start_bugsnag(config)
+
+  # Only add the retry plugin if --retry is not used on the command line
+  config.filters << Maze::Plugins::GlobalRetryPlugin.new(config) if config.options[:retry].zero?
+  config.filters << Maze::Plugins::BugsnagReportingPlugin.new(config)
+  cucumber_report_plugin = Maze::Plugins::CucumberReportPlugin.new
+  cucumber_report_plugin.install_plugin(config)
+end
+
+# Before each scenario
+Before do |scenario|
+  # Default to no dynamic try
+  Maze.dynamic_retry = false
+
+  if ENV['BUILDKITE']
+    location = "\e[90m\t# #{scenario.location}\e[0m"
+    $stdout.puts "--- Scenario: #{scenario.name} #{location}"
+  end
+
+  # Invoke the internal hook for the mode of operation
+  Maze.internal_hooks.before
+
+  # Call any blocks registered by the client
+  Maze.hooks.call_before scenario
+end
+
+# General processing to be run after each scenario
+After do |scenario|
+  # If we're running on macos, take a screenshot if the scenario fails
+  if Maze.config.os == "macos" && scenario.status == :failed
+    Maze::MacosUtils.capture_screen(scenario)
+  end
+
+  # Call any blocks registered by the client
+  Maze.hooks.call_after scenario
+
+  # Make sure we reset to HTTP 200 return status after each scenario
+  Maze::Server.status_code = 200
+  Maze::Server.reset_status_code = false
+
+  # Similarly for the response delay
+  Maze::Server.response_delay_ms = 0
+  Maze::Server.reset_response_delay = false
+
+  # Stop document server if started by the Cucumber step
+  Maze::DocumentServer.manual_stop
+
+  # Stop terminating server if started by the Cucumber step
+  Maze::TerminatingServer.stop
+
+  # This is here to stop sessions from one test hitting another.
+  # However this does mean that tests take longer.
+  # In addition, reset the last captured exit code
+  # TODO:SM We could try and fix this by generating unique endpoints
+  # for each test.
+  Maze::Docker.reset
+
+  # Make sure that any scripts are killed between test runs
+  # so future tests are run from a clean slate.
+  Maze::Runner.kill_running_scripts
+
+  Maze::Proxy.instance.stop
+
+  # Log unprocessed requests on Buildkite if the scenario fails
+  if (scenario.failed? && Maze.config.log_requests) || Maze.config.always_log
+    $stdout.puts '^^^ +++'
+    output_received_requests('errors')
+    output_received_requests('sessions')
+    output_received_requests('builds')
+    output_received_requests('logs')
+    output_received_requests('invalid requests')
+  end
+
+  # Log all received requests to file
+  write_requests(scenario) if Maze.config.file_log
+
+  # Invoke the internal hook for the mode of operation
+  Maze.internal_hooks.after
+
+ensure
+  # Request arrays in particular are cleared here, rather than in the Before hook, to allow requests to be registered
+  # when a test fixture starts (which can be before the first Before scenario hook fires).
+  Maze::Server.errors.clear
+  Maze::Server.sessions.clear
+  Maze::Server.builds.clear
+  Maze::Server.uploads.clear
+  Maze::Server.sourcemaps.clear
+  Maze::Server.logs.clear
+  Maze::Server.invalid_requests.clear
+  Maze::Runner.environment.clear
+  Maze::Store.values.clear
+  Maze::Aws::Sam.reset!
+end
+
+def output_received_requests(request_type)
+  request_queue = Maze::Server.list_for(request_type)
+  if request_queue.empty?
+    $logger.info "No #{request_type} received"
+  else
+    count = request_queue.size_all
+    $logger.info "#{count} #{request_type} were received:"
+    request_queue.all.each.with_index(1) do |request, number|
+      $stdout.puts "--- #{request_type} #{number} of #{count}"
+      Maze::LogUtil.log_hash(Logger::Severity::INFO, request)
+    end
+  end
+end
+
+# Writes each list of requests to a separate file under, e.g:
+# maze_output/failed/scenario_name/errors.log
+def write_requests(scenario)
+  folder1 = File.join(Dir.pwd, 'maze_output')
+  folder2 = scenario.failed? ? 'failed' : 'passed'
+  folder3 = Maze::Helper.to_friendly_filename(scenario.name)
+
+  path = File.join(folder1, folder2, folder3)
+
+  FileUtils.makedirs(path)
+
+  request_types = %w[errors sessions builds uploads logs sourcemaps invalid]
+
+  request_types.each do |request_type|
+    list = Maze::Server.list_for(request_type).all
+    next if list.empty?
+
+    filename = "#{request_type}.log"
+    filepath = File.join(path, filename)
+
+    counter = 1
+    File.open(filepath, 'w+') do |file|
+      list.each do |request|
+        file.puts "=== Request #{counter} of #{list.size} ==="
+        if request[:invalid]
+          invalid_request = true
+          uri = request[:request][:request_uri]
+          headers = request[:request][:header]
+          body = request[:request][:body]
+        else
+          invalid_request = false
+          uri = request[:request].request_uri
+          headers = request[:request].header
+          body = request[:body]
+        end
+        file.puts "URI: #{uri}"
+        file.puts "HEADERS:"
+        headers.each do |key, values|
+          file.puts "  #{key}: #{values.map {|v| "'#{v}'"}.join(' ')}"
+        end
+        file.puts
+        file.puts "BODY:"
+        if !invalid_request && headers["content-type"].first == 'application/json'
+          file.puts JSON.pretty_generate(body)
+        else
+          file.puts body
+        end
+        file.puts
+        if request.include?(:reason)
+          file.puts "REASON:"
+          file.puts request[:reason]
+          file.puts
+        end
+        counter += 1
+      end
+    end
+  end
+end
+
+# Check for invalid requests after each scenario.  This is its own hook as failing a scenario raises an exception
+# and we need the logic in the other After hook to be performed.
+# Furthermore, this hook should appear after the general hook as they are executed in reverse order by Cucumber.
+After do |scenario|
+  unless Maze::Server.invalid_requests.empty?
+    msg = "#{Maze::Server.invalid_requests.size_all} invalid request(s) received during scenario"
+    scenario.fail msg
+  end
+end
+
+# After all tests
+AfterAll do
+
+  if Maze.timers.size.positive?
+    $stdout.puts '--- Timer summary'
+    Maze.timers.report
+  end
+
+  $stdout.puts '+++ All scenarios complete'
+
+  # Stop the mock server
+  Maze::Server.stop
+
+  # In order to not impact future test runs, we down
+  # all services (which removes networks etc) so that
+  # future test runs are from a clean slate.
+  Maze::Docker.down_all_services
+
+  # Invoke the internal hook for the mode of operation
+  Maze.internal_hooks.after_all
+end
+
+at_exit do
+  Maze.internal_hooks.at_exit
+end

data/lib/maze/appium_server.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'pty'
+require 'logger'
+
+module Maze
+  # Basic shell that runs an Appium server on a separate thread
+  class AppiumServer
+    class << self
+      # @return [string|nil] The PID of the appium process (if available)
+      attr_reader :pid
+
+      # @return [thread|nil] The thread running the appium process (if available)
+      attr_reader :appium_thread
+
+      # @return [Logger|nil] The logger used for creating the log file
+      attr_reader :appium_logger
+
+      # Starts a separate thread running the appium server so long as:
+      #   - An instance of the appium server isn't already running
+      #   - The port configured is available
+      #   - The appium command is available via CLI
+      #
+      # @param address [String] The IP address on which to start the appium server
+      # @param port [String] The port on which to start the appium server
+      def start(address: '0.0.0.0', port: '4723')
+        return if running
+
+        # Check if the appium server appears to be running already, warning and carrying on if so
+        unless appium_port_available?(port)
+          $logger.warn "Requested appium port:#{port} is in use. Aborting built-in appium server launch"
+          return
+        end
+
+        # Check if appium is installed, warning if not
+        unless appium_available?
+          $logger.warn 'Appium is unavailable to be started from the command line. Install using `npm i -g appium`'
+          return
+        end
+
+        start_logger
+
+        command = "appium -a #{address} -p #{port}"
+        @appium_thread = Thread.new do
+          PTY.spawn(command) do |stdout, _stdin, pid|
+            @pid = pid
+            $logger.debug("Appium:#{@pid}") { 'Appium server started' }
+            stdout.each do |line|
+              log_line(line)
+            end
+          end
+        end
+
+        # Temporary sleep to allow appium to start
+        sleep 2
+      end
+
+      # Checks whether the server is running, as indicated by the @pid and the appium thread being alive
+      #
+      # @return [Boolean] Whether the local appium server is running
+      def running
+        @appium_thread&.alive? ? true : false
+      end
+
+      # Stops the appium server, if running, using SIGINT for correct shutdown
+      def stop
+        return unless running
+
+        $logger.debug("Appium:#{@pid}") { 'Stopping appium server' }
+        Process.kill('INT', @pid)
+        @pid = nil
+        @appium_thread.join
+        @appium_thread = nil
+      end
+
+      private
+
+      # Checks if the `appium` command is available on CI
+      #
+      # @return [Boolean] Whether the appium command is available
+      def appium_available?
+        `appium -v`
+        true
+      rescue Errno::ENOENT
+        false
+      end
+
+      # Starts the logger targeting a file defined by the APPIUM_LOGFILE config option
+      def start_logger
+        @appium_logger = ::Logger.new(Maze.config.appium_logfile)
+        @appium_logger.datetime_format = '%Y-%m-%d %H:%M:%S'
+      end
+
+      # Logs to a known file, creating the outstream if it isn't already present
+      #
+      # @param line [String] The line to log
+      def log_line(line)
+        return if @appium_logger.nil?
+        @appium_logger.info("Appium:#{@pid}") { line }
+      end
+
+      # Checks if the given port is already in use
+      #
+      # @param port [String] The port that should be available
+      #
+      # @return [Boolean] Whether something is running on the given port
+      def appium_port_available?(port)
+        `netstat -vanp tcp | awk '{ print $4 }' | grep "\.#{port}$"`.empty?
+      end
+    end
+  end
+end

data/lib/maze/assertions/request_set_assertions.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'test/unit'
+require_relative '../helper'
+
+module Maze
+  module Assertions
+    # Provides helper routines for checking sets of requests against values in a table.
+    class RequestSetAssertions
+      class << self
+
+        # Checks that a set of requests satisfy the properties expressed by the table given.
+        #
+        # @param requests [Hash[]] Requests to check
+        # @param table [Cucumber::MultilineArgument::DataTable] Table of expected values, where:
+        #   - headings can be provided as key paths (e.g. events.0.breadcrumbs.0.name)
+        #   - table values can be written as "null" for nil
+        def assert_requests_match(requests, table)
+          Maze.check.equal(table.hashes.length,
+                           requests.length,
+                           'Number of requests do not match number of entries in table.')
+          matches = matching_rows requests, table
+          return if matches.length == table.hashes.length
+
+          # Not all matched - log diagnostic before failing assertion
+          $logger.error "Only #{matches.length} of #{requests.length} matched:"
+          $logger.info matches.keys.sort
+          matches.sort.to_h.each do |row, request|
+            $logger.info "#{table.rows[row]} matched by request element #{matches[request]}"
+          end
+          Maze.check.equal(requests.length, matches.length, 'Not all requests matched a row in the table.')
+        end
+
+        # Given arrays of requests and table-based criteria, determines which rows of the table
+        # are satisfied by one of the requests.  Where multiple rows in the table specify the same
+        # criteria, there must be multiple requests provided to satisfy each.
+        #
+        # @param requests [Hash[]] Requests to check
+        # @param table [Cucumber::MultilineArgument::DataTable] Table of expected values, where:
+        #   - headings can be provided as key paths (e.g. events.0.breadcrumbs.0.name)
+        #   - table values can be written as "null" for nil
+        # @return [Hash] A hash of row to request indexes, indicating the first request matching each row.
+        #   E.g. {0 => 2} means that the first row was satisfied by the 3rd request.
+        def matching_rows(requests, table)
+
+          # iterate through each row in the table. exactly 1 request should match each row.
+          row_to_request_matches = {}
+          table.hashes.each_with_index do |row, row_index|
+            requests.each_with_index do |request, request_index|
+              # Skip if row already matched
+              next if row_to_request_matches.values.include? request_index
+              # Skip if no body in this request
+              next unless request.key?(:body)
+              next unless request_matches_row(request[:body], row)
+
+              # Record the match
+              row_to_request_matches[row_index] = request_index
+            end
+          end
+          row_to_request_matches
+        end
+
+        # Determines if a request body satisfies the criteria expressed by a row.
+        # The special string "null" is interpreted as nil in comparisons and
+        # regular expressions are assumed is the start and end of the string is a '/'.
+        #
+        # @param body [Hash] Request body to consider
+        # @param row [Hash] Hash of keys to expected value, where the keys given can
+        #   be a Mongo-style dot notation path.
+        def request_matches_row(body, row)
+          row.each do |key, expected_value|
+            obs_val = Maze::Helper.read_key_path(body, key)
+            next if ('null'.eql? expected_value) && obs_val.nil? # Both are null/nil
+            next if ('@not_null'.eql? expected_value) && !obs_val.nil? # The value isn't null
+
+            unless obs_val.nil?
+              if expected_value[0] == '/' && expected_value[-1] == '/'
+                # Treat as regexp
+                regex = Regexp.new expected_value[1, expected_value.length - 2]
+                next if regex.match? obs_val.to_s # Value matches regex
+              elsif expected_value.eql? obs_val.to_s
+                # Values match
+                next
+              end
+            end
+
+            # Match not found - return false
+            return false
+          end
+          # All matched - return true
+          true
+        end
+      end
+    end
+  end
+end
+

data/lib/maze/aws/sam.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'shellwords'
+
+module Maze
+  module Aws
+    # Interacts with the AWS SAM CLI to invoke Lambda functions
+    # Note that the SAM CLI must be installed on the host machine as it does not
+    # run in a Docker container! For this reason the "start-api" command is not
+    # supported as it could easily cause port clashes and zombie processes
+    class Sam
+      class << self
+        attr_reader :last_response, :last_exit_code
+
+        # Invoke the given lambda with an optional event
+        #
+        # This happens synchronously so there is no need to wait for a response
+        #
+        # @param directory [String] The directory containing the lambda
+        # @param lambda [String] The name of the lambda to invoke
+        # @param event [String, nil] An optional event file to invoke with
+        #
+        # @return [void]
+        def invoke(directory, lambda, event = nil)
+          command = build_invoke_command(lambda, event)
+
+          output, @last_exit_code = Maze::Runner.run_command("cd #{directory} && #{command}")
+
+          @last_response = parse(output)
+        end
+
+        # Reset the last response and last exit code
+        #
+        # @return [void]
+        def reset!
+          @last_response = nil
+          @last_exit_code = nil
+        end
+
+        private
+
+        # Build the command to invoke the given lambda with the given event
+        #
+        # @param lambda [String] The name of the lambda to invoke
+        # @param event [String, nil] An optional event file to invoke with
+        #
+        # @return [String]
+        def build_invoke_command(lambda, event)
+          command = "sam local invoke #{Shellwords.escape(lambda)}"
+          command += " --event #{Shellwords.escape(event)}" unless event.nil?
+          command += " --docker-network #{Shellwords.escape(ENV['NETWORK_NAME'])}" if ENV.key?('NETWORK_NAME')
+
+          command
+        end
+
+        # The command output contains all stdout/stderr lines in an array. The
+        # Lambda response is the last line of output as JSON. The response body is
+        # also JSON, so we have to parse twice to get a Hash from the output
+        #
+        # @param output [Array<String>] The command's output as an array of lines
+        #
+        # @return [Hash]
+        def parse(output)
+          unless valid?(output)
+            raise <<~ERROR
+              Unable to parse Lambda output!
+              The likely cause is:
+                > #{output.last.chomp}
+
+              Full output:
+                > #{output.map(&:chomp).join("\n  > ")}
+            ERROR
+          end
+
+          # Attempt to parse the last line of output as this is where a JSON
+          # response would be. It's possible for a Lambda to output nothing,
+          # e.g. if it forcefully exited, so we allow JSON parse failures here
+          begin
+            parsed_output = JSON.parse(output.last)
+          rescue JSON::ParserError
+            return {}
+          end
+
+          # Error output has no "body" of additional JSON so we can stop here
+          return parsed_output unless parsed_output.key?('body')
+
+          # The body is _usually_ JSON but doesn't have to be. We attempt to
+          # parse it anyway because it allows us to assert against it easily,
+          # but if this fails then it may just be in another format, e.g. HTML
+          begin
+            parsed_output['body'] = JSON.parse(parsed_output['body'])
+          rescue JSON::ParserError
+            # Ignore
+          end
+
+          parsed_output
+        end
+
+        # Check if the output looks valid. There should be a "END" marker with a
+        # request ID if the lambda invocation completed successfully
+        #
+        # @param output [Array<String>] The command's output as an array of lines
+        #
+        # @return [Boolean]
+        def valid?(output)
+          output.any? { |line| line =~ /^END RequestId:/ }
+        end
+      end
+    end
+  end
+end

data/lib/maze/bitbar_devices.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+require 'json'
+
+module Maze
+  # Provides a source of capabilities used to run tests against specific BitBar devices
+  # noinspection RubyStringKeysInHashInspection
+  class BitBarDevices
+    APPIUM_1_9_1 = '1.9.1'
+    APPIUM_1_15_0 = '1.15.0'
+    APPIUM_1_20_2 = '1.20.2'
+
+    BASE_URI = 'https://cloud.bitbar.com/api/v2/me'
+    FILTER_PATH = 'devices/filters'
+
+    DEVICE_GROUP_IDS = {
+      # Classic, non-specific devices for each Android version
+      'ANDROID_10_0' => '46024',
+
+      # iOS devices
+      'IOS_14' => '46025'
+    }
+
+    class << self
+      def call_bitbar_api(path, query, api_key)
+        encoded_query = URI.encode_www_form(query)
+        uri = URI("#{BASE_URI}/#{path}?#{encoded_query}")
+        request = Net::HTTP::Get.new(uri)
+        request.basic_auth(api_key, '')
+
+        res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+
+        JSON.parse(res.body)
+      end
+
+      def get_filtered_device_name(device_group_id, api_key)
+        path = "device-groups/#{device_group_id}/devices"
+        query = {
+          'filter': "online_eq_true"
+        }
+        all_devices = call_bitbar_api(path, query, api_key)
+        filtered_devices = all_devices['data'].reject { |device| device['locked'] }
+        filtered_devices.first['displayName']
+      end
+
+      def get_device(device_group, platform, platform_version, api_key)
+        device_group_id = DEVICE_GROUP_IDS[device_group]
+        device_name = get_filtered_device_name(device_group_id, api_key)
+        case platform.downcase
+        when 'android'
+          automation_name = 'UiAutomator1' if platform_version.start_with?('5')
+          make_android_hash(device_name, nil, automation_name)
+        when 'ios'
+          make_ios_hash(device_name)
+        else
+          throw "Invalid device platform specified #{platform}"
+        end
+      end
+
+      def make_android_hash(device, appium_version = nil, automation_name = nil)
+        hash = {
+          'platformName' => 'Android',
+          'bitbar_device' => device,
+          'bitbar_target' => 'android',
+          'deviceName' => 'Android Phone'
+        }
+        hash['bitbar_appiumVersion'] = appium_version if appium_version
+        hash['automationName'] = automation_name if automation_name
+        hash.freeze
+      end
+
+      def make_ios_hash(device)
+        {
+          'platformName' => 'iOS',
+          'bitbar_device' => device,
+          'bitbar_target' => 'ios',
+          'deviceName' => 'iPhone device',
+          'automationName' => 'XCUITest'
+        }.freeze
+      end
+    end
+  end
+end