fastlane-plugin-rescan_flaky_tests 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d181fd6674c26468ed557a9247aa56edf05a138c
+  data.tar.gz: 6a009b3b84a005155582c5abeeaa62f5c310b309
+SHA512:
+  metadata.gz: 6f74882b3a2e3fd3757573488b472c3ea4e441a499146ace4da436fd9f89ea72e1d0a2d2b3209b775db04cdcd51330590455aace309223cb971736e898ac022a
+  data.tar.gz: 1a115ffbb8929a12c4131d06329a36c77e564c40def52a23d80e3907a069f0cae664678dee37f3d172fd5c141610c793b9f7ec504012095339ab22caf5571c9a

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Ichiko Moro <ichiko.revjune@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,53 @@
+# rescan_flaky_tests plugin
+
+[![fastlane Plugin Badge](https://rawcdn.githack.com/fastlane/fastlane/master/fastlane/assets/plugin-badge.svg)](https://rubygems.org/gems/fastlane-plugin-rescan_flaky_tests)
+
+
+## Getting Started
+
+This project is a [_fastlane_](https://github.com/fastlane/fastlane) plugin. To get started with `fastlane-plugin-rescan_flaky_tests`, add it to your project by running:
+
+```bash
+fastlane add_plugin rescan_flaky_tests
+```
+
+## About rescan_flaky_tests
+
+Re-run `scan` action for each failed test case.
+
+When you running UI tests (XCUITest, EarlGrey or others), sometime that failed with un clear reason.
+
+This plugin re-scan each failed test cases. And report result under `#{output_directory}/rescan/NN`.
+
+## Example
+
+Check out the [example `Fastfile`](example/SampleDice/fastlane/Fastfile) to see how to use this plugin. Try it by cloning the repo, running `cd example/SampleDice; fastlane install_plugins` and `bundle exec fastlane test_ui`.
+
+## Run tests for this plugin
+
+To run both the tests, and code style validation, run
+
+```
+rake
+```
+
+To automatically fix many of the styling issues, use
+```
+rubocop -a
+```
+
+## Issues and Feedback
+
+For any other issues and feedback about this plugin, please submit it to this repository.
+
+## Troubleshooting
+
+If you have trouble using plugins, check out the [Plugins Troubleshooting](https://docs.fastlane.tools/plugins/plugins-troubleshooting/) guide.
+
+## Using _fastlane_ Plugins
+
+For more information about how the `fastlane` plugin system works, check out the [Plugins documentation](https://docs.fastlane.tools/plugins/create-plugin/).
+
+## About _fastlane_
+
+_fastlane_ is the easiest way to automate beta deployments and releases for your iOS and Android apps. To learn more, check out [fastlane.tools](https://fastlane.tools).

data/lib/fastlane/plugin/rescan_flaky_tests.rb

@@ -0,0 +1,16 @@
+require 'fastlane/plugin/rescan_flaky_tests/version'
+
+module Fastlane
+  module RescanFlakyTests
+    # Return all .rb files inside the "actions" and "helper" directory
+    def self.all_classes
+      Dir[File.expand_path('**/{actions,helper}/*.rb', File.dirname(__FILE__))]
+    end
+  end
+end
+
+# By default we want to import all available actions and helpers
+# A plugin can contain any number of actions and plugins
+Fastlane::RescanFlakyTests.all_classes.each do |current|
+  require current
+end

data/lib/fastlane/plugin/rescan_flaky_tests/actions/rescan_flaky_tests_action.rb

@@ -0,0 +1,126 @@
+module Fastlane
+  module Actions
+    class RescanFlakyTestsAction < Action
+      def self.run(params)
+        first_report_path = params[:report_file]
+        unless File.exist?(first_report_path)
+          UI.user_error!("The report file '#{first_report_path}' is not found.")
+        end
+
+        test_cases = Helper::RescanFlakyTestsHelper.extract_failed_test_cases(first_report_path)
+        if test_cases.nil?
+          UI.user_error!("The report should be junit xml file.")
+        end
+        if test_cases.empty?
+          UI.important("There is no test to retry.")
+          return true
+        end
+
+        failed_test_limit = params[:failed_test_limit]
+        if failed_test_limit > 0 && failed_test_limit < test_cases.count
+          UI.error("Stop rescan because there are too many faild cases, #{test_cases.count} > #{failed_test_limit} (threshold).")
+          exit 0
+        end
+
+        rescan_output_root = params[:output_directory]
+
+        last_exception = nil
+
+        test_cases.each_with_index do |test_case, i|
+          retry_output_directory = File.join(rescan_output_root, "rescan", (i + 1).to_s)
+          UI.important("Rescan '#{test_case}'.")
+          UI.important("Reports are saved '#{retry_output_directory}'.")
+
+          begin
+            scan_options = params[:scan]
+            scan_options[:only_testing] = [test_case]
+            scan_options[:output_directory] = retry_output_directory
+            scan_options[:buildlog_path] = retry_output_directory
+
+            scan_params = FastlaneCore::Configuration.create(ScanAction.available_options, scan_options)
+            ::Fastlane::Actions::ScanAction.run(scan_params)
+          rescue => e
+            last_exception = e
+          end
+        end
+
+        raise last_exception unless last_exception.nil?
+      end
+
+      def self.description
+        "Re-run `scan` action for each failed test case."
+      end
+
+      def self.authors
+        ["Ichiko Moro"]
+      end
+
+      def self.details
+        "When you running UI tests (XCUITest, EarlGrey or others), sometime that failed with un clear reason. \n\
+This plugin re-scan each failed test cases. And report result under `\#{output_directory}/rescan/NN`."
+      end
+
+      def self.available_options
+        containing = FastlaneCore::Helper.fastlane_enabled_folder_path
+
+        [
+          FastlaneCore::ConfigItem.new(key: :report_file,
+                                       env_name: 'RESCAN_REPORT_FILE',
+                                       description: "JUnit report file path generated by scan, rescan target test cases extract from the file",
+                                       type: String,
+                                       default_value: File.join(containing, "test_output", "report.junit"),
+                                       default_value_dynamic: true),
+          FastlaneCore::ConfigItem.new(key: :output_directory,
+                                       env_name: 'RESCAN_OUTPUT_DIRECTORY',
+                                       description: "Each rescaned report and build log is saved in rescan_NN under the path",
+                                       type: String,
+                                       default_value: File.join(containing, "test_output"),
+                                       default_value_dynamic: true,
+                                       optional: true),
+          FastlaneCore::ConfigItem.new(key: :failed_test_limit,
+                                       env_name: 'RESCAN_FAILED_TEST_LIMIT',
+                                       description: "Stop rescan if faild test cases count is greater than this threshold. No limitation if this value is ZERO",
+                                       type: Integer,
+                                       default_value: 0,
+                                       optional: true),
+          FastlaneCore::ConfigItem.new(key: :scan,
+                                       description: "Parameters that are passed to the scan action",
+                                       type: Hash,
+                                       default_value: {},
+                                       optional: true)
+        ]
+      end
+
+      def self.is_supported?(platform)
+        [:ios, :mac].include?(platform)
+      end
+
+      def self.example_code
+        [
+          '# shoud run `scan` before to get first test result.',
+          'begin
+            scan
+          rescue
+            rescan_flaky_tests
+          end',
+          '# use rescue block or `fail_build` option',
+          'scan(fail_build: false)
+          rescan_flaky_tests',
+          '# recommend to pack scan options',
+          'scan_options = {
+            ...
+          }
+          begin
+            scan(scan_options)
+          rescue
+            rescan_flaky_tests(
+              scan_options.merge {
+                report_file: "path/to/report.junit"
+              }
+            )
+          end'
+        ]
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/rescan_flaky_tests/helper/rescan_flaky_tests_helper.rb

@@ -0,0 +1,28 @@
+require 'rexml/document'
+
+module Fastlane
+  module Helper
+    class RescanFlakyTestsHelper
+      # class methods that you define here become available in your action
+      # as `Helper::RescanEachFragileTestsHelper.your_method`
+      #
+      def self.extract_failed_test_cases(report_file)
+        xml_doc = REXML::Document.new(File.open(report_file))
+
+        if xml_doc.elements.collect("testsuites") { |e| e }.count == 0
+          return nil
+        end
+
+        test_cases = []
+        xml_doc.elements.each("testsuites/testsuite/testcase[failure]") do |test_case|
+          (bundle, test_suite) = test_case.attributes["classname"].split(".")
+          method = test_case.attributes["name"]
+          # xcodebuild changes '-' to '_'
+          bundle.gsub!("_", "-")
+          test_cases << "#{bundle}/#{test_suite}/#{method}"
+        end
+        test_cases
+      end
+    end
+  end
+end

data/lib/fastlane/plugin/rescan_flaky_tests/version.rb

@@ -0,0 +1,5 @@
+module Fastlane
+  module RescanFlakyTests
+    VERSION = "0.1.0"
+  end
+end

metadata

@@ -0,0 +1,161 @@
+--- !ruby/object:Gem::Specification
+name: fastlane-plugin-rescan_flaky_tests
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ichiko Moro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-03-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fastlane
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.70.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.70.2
+description: 
+email: ichiko.revjune@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/fastlane/plugin/rescan_flaky_tests.rb
+- lib/fastlane/plugin/rescan_flaky_tests/actions/rescan_flaky_tests_action.rb
+- lib/fastlane/plugin/rescan_flaky_tests/helper/rescan_flaky_tests_helper.rb
+- lib/fastlane/plugin/rescan_flaky_tests/version.rb
+homepage: https://github.com/ichiko/fastlane-plugin-rescan_each_fragile_tests
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.1
+signing_key: 
+specification_version: 4
+summary: Re-run `scan` action for each failed test cases.
+test_files: []