gurke 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2311a29357e984f6868342e33b6521a51c7697774155d061090b21484a446a8
-  data.tar.gz: 314bc36b51bd2236407ea0489679edca0aaa78ce47a2870348557a73a66637d1
+  metadata.gz: 0dfc9f938d485d50f5e8ac6f3d7c978b2cb9266f478bf8a8e1a29d186e043164
+  data.tar.gz: 27ef19f2031a73e0cf4105b69587ee088f3bb17f8f74305dc2b5dad09e282d44
 SHA512:
-  metadata.gz: d658e154b2d261d8b4a529f8f7d6b474e8519a042e15808cdc8492a2084c4d0ffa1020cf997ef4900c33a79200cc31886024905da3725038f217b6c362241372
-  data.tar.gz: 2d2ec2d30909f5b6b3f2673cdb079ef2342c50f9cf7bdf9d9f40e62d022ab08d1570ec4f22212a7c337f9e8bcdef277dd4c862a701d923a4644e10d4cb68ed03
+  metadata.gz: 49a9f970050511d4946669c040dc37ba2d303a76184518a34ec7ecc74e2f129bbbff321e15bf167220cc4ec3542d13a4051fa95fa2bc3db74d8fc229b5a3fdf1
+  data.tar.gz: b46d5a895d606a0b772f96cd293ef27f2d62529516ffa5950036682eacb45882a3374bed01f4af41fd6d132386d1e6b8d69c53c715c3fd2f7c33fb388683eedf

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 3.2.0
+
+* Add @flaky tag support for retrying flaky scenarios once
+
 ## 3.1.0
 
 * Update TeamCity formatter to extend default formatter

data/README.md

@@ -162,6 +162,20 @@ If you append one or more line numbers - separated by colons - only the scenario
 gurke features/my_feature.feature:14:34
 ```
 
+### Flaky scenarios
+
+If you have scenarios that might fail sometime, you can mark them as `@flaky`:
+
+```
+Feature: F
+  @flaky
+  Scenario: I am flaky
+    Given I fail the first time
+    Then I will be retried a second time
+```
+
+Gurke will retry a marked scenario only once if a step failed.
+
 ### DRb background server (experimental)
 
 You can run a DRb server in the background that has a running test environment (whatever that means to you) by running `gurke --drb-server`. This will load your test environment and execute all before `:system` hooks.

data/features/gurke/retry_scenario.feature

@@ -0,0 +1,70 @@
+Feature: Pending Steps
+  In order to fail less often with flaky scenarios
+  As a CI administrator and tester
+  I want to have marked scenarios retried once
+
+  Background:
+    Given a file "features/support/steps/test_steps.rb" with the following content exists
+      """
+      $try = 0
+
+      module TestSteps
+        step("I fail the first time") do
+          fail 'first time' if ($try += 1) < 2
+        end
+
+        step("I fail always") do
+          fail 'always'
+        end
+
+        step("I do not fail") do
+          # noop
+        end
+      end
+
+      Gurke.configure{|c| c.include TestSteps }
+      """
+
+  Scenario: Run a flaky scenario
+    Given a file "features/test.feature" with the following content exists
+      """
+      Feature: F
+        @flaky
+        Scenario: Scenario Failure
+          Given I fail the first time
+      """
+    When I run the tests
+    Then the program exit code should be null
+    And the program output should include "Given I fail the first time (failure)"
+    And the program output should include "Given I fail the first time (passed)"
+    And the program output should include "Retry flaky scenario due to previous failure:"
+    And the program output should include "1 scenarios: 0 failing, 0 pending"
+
+  Scenario: Run a marked but always failing scenario
+    Given a file "features/test.feature" with the following content exists
+      """
+      Feature: F
+        @flaky
+        Scenario: Scenario Failure
+          Given I fail always
+      """
+    When I run the tests
+    Then the program exit code should be non-null
+    And the program output should include "Given I fail always (failure)"
+    And the program output should include "Retry flaky scenario due to previous failure:"
+    And the program output should not include "Given I fail always (passed)"
+    And the program output should include "1 scenarios: 1 failing, 0 pending"
+
+  Scenario: Run a marked but passing scenario
+    Given a file "features/test.feature" with the following content exists
+      """
+      Feature: F
+        @flaky
+        Scenario: Scenario Failure
+          Given I do not fail
+      """
+    When I run the tests
+    Then the program exit code should be null
+    And the program output should include "Given I do not fail (passed)"
+    And the program output should not include "Retry flaky scenario due to previous failure:"
+    And the program output should include "1 scenarios: 0 failing, 0 pending"

data/lib/gurke/reporter.rb

@@ -30,6 +30,8 @@ module Gurke
       after_feature
       after_scenario
       after_step
+
+      retry_scenario
     ].freeze
 
     # Called before the execution of any feature and before any
@@ -194,6 +196,17 @@ module Gurke
         "#{self.class.name}#end_scenario must be implemented in subclass."
     end
 
+    # Called when a flaky scenario is retried.
+    #
+    # @param scenario [Scenario] Current scenario.
+    #
+    # @api public
+    #
+    def retry_scenario(_scenario)
+      raise NotImplementedError.new \
+        "#{self.class.name}#retry_scenario must be implemented in subclass."
+    end
+
     # Called after each scenario and after all hooks.
     #
     # @param scenario [Scenario] Current scenario.

data/lib/gurke/reporters/default_reporter.rb

@@ -73,6 +73,10 @@ module Gurke::Reporters
       io.flush
     end
 
+    def retry_scenario(*)
+      io.print "  Retry flaky scenario due to previous failure:\n"
+    end
+
     def after_scenario(*)
       io.puts
     end

data/lib/gurke/reporters/team_city_reporter.rb

@@ -14,12 +14,19 @@ module Gurke::Reporters
 
     def before_scenario(scenario)
       @status_reported = false
+      @retry = false
 
       publish :testStarted, name: scenario.name
 
       super
     end
 
+    def retry_scenario(scenario)
+      @retry = true
+
+      super
+    end
+
     def after_scenario(scenario)
       publish :testFinished, name: scenario.name
 
@@ -45,10 +52,13 @@ module Gurke::Reporters
     def step_failed(step, *args)
       super(step, *args, exception: false)
 
-      report :testFailed,
-        name: step.scenario.name,
-        message: step.exception.inspect,
-        backtrace: step.exception.backtrace.join('\n')
+      unless step.scenario.retryable? && !retry?
+        # do not report test as failed if it will be retries
+        report :testFailed,
+          name: step.scenario.name,
+          message: step.exception.inspect,
+          backtrace: step.exception.backtrace.join('\n')
+      end
 
       print_exception(step.exception)
     end
@@ -59,6 +69,10 @@ module Gurke::Reporters
       @status_reported
     end
 
+    def retry?
+      @retry
+    end
+
     def report(*args)
       return if status_reported?
 

data/lib/gurke/scenario.rb

@@ -136,6 +136,10 @@ module Gurke
       @state     = :pending
     end
 
+    def retryable?
+      @tags.any? {|t| t.name == 'flaky' }
+    end
+
     # @api private
     #
     def passed!
@@ -158,12 +162,26 @@ module Gurke
       runner.hook :scenario, self, world do
         run_scenario runner, reporter
       end
+
+      if failed? && retryable?
+        reporter.invoke :retry_scenario, self
+        reset!
+
+        runner.hook :scenario, self, world do
+          run_scenario runner, reporter
+        end
+      end
     ensure
       reporter.invoke :after_scenario, self
     end
 
     private
 
+    def reset!
+      @state = nil
+      @exception = nil
+    end
+
     def run_scenario(runner, reporter)
       reporter.invoke :start_scenario, self
 

data/lib/gurke/version.rb

@@ -3,7 +3,7 @@
 module Gurke
   module VERSION
     MAJOR = 3
-    MINOR = 1
+    MINOR = 2
     PATCH = 0
     STAGE = nil
     STRING = [MAJOR, MINOR, PATCH, STAGE].reject(&:nil?).join('.').freeze

data/spec/gurke/reporters/default_reporter_spec.rb

@@ -159,6 +159,19 @@ RSpec.describe Gurke::Reporters::DefaultReporter do
     end
   end
 
+  describe '#retry_scenario' do
+    let(:scenario) { double('scenario') }
+
+    subject { reporter.retry_scenario(scenario); super() }
+
+    it do
+      is_expected.to eq unindent <<~TEXT
+        .  Retry flaky scenario due to previous failure:
+        .
+      TEXT
+    end
+  end
+
   describe '#after_scenario' do
     let(:scenario) { double('scenario') }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gurke
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Jan Graichen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-26 00:00:00.000000000 Z
+date: 2018-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -86,6 +86,7 @@ files:
 - features/gurke/include_by_tags.feature
 - features/gurke/other_reporter.feature
 - features/gurke/pending_steps.feature
+- features/gurke/retry_scenario.feature
 - features/gurke/run_specific_directories.feature
 - features/gurke/run_specific_scenarios.feature
 - features/gurke/step_specific_definitions.feature
@@ -154,6 +155,7 @@ test_files:
 - features/gurke/include_by_tags.feature
 - features/gurke/other_reporter.feature
 - features/gurke/pending_steps.feature
+- features/gurke/retry_scenario.feature
 - features/gurke/run_specific_directories.feature
 - features/gurke/run_specific_scenarios.feature
 - features/gurke/step_specific_definitions.feature