rspecq 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6b4c91525a2fb29e2198f290877ffe5ef1e753dafe0f9babd75a581e25d7af8
-  data.tar.gz: 27d3705ee014a5dc77514238b36386eaf11d1bb76601ab25c463796184ea5795
+  metadata.gz: 87f2cf8ea2c0b03bf491442902644d24dd001265b2550c0403073d3f34d99903
+  data.tar.gz: 646b803ece2861f4ac8faced44f620ea5608636be38033f6931c0550bf4d79bd
 SHA512:
-  metadata.gz: 21803fa664abe45f173f7121dc948ebc8dbc0df41046f8e6269e8ec3751b647701b07a8b6a872aad0da22f438c043ebe6bf0fc69bc9c9c8b327e3b157dccab04
-  data.tar.gz: 4d683884610e2e28ca5ce5cf891e37135886ea4c6ec905f26607c1f0fdd62c1952e9c49983bb0191eb5d42433a800bf88f8227354a213b50645742f21f95af16
+  metadata.gz: 418cf064a63ff4ea790495f578bb02f7c76ce6c0536af7a8fbdd8d6a293182f75cc15c587f2bcf798175f1ca0c427a208d8db326d157eacfcc199b783ce00d26
+  data.tar.gz: c109a02fc2412f7bb412422d97ee8096d85795893904d7034bbf7928d9c4d8fc5492f279d3f8cfaa77a4065f2ec591ac23915a7d53c181340610c057558fa16f

data/CHANGELOG.md

@@ -4,6 +4,15 @@ Breaking changes are prefixed with a "[BREAKING]" label.
 
 ## master (unreleased)
 
+
+
+## 0.2.0 (2020-08-31)
+
+### Added
+
+- Flaky jobs are now printed by the reporter in the final build output and also
+  emitted to Sentry (if the integration is enabled) [[#26](https://github.com/skroutz/rspecq/pull/26)]
+
 ## 0.1.0 (2020-08-27)
 
 ### Added

data/README.md

@@ -13,7 +13,7 @@ and [ci-queue](https://github.com/Shopify/ci-queue).
 
 ## Features
 
-- Run an RSpec suite among many workers 
+- Run an RSpec suite among many workers
   (potentially located in different hosts) in a distributed fashion,
   facilitating faster CI builds.
 - Consolidated, real-time reporting of a build's progress.
@@ -21,11 +21,11 @@ and [ci-queue](https://github.com/Shopify/ci-queue).
   automatically scheduling slow spec files as individual examples. See
   [*Spec file splitting*](#spec-file-splitting).
 - Automatic retry of test failures before being considered legit, in order to
-  rule out flakiness. See [*Requeues*](#requeues).
+  rule out flakiness. Additionally, flaky tests are detected and provided to
+  the user. See [*Requeues*](#requeues).
 - Handles intermittent worker failures (e.g. network hiccups, faulty hardware etc.)
   by detecting non-responsive workers and requeing their jobs. See [*Worker failures*](#worker-failures)
-- [Sentry](https://sentry.io) integration for monitoring important
-  RSpecQ-level events.
+- Sentry integration for monitoring build-level events. See [*Sentry integration*](#sentry-integration).
 - [PLANNED] StatsD integration for various build-level metrics and insights.
   See [#2](https://github.com/skroutz/rspecq/issues/2).
 
@@ -124,10 +124,13 @@ dynamically (see #3).
 
 As a mitigation technique against flaky tests, if an example fails it will be
 put back to the queue to be picked up by another worker. This will be repeated
-up to a certain number of times (set with the `--max-requeues` option), after 
-which the example will be considered a legit failure and printed as such in the 
+up to a certain number of times (set with the `--max-requeues` option), after
+which the example will be considered a legit failure and printed as such in the
 final report.
 
+Flaky tests are also detected and printed as such in the final report. They are
+also emitted to Sentry (see [Sentry integration](#sentry-integration)).
+
 ### Worker failures
 
 It's not uncommon for CI processes to encounter unrecoverable failures for
@@ -173,8 +176,8 @@ rejects).
 The downside of this design is that it's more complicated, since the scheduling
 of spec files happens based on timings calculated from previous runs. This
 means that RSpecQ maintains a key with the timing of each job and updates it
-on every run (if the `--timings` option was used). Also, RSpecQ has a "slow
-file threshold" which, currently has to be set manually (but this can be
+on every run (if the `--update-timings` option was used). Also, RSpecQ has a
+"slow file threshold" which, currently has to be set manually (but this can be
 improved in the future).
 
 

data/lib/rspecq/queue.rb

@@ -184,6 +184,7 @@ module RSpecQ
       @redis.hgetall(key_errors)
     end
 
+    # True if the build is complete, false otherwise
     def exhausted?
       return false if !published?
 
@@ -216,6 +217,19 @@ module RSpecQ
       @redis.lrange(key_queue_unprocessed, 0, -1)
     end
 
+    # Returns the jobs considered flaky (i.e. initially failed but passed
+    # after being retried). Must be called after the build is complete,
+    # otherwise an exception will be raised.
+    def flaky_jobs
+      raise "Queue is not yet exhausted" if !exhausted?
+
+      requeued = @redis.hkeys(key_requeues)
+
+      return [] if requeued.empty?
+
+      requeued - @redis.hkeys(key_failures)
+    end
+
     # redis: STRING [STATUS_INITIALIZING, STATUS_READY]
     def key_queue_status
       key("queue", "status")

data/lib/rspecq/reporter.rb

@@ -45,8 +45,13 @@ module RSpecQ
       raise "Build not finished after #{@timeout} seconds" if !finished
 
       @queue.record_build_time(tests_duration)
+
+      flaky_jobs = @queue.flaky_jobs
+
       puts summary(@queue.example_failures, @queue.non_example_errors,
-                   humanize_duration(tests_duration))
+        flaky_jobs, humanize_duration(tests_duration))
+
+      flaky_jobs_to_sentry(flaky_jobs, tests_duration)
 
       exit 1 if !@queue.build_successful?
     end
@@ -60,7 +65,7 @@ module RSpecQ
     end
 
     # We try to keep this output consistent with RSpec's original output
-    def summary(failures, errors, duration)
+    def summary(failures, errors, flaky_jobs, duration)
       failed_examples_section = "\nFailed examples:\n\n"
 
       failures.each do |_job, msg|
@@ -81,6 +86,14 @@ module RSpecQ
                  "#{errors.count} errors"
       summary << "\n\n"
       summary << "Spec execution time: #{duration}"
+
+      if !flaky_jobs.empty?
+        summary << "\n\n"
+        summary << "Flaky jobs detected (count=#{flaky_jobs.count}):\n"
+        flaky_jobs.each { |j| summary << "  #{j}\n" }
+      end
+
+      summary
     end
 
     def failure_formatted(rspec_output)
@@ -90,5 +103,20 @@ module RSpecQ
     def humanize_duration(seconds)
       Time.at(seconds).utc.strftime("%H:%M:%S")
     end
+
+    def flaky_jobs_to_sentry(jobs, build_duration)
+      return if jobs.empty?
+
+      Raven.capture_message("Flaky jobs detected", level: "warning", extra: {
+        build: @build_id,
+        build_timeout: @timeout,
+        queue: @queue.inspect,
+        object: self.inspect,
+        pid: Process.pid,
+        flaky_jobs: jobs,
+        flaky_jobs_count: jobs.count,
+        build_duration: build_duration
+      })
+    end
   end
 end

data/lib/rspecq/version.rb

@@ -1,3 +1,3 @@
 module RSpecQ
-  VERSION = "0.1.0".freeze
+  VERSION = "0.2.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspecq
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Agis Anastasopoulos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-27 00:00:00.000000000 Z
+date: 2020-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core
@@ -148,7 +148,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubyforge_project: 
+rubygems_version: 2.7.6.2
 signing_key: 
 specification_version: 4
 summary: Optimally distribute and run RSpec suites among parallel workers; for faster