parallel_tests_report 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4c6de34b2f648c93e2faaf8c9c041b358fce7c0be16586a4f7f80b812d56cf3a
+  data.tar.gz: 5946b56b3b7fef0c6ce10ceef38eab92d12baf2cb32a9962c4b9ff575c19bf57
+SHA512:
+  metadata.gz: 30162444f35a72cf1c0378ad3403a4f0e66229095dca13de719777af66994e60334bde49f49ffd743d95b3122a240167c4a4329c0c7145275e17679ab56045fa
+  data.tar.gz: 9b134d077a65d5b132e2eead49ef3b694c56734dafca2080df13b19ed059e029730cce78ef4d5a8372a00d910b3e7bbb08818aea22512dd759281ad4cf5d0f4d

data/README.md

@@ -0,0 +1,34 @@
+# parallel-tests-report
+
+Works with [parallel_tests](https://github.com/grosser/parallel_tests) gem to generate a consolidated report for the spec groups executed by parallel_tests.
+
+The report generated will include:
+- List of top 20 slowest examples.
+- Rspec command to reproduce the failed example with the bisect option and seed value used.
+
+This gem will also verify the time taken for a test against configured threshold value and report if the time has exceeded.
+
+## How it works
+- parallel_tests gem is configured to use a custom formatter provided by this gem using `--format` and `--out` options.
+- Once tests are executed a rake task provided by this gem can be executed to parse the json and generate the report.
+
+## Installation
+Include the gem in your Gemfile
+
+`gem 'parallel_tests_report'`
+
+`$ bundle install`
+
+Add the following to the Rakefile before load_task(In Rails application):
+
+`require 'parallel_tests_report'`
+
+## Usage
+- add `--format` and `--out` option to `.rspec` or `.rspec_parallel`
+  - `--format ParallelTestsReport::JsonFormatter --out tmp/test-results/rspec.json`
+- execute the rake task after specs are executed 
+  - `bundle exec parallel_tests_report rake generate:report <TIME_LIMIT_IN_SECONDS> tmp/test-results/rspec.json`
+  - <TIME_LIMIT_IN_SECONDS> is the maximum time an example can take. Default is 10 seconds.
+  - <OUTPUT_FILE> is the file specified in the --out option. Default is 'tmp/test-results/rspec.json'
+
+#### This rake task can be configured to run after specs are executed in a continuous integration setup, it also produces a junit xml file for time limit exceeding check.

data/Rakefile

@@ -0,0 +1,3 @@
+require 'parallel_tests_report'
+
+Dir.glob("lib/parallel_tests_report/tasks/*.rake").each { |f| import f }

data/bin/parallel_tests_report

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+gem_dir = File.expand_path("..",File.dirname(__FILE__))
+$LOAD_PATH.unshift gem_dir
+exec_type = ARGV[0]
+if exec_type == 'rake' then
+    require 'rake'
+    require 'pp'
+    pwd=Dir.pwd
+    Dir.chdir(gem_dir)
+    Rake.application.init
+    Rake.application.load_rakefile
+    Dir.chdir(pwd)
+    Rake::Task[ARGV[1]].invoke(ARGV[2], ARGV[3])
+end

data/lib/parallel_tests_report.rb

@@ -0,0 +1,6 @@
+require "rspec/core"
+require "rspec/core/formatters/base_formatter"
+
+module ParallelTestsReport
+  require 'parallel_tests_report/railtie' if defined?(Rails)
+end

data/lib/parallel_tests_report/generate_report.rb

@@ -0,0 +1,114 @@
+require 'parallel_tests_report'
+require 'json'
+require 'nokogiri'
+
+class ParallelTestsReport::GenerateReport
+  def start(time_limit, output)
+    all_examples = []
+    slowest_examples = []
+    failed_examples = []
+    time_exceeding_examples = []
+    rerun_failed = []
+    errors = []
+
+    return if File.zero?(output)
+
+    File.foreach(output) do |line|
+      parallel_suite = JSON.parse(line)
+      all_examples += parallel_suite["examples"]
+      slowest_examples += parallel_suite["profile"]["examples"]
+      failed_examples += parallel_suite["examples"].select {|ex| ex["status"] == "failed" }
+      time_exceeding_examples += parallel_suite["examples"].select {|ex| ex["run_time"] >= time_limit}
+      errors << parallel_suite["messages"][0] if parallel_suite["examples"].size == 0
+    end
+
+    if slowest_examples.size > 0
+      slowest_examples = slowest_examples.sort_by do |ex|
+        -ex["run_time"]
+      end.first(20)
+      puts "Top #{slowest_examples.size} slowest examples\n"
+      slowest_examples.each do |ex|
+        puts <<-TEXT
+  #{ex["full_description"]}
+      #{ex["run_time"]} #{"seconds"} #{ex["file_path"]} #{ex["line_number"]}
+        TEXT
+      end
+    end
+
+    if failed_examples.size > 0
+      puts "\nFailed Examples:\n"
+      failed_examples.each do |ex|
+        puts <<-TEXT
+  => #{ex["full_description"]}
+      #{ex["run_time"]} #{"seconds"} #{ex["file_path"]} #{ex["line_number"]}
+        #{ex["exception"]["message"]}
+        TEXT
+        all_examples.each do |e|
+          rerun_failed << e["file_path"].to_s if e["parallel_test_proessor"] == ex["parallel_test_proessor"] && !rerun_failed.include?(e["file_path"])
+        end
+        str = ""
+        rerun_failed.each do |e|
+          str += e + " "
+        end
+        puts <<-TEXT
+  \n\s\sIn case the failure: "#{ex["full_description"]}" is due to random ordering, run the following command to isolate the minimal set of examples that reproduce the same failures:
+    `bundle exec rspec #{str} --seed #{ex['seed']} --bisect`\n
+        TEXT
+        rerun_failed.clear
+      end
+    end
+
+    if errors.size > 0
+      puts "\Errors:\n"
+      errors.each do |err|
+        puts <<-TEXT
+        #{err}
+        TEXT
+      end
+    end
+
+    if time_exceeding_examples.size > 0 || errors.size > 0
+      generate_xml(errors, time_exceeding_examples, time_limit)
+    end
+
+    if time_exceeding_examples.size > 0
+      puts "\nExecution time is exceeding the threshold of #{@time_limit} seconds for following tests:"
+      time_exceeding_examples.each do |ex|
+        puts <<-TEXT
+  => #{ex["full_description"]}: #{ex["run_time"]} #{"Seconds"}
+        TEXT
+      end
+    else
+      puts "Runtime check Passed."
+    end
+
+    if failed_examples.size > 0 || errors.size > 0 || time_exceeding_examples.size > 0
+      fail_message = "Tests Failed"
+      puts "\e[31m#{fail_message}\e[0m"
+      exit 1
+    end
+  end
+
+  def generate_xml(errors, time_exceeding_examples, time_limit)
+    builder = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml|
+      xml.testsuite {
+        time_exceeding_examples.each do |arr|
+          classname = "#{arr["file_path"]}".sub(%r{\.[^/]*\Z}, "").gsub("/", ".").gsub(%r{\A\.+|\.+\Z}, "")
+          xml.testcase("classname" => "#{classname}", "name" => "#{arr["full_description"]}", "file" => "#{arr["file_path"]}", "time" => "#{arr["run_time"]}") {
+            xml.failure "Execution time is exceeding the threshold of #{time_limit} seconds"
+          }
+        end
+        errors.each do |arr|
+          file_path = arr[/(?<=An error occurred while loading ).*/]
+          classname = "#{file_path}".sub(%r{\.[^/]*\Z}, "").gsub("/", ".").gsub(%r{\A\.+|\.+\Z}, "")
+          xml.testcase("classname" => "#{classname}", "name" => "An error occurred while loading", "file" => "#{file_path}", "time" => "0.0") {
+            xml.failure arr.gsub(/\e\[([;\d]+)?m/, "").gsub(/An error occurred while loading #{file_path}\n/, "")
+          }
+        end
+      }
+    end
+    File.open('tmp/test-results/time_limit_exceeded.xml', 'w') do |file|
+      file << builder.to_xml
+    end
+  end
+end

data/lib/parallel_tests_report/json_formatter.rb

@@ -0,0 +1,113 @@
+# With this JsonFormatter we are generating a file which contains a json for each and every parallel_test_suite. It contains all the passed, failed, pending and profiled examples with their full_description, file_path, run_time, seed value and parallel_test_proessor number.
+# We parse each line of this file, to generate a report to show slowest examples, failed examples, errors and runtime checks.
+
+# And example to show the structure of the file:
+# This is a single line containing details for one parallel_test_suite
+=begin
+{"messages":["Run options: exclude {:asrun=\u003etrue, :to_be_implemented=\u003etrue, :integration=\u003etrue}"],"seed":2121,"examples":[{"full_description":"An example", updated_at, id, media_id","status":"passed","file_path":"/path/to/the/example","line_number":01,"run_time":2.269736609,"parallel_test_proessor":1,"seed":2121},{"full_description":"Another Example","status":"passed","file_path":"path/to/another/exmple","line_number":20,"run_time":4.139183023,"parallel_test_proessor":1,"seed":2121}],"profile":{"examples":[{"full_description":"An example", updated_at, id, media_id","status":"passed","file_path":"/path/to/the/example","line_number":01,"run_time":2.269736609,"parallel_test_proessor":1,"seed":2121},{"full_description":"Another Example","status":"passed","file_path":"path/to/another/exmple","line_number":20,"run_time":4.139183023,"parallel_test_proessor":1,"seed":2121}]}}
+=end
+
+require 'parallel_tests_report'
+
+class ParallelTestsReport::JsonFormatter < RSpec::Core::Formatters::BaseFormatter
+  RSpec::Core::Formatters.register self, :message, :dump_profile, :seed, :stop, :close
+  attr_reader :output_hash, :output
+  def initialize(output)
+    super
+    @output ||= output
+    if String === @output
+      #open the file given as argument in --out
+      FileUtils.mkdir_p(File.dirname(@output))
+      # overwrite previous results
+      File.open(@output, 'w'){}
+      @output = File.open(@output, 'a')
+      # close and restart in append mode
+    elsif File === @output
+      @output.close
+      @output = File.open(@output.path, 'a')
+    end
+    @output_hash = {}
+
+    if ENV['TEST_ENV_NUMBER'].to_i != 0
+      @n = ENV['TEST_ENV_NUMBER'].to_i
+    else
+      @n = 1
+    end
+  end
+
+  def message(notification)
+    (@output_hash[:messages] ||= []) << notification.message
+  end
+
+  def seed(notification)
+    return unless notification.seed_used?
+    @output_hash[:seed] = notification.seed
+  end
+
+  def close(_notification)
+    #close the file after all the processes are finished
+    @output.close if (IO === @output) & (@output != $stdout)
+  end
+
+  def stop(notification)
+    #adds to @output_hash, an array of examples which run in a particular processor
+    @output_hash[:examples] = notification.examples.map do |example|
+      format_example(example).tap do |hash|
+        e = example.exception
+        if e
+          hash[:exception] =  {
+            :class => e.class.name,
+            :message => e.message,
+            :backtrace => e.backtrace,
+          }
+        end
+      end
+    end
+  end
+
+  def dump_profile(profile)
+    dump_profile_slowest_examples(profile)
+  end
+
+  def dump_profile_slowest_examples(profile)
+    #adds to @output_hash, an array of 20 slowest examples
+    lock_output do
+      @output_hash[:profile] = {}
+      @output_hash[:profile][:examples] = profile.slowest_examples.map do |example|
+        format_example(example)
+      end
+    end
+    #write the @output_hash to the file
+    output.puts @output_hash.to_json
+    output.flush
+  end
+
+protected
+  #to make a single file for all the parallel processes
+  def lock_output
+    if File === @output
+      begin
+        @output.flock File::LOCK_EX
+        yield
+      ensure
+        @output.flock File::LOCK_UN
+      end
+    else
+      yield
+    end
+  end
+
+private
+
+  def format_example(example)
+    {
+      :full_description => example.full_description,
+      :status => example.execution_result.status.to_s,
+      :file_path => example.metadata[:file_path],
+      :line_number  => example.metadata[:line_number],
+      :run_time => example.execution_result.run_time,
+      :parallel_test_proessor => @n,
+      :seed => @output_hash[:seed]
+    }
+  end
+end

data/lib/parallel_tests_report/railtie.rb

@@ -0,0 +1,13 @@
+require 'parallel_tests_report'
+require 'rails'
+
+module ParallelTestsReport
+  class Railtie < Rails::Railtie
+    railtie_name :parallel_tests_report
+
+    rake_tasks do
+      path = File.expand_path(__dir__)
+      Dir.glob("#{path}/tasks/**/*.rake").each { |f| load f }
+    end
+  end
+end

data/lib/parallel_tests_report/tasks/generate_report.rake

@@ -0,0 +1,24 @@
+require_relative '../../parallel_tests_report/generate_report.rb'
+
+namespace :generate do
+  task :report, [:time_limit, :output] do |t,args|
+    output = args[:output].to_s
+    if output == ""
+      if(args[:time_limit] != nil && args[:time_limit].to_f == 0.0 && args[:time_limit] != '0.0') # If only one argument is given while calling the rake_task and that is :output.
+        #Since, first argument is :time_limit, assigning that to output.
+        output = args[:time_limit].to_s
+      else
+        output = 'tmp/test-results/rspec.json' # default :output file
+      end
+    end
+    time_limit = args[:time_limit].to_f
+    if time_limit == 0.0
+      if args[:time_limit] == "0.0" #if :time_limit itself is 0.0
+        time_limit = 0.0
+      else
+        time_limit = 10.0 # default :time_limit
+      end
+    end
+    ParallelTestsReport::GenerateReport.new.start time_limit,output
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: parallel_tests_report
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Akshat Birani
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-08-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Works with parallel_tests ruby gem to generate a report having a list
+  of slowest and failed examples.
+email: akshat@amagi.com
+executables:
+- parallel_tests_report
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- bin/parallel_tests_report
+- lib/parallel_tests_report.rb
+- lib/parallel_tests_report/generate_report.rb
+- lib/parallel_tests_report/json_formatter.rb
+- lib/parallel_tests_report/railtie.rb
+- lib/parallel_tests_report/tasks/generate_report.rake
+homepage: https://github.com/amagimedia/parallel-tests-report
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Generate report for parallel_tests
+test_files: []