flaky_stats 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2f46e85bcc221c806457d3f481f5ffa56c893cd2
4
+ data.tar.gz: 823883da6f051e786ec1f894a37fb3a9e91641d9
5
+ SHA512:
6
+ metadata.gz: 31002d561575e96e7d9d024f10855b04b50ff0003219883976969d04a2de48b4e6a6430fdde2327f8ea53434cf931c44aeed5279e5806dcedb032369b96da826
7
+ data.tar.gz: 05c5ee0c41c6c4a070b07e4258d4fdb6d7cdcf0ddc1b7823c60295407689de0804745e08b64309de0b4d47d6ee54634f2a0d0d4cad2b10cf3a0c6ded0c1ec1d5
data/.gitignore ADDED
@@ -0,0 +1,13 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+
11
+ # rspec failure tracking
12
+ .rspec_status
13
+ *.gem
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
data/.travis.yml ADDED
@@ -0,0 +1,5 @@
1
+ sudo: false
2
+ language: ruby
3
+ rvm:
4
+ - 2.4.1
5
+ before_install: gem install bundler -v 1.14.6
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in flaky_stats.gemspec
4
+ gemspec
data/README.md ADDED
@@ -0,0 +1,41 @@
1
+ # FlakyStats
2
+
3
+ Collects broken tests from a parallel failing test log, uses them to rerun the flaky tests and if they pass whilst running them on their own add them to the flaky_stats.log file.
4
+
5
+ At the end of the tests we can display a flaky stats summary which should display the top flaky tests in order of the most flakiest.
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'flaky_stats'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install flaky_stats
22
+
23
+ ## Usage
24
+
25
+ $ rake parallel:rerun
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/map7/flaky_stats.
36
+
37
+ ## Ref
38
+
39
+ http://blog.nathanhumbert.com/2010/02/rails-3-loading-rake-tasks-from-gem.html
40
+
41
+
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "flaky_stats"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,28 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'flaky_stats/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "flaky_stats"
8
+ spec.version = FlakyStats::VERSION
9
+ spec.authors = ["Michael Pope"]
10
+ spec.email = ["michael@dtcorp.com.au"]
11
+
12
+ spec.summary = %q{Records flaky tests}
13
+ spec.description = %q{Flaky stats will rerun broken tests from your parallel tests and if they pass then they must be flaky so it records them.}
14
+ spec.homepage = "http://github.com/map7/flaky_stats"
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
+ f.match(%r{^(test|spec|features)/})
19
+ end
20
+ spec.bindir = "exe"
21
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
22
+ spec.require_paths = ["lib"]
23
+
24
+ spec.add_development_dependency "bundler", "~> 1.14"
25
+ spec.add_development_dependency "rake", "~> 10.0"
26
+ spec.add_development_dependency "rspec", "~> 3.0"
27
+ spec.add_development_dependency "timecop", "= 0.8.1"
28
+ end
@@ -0,0 +1,37 @@
1
+ module FlakyStats
2
+ class FlakyTests
3
+
4
+ def initialize(options = {})
5
+ @logfile = options[:logfile]
6
+ end
7
+
8
+ def form_data(failed_file = {})
9
+ return [Time.now,failed_file[:filename],failed_file[:lineno],1].join ","
10
+ end
11
+
12
+ # Run each failing test singularly and return a list of flaky tests.
13
+ def run(failed_files)
14
+ real_flaky_tests = []
15
+
16
+ puts "\n\n"
17
+ puts "-------------------------------------------------------------------"
18
+ puts "Rerunning failing tests in single thread"
19
+ puts "-------------------------------------------------------------------"
20
+
21
+ # sleep 10 # Settle everything down.
22
+
23
+ # Run all failing tests separately with '--format doc' on the end.
24
+ failed_files.each do |failed_file|
25
+ # sleep 2 # Settle everything down.
26
+ status = system("rspec --format doc #{failed_file[:filename]}")
27
+
28
+ # This is a flaky test only, so record it
29
+ if status == true
30
+ real_flaky_tests << form_data(failed_file)
31
+ end
32
+ end
33
+
34
+ return real_flaky_tests
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,46 @@
1
+ # Logfile
2
+ #
3
+ # Deals with reading failing logs, remove old log files and
4
+ # logging the flaky stats
5
+
6
+ module FlakyStats
7
+ class LogFile
8
+ def initialize(options = {})
9
+ @failing_log = options[:failing_log]
10
+ @logfile = options[:logfile]
11
+ end
12
+
13
+ def get_error_info(line)
14
+ information= line.partition(/\.\/.*.rb/)
15
+ filename = information[1]
16
+ lineno = information[2].partition(/:/)[2].to_i
17
+ return {filename: filename, lineno: lineno}
18
+ end
19
+
20
+ # Read the failing log from our test stack
21
+ def read_failing_log()
22
+ failed_files = []
23
+
24
+ # Read in the file
25
+ file = File.readlines(@failing_log)
26
+
27
+ # Get lines which begin with rspec
28
+ file.each do |line|
29
+ if line =~ /rspec \.\//
30
+ # Get the file name only
31
+ failed_files << get_error_info(line)
32
+ end
33
+ end
34
+
35
+ return failed_files
36
+ end
37
+
38
+ # Log flaky stats
39
+ def write_flaky_stats(real_flaky_tests)
40
+ File.open(@logfile, "a") do |log|
41
+ real_flaky_tests.each {|data| log.puts data}
42
+ end
43
+ end
44
+
45
+ end
46
+ end
@@ -0,0 +1,11 @@
1
+ require 'flaky_stats'
2
+ require 'rails'
3
+ module FlakyStats
4
+ class Railtie < Rails::Railtie
5
+ railtie_name :flaky_stats
6
+
7
+ rake_tasks do
8
+ load "tasks/flaky_stats.rake"
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,41 @@
1
+ # Summary
2
+ #
3
+ # Calculate and display summary information for errors and flaky stats
4
+
5
+ module FlakyStats
6
+ class Summary
7
+ def initialize(options)
8
+ @failing_log = options[:failing_log]
9
+ @logfile = options[:logfile]
10
+ end
11
+
12
+ def display_error_summary()
13
+ puts "\n\n"
14
+ puts "-------------------------------------------------------------------"
15
+ puts "Errors summary"
16
+ puts "-------------------------------------------------------------------"
17
+
18
+ system("cat #{@failing_log}")
19
+ end
20
+
21
+ def calc_flaky_summary
22
+ sum=Hash.new(0)
23
+ CSV.foreach(@logfile) do |row|
24
+ sum[row[1]]+=row[3].to_i
25
+ end
26
+ return sum
27
+ end
28
+
29
+ def display_flaky_summary
30
+ puts "\n\n"
31
+ puts "-------------------------------------------------------------------"
32
+ puts "Flaky summary"
33
+ puts "-------------------------------------------------------------------"
34
+
35
+ calc_flaky_summary.each do |k,v|
36
+ puts "#{k} = #{v}" if v > 1
37
+ end
38
+ puts "\n"
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,3 @@
1
+ module FlakyStats
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,8 @@
1
+ require "flaky_stats/version"
2
+ require "flaky_stats/summary"
3
+ require "flaky_stats/logfile"
4
+ require "flaky_stats/flaky_tests"
5
+
6
+ module FlakyStats
7
+ require 'flaky_stats/railtie' if defined?(Rails)
8
+ end
@@ -0,0 +1,26 @@
1
+ require 'csv'
2
+
3
+ LOGFILE=Rails.root + "log/flaky_tests.log"
4
+ FAILING_LOG=Rails.root + "tmp/failing_specs.log"
5
+
6
+ desc "Rerun failing parallel tests in a single thread"
7
+ task :flaky_stats => :environment do |t|
8
+ if File.exist?(FAILING_LOG)
9
+
10
+ # Read failing log
11
+ logfile = FlakyStats::LogFile.new(failing_log: FAILING_LOG, logfile: LOGFILE)
12
+ failed_files = logfile.read_failing_log()
13
+
14
+ # Run tests singularly
15
+ flaky_tests = FlakyStats::FlakyTests.new(logfile: LOGFILE)
16
+ real_flaky_tests = flaky_tests.run(failed_files)
17
+
18
+ # Write out the log file of real flaky tests
19
+ logfile.write_flaky_stats(real_flaky_tests)
20
+
21
+ # Display summaries
22
+ summary = FlakyStats::Summary.new(failing_log: FAILING_LOG, logfile: LOGFILE)
23
+ summary.display_error_summary()
24
+ summary.display_flaky_summary()
25
+ end
26
+ end
metadata ADDED
@@ -0,0 +1,117 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: flaky_stats
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Michael Pope
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2017-05-12 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.14'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.14'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: timecop
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.8.1
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '='
67
+ - !ruby/object:Gem::Version
68
+ version: 0.8.1
69
+ description: Flaky stats will rerun broken tests from your parallel tests and if they
70
+ pass then they must be flaky so it records them.
71
+ email:
72
+ - michael@dtcorp.com.au
73
+ executables: []
74
+ extensions: []
75
+ extra_rdoc_files: []
76
+ files:
77
+ - ".gitignore"
78
+ - ".rspec"
79
+ - ".travis.yml"
80
+ - Gemfile
81
+ - README.md
82
+ - Rakefile
83
+ - bin/console
84
+ - bin/setup
85
+ - flaky_stats.gemspec
86
+ - lib/flaky_stats.rb
87
+ - lib/flaky_stats/flaky_tests.rb
88
+ - lib/flaky_stats/logfile.rb
89
+ - lib/flaky_stats/railtie.rb
90
+ - lib/flaky_stats/summary.rb
91
+ - lib/flaky_stats/version.rb
92
+ - lib/tasks/flaky_stats.rake
93
+ homepage: http://github.com/map7/flaky_stats
94
+ licenses:
95
+ - MIT
96
+ metadata: {}
97
+ post_install_message:
98
+ rdoc_options: []
99
+ require_paths:
100
+ - lib
101
+ required_ruby_version: !ruby/object:Gem::Requirement
102
+ requirements:
103
+ - - ">="
104
+ - !ruby/object:Gem::Version
105
+ version: '0'
106
+ required_rubygems_version: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ requirements: []
112
+ rubyforge_project:
113
+ rubygems_version: 2.6.11
114
+ signing_key:
115
+ specification_version: 4
116
+ summary: Records flaky tests
117
+ test_files: []