flaky_stats 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.rspec +2 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/README.md +41 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/flaky_stats.gemspec +28 -0
- data/lib/flaky_stats/flaky_tests.rb +37 -0
- data/lib/flaky_stats/logfile.rb +46 -0
- data/lib/flaky_stats/railtie.rb +11 -0
- data/lib/flaky_stats/summary.rb +41 -0
- data/lib/flaky_stats/version.rb +3 -0
- data/lib/flaky_stats.rb +8 -0
- data/lib/tasks/flaky_stats.rake +26 -0
- metadata +117 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2f46e85bcc221c806457d3f481f5ffa56c893cd2
|
4
|
+
data.tar.gz: 823883da6f051e786ec1f894a37fb3a9e91641d9
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 31002d561575e96e7d9d024f10855b04b50ff0003219883976969d04a2de48b4e6a6430fdde2327f8ea53434cf931c44aeed5279e5806dcedb032369b96da826
|
7
|
+
data.tar.gz: 05c5ee0c41c6c4a070b07e4258d4fdb6d7cdcf0ddc1b7823c60295407689de0804745e08b64309de0b4d47d6ee54634f2a0d0d4cad2b10cf3a0c6ded0c1ec1d5
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# FlakyStats
|
2
|
+
|
3
|
+
Collects broken tests from a parallel failing test log, uses them to rerun the flaky tests and if they pass whilst running them on their own add them to the flaky_stats.log file.
|
4
|
+
|
5
|
+
At the end of the tests we can display a flaky stats summary which should display the top flaky tests in order of the most flakiest.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'flaky_stats'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install flaky_stats
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
$ rake parallel:rerun
|
26
|
+
|
27
|
+
## Development
|
28
|
+
|
29
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
30
|
+
|
31
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
32
|
+
|
33
|
+
## Contributing
|
34
|
+
|
35
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/map7/flaky_stats.
|
36
|
+
|
37
|
+
## Ref
|
38
|
+
|
39
|
+
http://blog.nathanhumbert.com/2010/02/rails-3-loading-rake-tasks-from-gem.html
|
40
|
+
|
41
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "flaky_stats"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/flaky_stats.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'flaky_stats/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "flaky_stats"
|
8
|
+
spec.version = FlakyStats::VERSION
|
9
|
+
spec.authors = ["Michael Pope"]
|
10
|
+
spec.email = ["michael@dtcorp.com.au"]
|
11
|
+
|
12
|
+
spec.summary = %q{Records flaky tests}
|
13
|
+
spec.description = %q{Flaky stats will rerun broken tests from your parallel tests and if they pass then they must be flaky so it records them.}
|
14
|
+
spec.homepage = "http://github.com/map7/flaky_stats"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
spec.bindir = "exe"
|
21
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_development_dependency "bundler", "~> 1.14"
|
25
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
26
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
27
|
+
spec.add_development_dependency "timecop", "= 0.8.1"
|
28
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module FlakyStats
|
2
|
+
class FlakyTests
|
3
|
+
|
4
|
+
def initialize(options = {})
|
5
|
+
@logfile = options[:logfile]
|
6
|
+
end
|
7
|
+
|
8
|
+
def form_data(failed_file = {})
|
9
|
+
return [Time.now,failed_file[:filename],failed_file[:lineno],1].join ","
|
10
|
+
end
|
11
|
+
|
12
|
+
# Run each failing test singularly and return a list of flaky tests.
|
13
|
+
def run(failed_files)
|
14
|
+
real_flaky_tests = []
|
15
|
+
|
16
|
+
puts "\n\n"
|
17
|
+
puts "-------------------------------------------------------------------"
|
18
|
+
puts "Rerunning failing tests in single thread"
|
19
|
+
puts "-------------------------------------------------------------------"
|
20
|
+
|
21
|
+
# sleep 10 # Settle everything down.
|
22
|
+
|
23
|
+
# Run all failing tests separately with '--format doc' on the end.
|
24
|
+
failed_files.each do |failed_file|
|
25
|
+
# sleep 2 # Settle everything down.
|
26
|
+
status = system("rspec --format doc #{failed_file[:filename]}")
|
27
|
+
|
28
|
+
# This is a flaky test only, so record it
|
29
|
+
if status == true
|
30
|
+
real_flaky_tests << form_data(failed_file)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
return real_flaky_tests
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# Logfile
|
2
|
+
#
|
3
|
+
# Deals with reading failing logs, remove old log files and
|
4
|
+
# logging the flaky stats
|
5
|
+
|
6
|
+
module FlakyStats
|
7
|
+
class LogFile
|
8
|
+
def initialize(options = {})
|
9
|
+
@failing_log = options[:failing_log]
|
10
|
+
@logfile = options[:logfile]
|
11
|
+
end
|
12
|
+
|
13
|
+
def get_error_info(line)
|
14
|
+
information= line.partition(/\.\/.*.rb/)
|
15
|
+
filename = information[1]
|
16
|
+
lineno = information[2].partition(/:/)[2].to_i
|
17
|
+
return {filename: filename, lineno: lineno}
|
18
|
+
end
|
19
|
+
|
20
|
+
# Read the failing log from our test stack
|
21
|
+
def read_failing_log()
|
22
|
+
failed_files = []
|
23
|
+
|
24
|
+
# Read in the file
|
25
|
+
file = File.readlines(@failing_log)
|
26
|
+
|
27
|
+
# Get lines which begin with rspec
|
28
|
+
file.each do |line|
|
29
|
+
if line =~ /rspec \.\//
|
30
|
+
# Get the file name only
|
31
|
+
failed_files << get_error_info(line)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
return failed_files
|
36
|
+
end
|
37
|
+
|
38
|
+
# Log flaky stats
|
39
|
+
def write_flaky_stats(real_flaky_tests)
|
40
|
+
File.open(@logfile, "a") do |log|
|
41
|
+
real_flaky_tests.each {|data| log.puts data}
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# Summary
|
2
|
+
#
|
3
|
+
# Calculate and display summary information for errors and flaky stats
|
4
|
+
|
5
|
+
module FlakyStats
|
6
|
+
class Summary
|
7
|
+
def initialize(options)
|
8
|
+
@failing_log = options[:failing_log]
|
9
|
+
@logfile = options[:logfile]
|
10
|
+
end
|
11
|
+
|
12
|
+
def display_error_summary()
|
13
|
+
puts "\n\n"
|
14
|
+
puts "-------------------------------------------------------------------"
|
15
|
+
puts "Errors summary"
|
16
|
+
puts "-------------------------------------------------------------------"
|
17
|
+
|
18
|
+
system("cat #{@failing_log}")
|
19
|
+
end
|
20
|
+
|
21
|
+
def calc_flaky_summary
|
22
|
+
sum=Hash.new(0)
|
23
|
+
CSV.foreach(@logfile) do |row|
|
24
|
+
sum[row[1]]+=row[3].to_i
|
25
|
+
end
|
26
|
+
return sum
|
27
|
+
end
|
28
|
+
|
29
|
+
def display_flaky_summary
|
30
|
+
puts "\n\n"
|
31
|
+
puts "-------------------------------------------------------------------"
|
32
|
+
puts "Flaky summary"
|
33
|
+
puts "-------------------------------------------------------------------"
|
34
|
+
|
35
|
+
calc_flaky_summary.each do |k,v|
|
36
|
+
puts "#{k} = #{v}" if v > 1
|
37
|
+
end
|
38
|
+
puts "\n"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/flaky_stats.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
LOGFILE=Rails.root + "log/flaky_tests.log"
|
4
|
+
FAILING_LOG=Rails.root + "tmp/failing_specs.log"
|
5
|
+
|
6
|
+
desc "Rerun failing parallel tests in a single thread"
|
7
|
+
task :flaky_stats => :environment do |t|
|
8
|
+
if File.exist?(FAILING_LOG)
|
9
|
+
|
10
|
+
# Read failing log
|
11
|
+
logfile = FlakyStats::LogFile.new(failing_log: FAILING_LOG, logfile: LOGFILE)
|
12
|
+
failed_files = logfile.read_failing_log()
|
13
|
+
|
14
|
+
# Run tests singularly
|
15
|
+
flaky_tests = FlakyStats::FlakyTests.new(logfile: LOGFILE)
|
16
|
+
real_flaky_tests = flaky_tests.run(failed_files)
|
17
|
+
|
18
|
+
# Write out the log file of real flaky tests
|
19
|
+
logfile.write_flaky_stats(real_flaky_tests)
|
20
|
+
|
21
|
+
# Display summaries
|
22
|
+
summary = FlakyStats::Summary.new(failing_log: FAILING_LOG, logfile: LOGFILE)
|
23
|
+
summary.display_error_summary()
|
24
|
+
summary.display_flaky_summary()
|
25
|
+
end
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: flaky_stats
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Michael Pope
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-05-12 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.14'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.14'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: timecop
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.8.1
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 0.8.1
|
69
|
+
description: Flaky stats will rerun broken tests from your parallel tests and if they
|
70
|
+
pass then they must be flaky so it records them.
|
71
|
+
email:
|
72
|
+
- michael@dtcorp.com.au
|
73
|
+
executables: []
|
74
|
+
extensions: []
|
75
|
+
extra_rdoc_files: []
|
76
|
+
files:
|
77
|
+
- ".gitignore"
|
78
|
+
- ".rspec"
|
79
|
+
- ".travis.yml"
|
80
|
+
- Gemfile
|
81
|
+
- README.md
|
82
|
+
- Rakefile
|
83
|
+
- bin/console
|
84
|
+
- bin/setup
|
85
|
+
- flaky_stats.gemspec
|
86
|
+
- lib/flaky_stats.rb
|
87
|
+
- lib/flaky_stats/flaky_tests.rb
|
88
|
+
- lib/flaky_stats/logfile.rb
|
89
|
+
- lib/flaky_stats/railtie.rb
|
90
|
+
- lib/flaky_stats/summary.rb
|
91
|
+
- lib/flaky_stats/version.rb
|
92
|
+
- lib/tasks/flaky_stats.rake
|
93
|
+
homepage: http://github.com/map7/flaky_stats
|
94
|
+
licenses:
|
95
|
+
- MIT
|
96
|
+
metadata: {}
|
97
|
+
post_install_message:
|
98
|
+
rdoc_options: []
|
99
|
+
require_paths:
|
100
|
+
- lib
|
101
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
102
|
+
requirements:
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
version: '0'
|
106
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
requirements: []
|
112
|
+
rubyforge_project:
|
113
|
+
rubygems_version: 2.6.11
|
114
|
+
signing_key:
|
115
|
+
specification_version: 4
|
116
|
+
summary: Records flaky tests
|
117
|
+
test_files: []
|