flaky_stats 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +9 -0
- data/flaky_stats.gemspec +1 -0
- data/lib/flaky_stats/flaky_tests.rb +7 -13
- data/lib/flaky_stats/output.rb +15 -0
- data/lib/flaky_stats/summary.rb +43 -13
- data/lib/flaky_stats/version.rb +1 -1
- data/lib/flaky_stats.rb +1 -0
- data/lib/tasks/flaky_stats.rake +4 -1
- metadata +18 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e1d9907d8e94b4251659df7cc93b064b0619c54e
|
4
|
+
data.tar.gz: 1328f215192ff1f1032ceebd4d6caf2622dec1e0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 60136e0c87ce993272b475a8efc5f7571c436cd2e3fe103347da93d6d8aed3192c5f22861902c766f88e700bd60a232b0fc982ffb3d3ec2beaa7809f2df7cff1
|
7
|
+
data.tar.gz: a02d3fdc30f4395976e6e254aba34a9ca785f38a99994d4dc8bbd1547a810b60bc1c526e8438d258aff72cfc931d1c79c4899efd30b240e759ef3f0900c77507
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -34,6 +34,15 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
34
34
|
|
35
35
|
Bug reports and pull requests are welcome on GitHub at https://github.com/map7/flaky_stats.
|
36
36
|
|
37
|
+
## TODO
|
38
|
+
|
39
|
+
- Create a rake task to show the flaky test summary
|
40
|
+
- Get working with more than parallel tests
|
41
|
+
- Test on CI environments
|
42
|
+
- Do away with the ',1' on the end of our data
|
43
|
+
This 1 on the end of each data line in the file is for a quick calculation, if this can be done without this then we can simplify our data.
|
44
|
+
- More tests
|
45
|
+
|
37
46
|
## Ref
|
38
47
|
|
39
48
|
http://blog.nathanhumbert.com/2010/02/rails-3-loading-rake-tasks-from-gem.html
|
data/flaky_stats.gemspec
CHANGED
@@ -1,6 +1,9 @@
|
|
1
|
+
require_relative 'output'
|
2
|
+
|
1
3
|
module FlakyStats
|
2
4
|
class FlakyTests
|
3
|
-
|
5
|
+
include Output
|
6
|
+
|
4
7
|
def initialize(options = {})
|
5
8
|
@logfile = options[:logfile]
|
6
9
|
end
|
@@ -10,25 +13,16 @@ module FlakyStats
|
|
10
13
|
end
|
11
14
|
|
12
15
|
# Run each failing test singularly and return a list of flaky tests.
|
13
|
-
def run(failed_files)
|
16
|
+
def run(failed_files = [])
|
14
17
|
real_flaky_tests = []
|
15
|
-
|
16
|
-
puts "\n\n"
|
17
|
-
puts "-------------------------------------------------------------------"
|
18
|
-
puts "Rerunning failing tests in single thread"
|
19
|
-
puts "-------------------------------------------------------------------"
|
20
|
-
|
21
|
-
# sleep 10 # Settle everything down.
|
18
|
+
heading("Rerunning failing tests in single thread")
|
22
19
|
|
23
20
|
# Run all failing tests separately with '--format doc' on the end.
|
24
21
|
failed_files.each do |failed_file|
|
25
|
-
# sleep 2 # Settle everything down.
|
26
22
|
status = system("rspec --format doc #{failed_file[:filename]}")
|
27
23
|
|
28
24
|
# This is a flaky test only, so record it
|
29
|
-
if status == true
|
30
|
-
real_flaky_tests << form_data(failed_file)
|
31
|
-
end
|
25
|
+
real_flaky_tests << form_data(failed_file) if status == true
|
32
26
|
end
|
33
27
|
|
34
28
|
return real_flaky_tests
|
data/lib/flaky_stats/summary.rb
CHANGED
@@ -1,20 +1,22 @@
|
|
1
|
+
require_relative "output"
|
2
|
+
require 'csv'
|
3
|
+
|
1
4
|
# Summary
|
2
5
|
#
|
3
6
|
# Calculate and display summary information for errors and flaky stats
|
4
7
|
|
5
8
|
module FlakyStats
|
6
9
|
class Summary
|
10
|
+
include Output
|
11
|
+
DEFAULT_ROLLOVER_DAYS = 14
|
12
|
+
|
7
13
|
def initialize(options)
|
8
14
|
@failing_log = options[:failing_log]
|
9
15
|
@logfile = options[:logfile]
|
10
16
|
end
|
11
17
|
|
12
18
|
def display_error_summary()
|
13
|
-
|
14
|
-
puts "-------------------------------------------------------------------"
|
15
|
-
puts "Errors summary"
|
16
|
-
puts "-------------------------------------------------------------------"
|
17
|
-
|
19
|
+
heading "Errors summary"
|
18
20
|
system("cat #{@failing_log}")
|
19
21
|
end
|
20
22
|
|
@@ -23,19 +25,47 @@ module FlakyStats
|
|
23
25
|
CSV.foreach(@logfile) do |row|
|
24
26
|
sum[row[1]]+=row[3].to_i
|
25
27
|
end
|
26
|
-
return sum
|
28
|
+
return order_stats(sum)
|
27
29
|
end
|
28
30
|
|
29
31
|
def display_flaky_summary
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
puts "-------------------------------------------------------------------"
|
34
|
-
|
35
|
-
calc_flaky_summary.each do |k,v|
|
36
|
-
puts "#{k} = #{v}" if v > 1
|
32
|
+
heading "Flaky summary"
|
33
|
+
reject_low_flaky_tests(calc_flaky_summary).each do |k,v|
|
34
|
+
puts "#{v}\t#{k}"
|
37
35
|
end
|
38
36
|
puts "\n"
|
39
37
|
end
|
38
|
+
|
39
|
+
def rollover(days = DEFAULT_ROLLOVER_DAYS)
|
40
|
+
lines = File.readlines(@logfile)
|
41
|
+
lines.reject{|line|
|
42
|
+
flaky_date = Time.parse(line.split(",")[0]).to_date
|
43
|
+
cutoff = Date.today - days
|
44
|
+
flaky_date < cutoff
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
def rollover_and_write(options = {})
|
49
|
+
days = options[:days] || DEFAULT_ROLLOVER_DAYS
|
50
|
+
output = options[:output] || "#{@logfile}.tmp"
|
51
|
+
|
52
|
+
File.open(output, "w") {|file|
|
53
|
+
rollover(days).each do |flaky|
|
54
|
+
file.write flaky
|
55
|
+
end
|
56
|
+
}
|
57
|
+
|
58
|
+
# Overwrite log file.
|
59
|
+
`cp #{@logfile} #{@logfile}.old`
|
60
|
+
`cp #{@logfile}.tmp #{@logfile}`
|
61
|
+
end
|
62
|
+
|
63
|
+
def reject_low_flaky_tests(data)
|
64
|
+
data.reject{|k,v| v <= 1}
|
65
|
+
end
|
66
|
+
|
67
|
+
def order_stats(hash)
|
68
|
+
hash.sort_by{|key,value| value}.reverse.to_h
|
69
|
+
end
|
40
70
|
end
|
41
71
|
end
|
data/lib/flaky_stats/version.rb
CHANGED
data/lib/flaky_stats.rb
CHANGED
data/lib/tasks/flaky_stats.rake
CHANGED
@@ -17,10 +17,13 @@ task :flaky_stats => :environment do |t|
|
|
17
17
|
|
18
18
|
# Write out the log file of real flaky tests
|
19
19
|
logfile.write_flaky_stats(real_flaky_tests)
|
20
|
-
|
20
|
+
|
21
21
|
# Display summaries
|
22
22
|
summary = FlakyStats::Summary.new(failing_log: FAILING_LOG, logfile: LOGFILE)
|
23
23
|
summary.display_error_summary()
|
24
24
|
summary.display_flaky_summary()
|
25
|
+
|
26
|
+
# Rollover logfile
|
27
|
+
summary.rollover_and_write()
|
25
28
|
end
|
26
29
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flaky_stats
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Pope
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - '='
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: 0.8.1
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: byebug
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
description: Flaky stats will rerun broken tests from your parallel tests and if they
|
70
84
|
pass then they must be flaky so it records them.
|
71
85
|
email:
|
@@ -86,6 +100,7 @@ files:
|
|
86
100
|
- lib/flaky_stats.rb
|
87
101
|
- lib/flaky_stats/flaky_tests.rb
|
88
102
|
- lib/flaky_stats/logfile.rb
|
103
|
+
- lib/flaky_stats/output.rb
|
89
104
|
- lib/flaky_stats/railtie.rb
|
90
105
|
- lib/flaky_stats/summary.rb
|
91
106
|
- lib/flaky_stats/version.rb
|
@@ -110,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
110
125
|
version: '0'
|
111
126
|
requirements: []
|
112
127
|
rubyforge_project:
|
113
|
-
rubygems_version: 2.
|
128
|
+
rubygems_version: 2.5.2
|
114
129
|
signing_key:
|
115
130
|
specification_version: 4
|
116
131
|
summary: Records flaky tests
|