rspec_log_formatter 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -13
- data/.gitignore +1 -1
- data/.ruby-version +1 -1
- data/README.md +53 -7
- data/bin/rspec_log_formatter +30 -0
- data/dummy/.gitignore +16 -0
- data/dummy/.rspec +1 -0
- data/dummy/Gemfile +47 -0
- data/dummy/README.rdoc +28 -0
- data/dummy/Rakefile +6 -0
- data/dummy/app/assets/images/.keep +0 -0
- data/dummy/app/assets/javascripts/application.js +16 -0
- data/dummy/app/assets/stylesheets/application.css +13 -0
- data/dummy/app/controllers/application_controller.rb +5 -0
- data/dummy/app/controllers/concerns/.keep +0 -0
- data/dummy/app/helpers/application_helper.rb +2 -0
- data/dummy/app/mailers/.keep +0 -0
- data/dummy/app/models/.keep +0 -0
- data/dummy/app/models/concerns/.keep +0 -0
- data/dummy/app/views/layouts/application.html.erb +14 -0
- data/dummy/bin/bundle +3 -0
- data/dummy/bin/rails +4 -0
- data/dummy/bin/rake +4 -0
- data/dummy/config/application.rb +23 -0
- data/dummy/config/boot.rb +4 -0
- data/dummy/config/database.yml +25 -0
- data/dummy/config/environment.rb +5 -0
- data/dummy/config/environments/development.rb +29 -0
- data/dummy/config/environments/production.rb +80 -0
- data/dummy/config/environments/test.rb +36 -0
- data/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/dummy/config/initializers/filter_parameter_logging.rb +4 -0
- data/dummy/config/initializers/inflections.rb +16 -0
- data/dummy/config/initializers/mime_types.rb +5 -0
- data/dummy/config/initializers/secret_token.rb +12 -0
- data/dummy/config/initializers/session_store.rb +3 -0
- data/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/dummy/config/locales/en.yml +23 -0
- data/dummy/config/routes.rb +56 -0
- data/dummy/config.ru +4 -0
- data/dummy/db/schema.rb +16 -0
- data/dummy/db/seeds.rb +7 -0
- data/dummy/lib/assets/.keep +0 -0
- data/dummy/lib/tasks/.keep +0 -0
- data/dummy/log/.keep +0 -0
- data/dummy/public/404.html +58 -0
- data/dummy/public/422.html +58 -0
- data/dummy/public/500.html +57 -0
- data/dummy/public/favicon.ico +0 -0
- data/dummy/public/robots.txt +5 -0
- data/dummy/spec/dummy_spec.rb +7 -0
- data/dummy/spec/spec_helper.rb +46 -0
- data/dummy/test/controllers/.keep +0 -0
- data/dummy/test/fixtures/.keep +0 -0
- data/dummy/test/helpers/.keep +0 -0
- data/dummy/test/integration/.keep +0 -0
- data/dummy/test/mailers/.keep +0 -0
- data/dummy/test/models/.keep +0 -0
- data/dummy/test/test_helper.rb +15 -0
- data/dummy/vendor/assets/javascripts/.keep +0 -0
- data/dummy/vendor/assets/stylesheets/.keep +0 -0
- data/lib/rspec_log_formatter/analysis/analyzer.rb +15 -50
- data/lib/rspec_log_formatter/analysis/pretty_printer.rb +10 -6
- data/lib/rspec_log_formatter/analysis/result.rb +8 -4
- data/lib/rspec_log_formatter/analysis/score.rb +42 -6
- data/lib/rspec_log_formatter/analyzer_formatter.rb +18 -6
- data/lib/rspec_log_formatter/formatter.rb +19 -12
- data/lib/rspec_log_formatter/history_manager.rb +46 -0
- data/lib/rspec_log_formatter/javascripts/chartkick.js +678 -0
- data/lib/rspec_log_formatter/performance_analyzer.rb +40 -0
- data/lib/rspec_log_formatter/templates/charts.html.erb +11 -0
- data/lib/rspec_log_formatter/version.rb +1 -1
- data/lib/rspec_log_formatter.rb +2 -1
- data/rspec_log_formatter.gemspec +1 -0
- data/spec/fixtures/test_slowing_down_over_time.history +7 -0
- data/spec/fixtures/varying_flakiness.history +9 -9
- data/spec/lib/rspec_log_analyzer/analysis/analyzer_spec.rb +25 -21
- data/spec/lib/rspec_log_analyzer/analysis/pretty_printer_spec.rb +10 -8
- data/spec/lib/rspec_log_analyzer/analyzer_formatter_spec.rb +3 -3
- data/spec/lib/rspec_log_analyzer/formatter_spec.rb +16 -18
- data/spec/lib/rspec_log_analyzer/history_manager_spec.rb +16 -0
- data/spec/lib/rspec_log_analyzer/performance_analyzer_spec.rb +16 -0
- data/specs.sh +4 -0
- metadata +95 -12
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
Dummy::Application.routes.draw do
|
|
2
|
+
# The priority is based upon order of creation: first created -> highest priority.
|
|
3
|
+
# See how all your routes lay out with "rake routes".
|
|
4
|
+
|
|
5
|
+
# You can have the root of your site routed with "root"
|
|
6
|
+
# root 'welcome#index'
|
|
7
|
+
|
|
8
|
+
# Example of regular route:
|
|
9
|
+
# get 'products/:id' => 'catalog#view'
|
|
10
|
+
|
|
11
|
+
# Example of named route that can be invoked with purchase_url(id: product.id)
|
|
12
|
+
# get 'products/:id/purchase' => 'catalog#purchase', as: :purchase
|
|
13
|
+
|
|
14
|
+
# Example resource route (maps HTTP verbs to controller actions automatically):
|
|
15
|
+
# resources :products
|
|
16
|
+
|
|
17
|
+
# Example resource route with options:
|
|
18
|
+
# resources :products do
|
|
19
|
+
# member do
|
|
20
|
+
# get 'short'
|
|
21
|
+
# post 'toggle'
|
|
22
|
+
# end
|
|
23
|
+
#
|
|
24
|
+
# collection do
|
|
25
|
+
# get 'sold'
|
|
26
|
+
# end
|
|
27
|
+
# end
|
|
28
|
+
|
|
29
|
+
# Example resource route with sub-resources:
|
|
30
|
+
# resources :products do
|
|
31
|
+
# resources :comments, :sales
|
|
32
|
+
# resource :seller
|
|
33
|
+
# end
|
|
34
|
+
|
|
35
|
+
# Example resource route with more complex sub-resources:
|
|
36
|
+
# resources :products do
|
|
37
|
+
# resources :comments
|
|
38
|
+
# resources :sales do
|
|
39
|
+
# get 'recent', on: :collection
|
|
40
|
+
# end
|
|
41
|
+
# end
|
|
42
|
+
|
|
43
|
+
# Example resource route with concerns:
|
|
44
|
+
# concern :toggleable do
|
|
45
|
+
# post 'toggle'
|
|
46
|
+
# end
|
|
47
|
+
# resources :posts, concerns: :toggleable
|
|
48
|
+
# resources :photos, concerns: :toggleable
|
|
49
|
+
|
|
50
|
+
# Example resource route within a namespace:
|
|
51
|
+
# namespace :admin do
|
|
52
|
+
# # Directs /admin/products/* to Admin::ProductsController
|
|
53
|
+
# # (app/controllers/admin/products_controller.rb)
|
|
54
|
+
# resources :products
|
|
55
|
+
# end
|
|
56
|
+
end
|
data/dummy/config.ru
ADDED
data/dummy/db/schema.rb
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# This file is auto-generated from the current state of the database. Instead
|
|
3
|
+
# of editing this file, please use the migrations feature of Active Record to
|
|
4
|
+
# incrementally modify your database, and then regenerate this schema definition.
|
|
5
|
+
#
|
|
6
|
+
# Note that this schema.rb definition is the authoritative source for your
|
|
7
|
+
# database schema. If you need to create the application database on another
|
|
8
|
+
# system, you should be using db:schema:load, not running all the migrations
|
|
9
|
+
# from scratch. The latter is a flawed and unsustainable approach (the more migrations
|
|
10
|
+
# you'll amass, the slower it'll run and the greater likelihood for issues).
|
|
11
|
+
#
|
|
12
|
+
# It's strongly recommended that you check this file into your version control system.
|
|
13
|
+
|
|
14
|
+
ActiveRecord::Schema.define(version: 0) do
|
|
15
|
+
|
|
16
|
+
end
|
data/dummy/db/seeds.rb
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# This file should contain all the record creation needed to seed the database with its default values.
|
|
2
|
+
# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
|
|
3
|
+
#
|
|
4
|
+
# Examples:
|
|
5
|
+
#
|
|
6
|
+
# cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }])
|
|
7
|
+
# Mayor.create(name: 'Emanuel', city: cities.first)
|
|
File without changes
|
|
File without changes
|
data/dummy/log/.keep
ADDED
|
File without changes
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>The page you were looking for doesn't exist (404)</title>
|
|
5
|
+
<style>
|
|
6
|
+
body {
|
|
7
|
+
background-color: #EFEFEF;
|
|
8
|
+
color: #2E2F30;
|
|
9
|
+
text-align: center;
|
|
10
|
+
font-family: arial, sans-serif;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
div.dialog {
|
|
14
|
+
width: 25em;
|
|
15
|
+
margin: 4em auto 0 auto;
|
|
16
|
+
border: 1px solid #CCC;
|
|
17
|
+
border-right-color: #999;
|
|
18
|
+
border-left-color: #999;
|
|
19
|
+
border-bottom-color: #BBB;
|
|
20
|
+
border-top: #B00100 solid 4px;
|
|
21
|
+
border-top-left-radius: 9px;
|
|
22
|
+
border-top-right-radius: 9px;
|
|
23
|
+
background-color: white;
|
|
24
|
+
padding: 7px 4em 0 4em;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
h1 {
|
|
28
|
+
font-size: 100%;
|
|
29
|
+
color: #730E15;
|
|
30
|
+
line-height: 1.5em;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
body > p {
|
|
34
|
+
width: 33em;
|
|
35
|
+
margin: 0 auto 1em;
|
|
36
|
+
padding: 1em 0;
|
|
37
|
+
background-color: #F7F7F7;
|
|
38
|
+
border: 1px solid #CCC;
|
|
39
|
+
border-right-color: #999;
|
|
40
|
+
border-bottom-color: #999;
|
|
41
|
+
border-bottom-left-radius: 4px;
|
|
42
|
+
border-bottom-right-radius: 4px;
|
|
43
|
+
border-top-color: #DADADA;
|
|
44
|
+
color: #666;
|
|
45
|
+
box-shadow:0 3px 8px rgba(50, 50, 50, 0.17);
|
|
46
|
+
}
|
|
47
|
+
</style>
|
|
48
|
+
</head>
|
|
49
|
+
|
|
50
|
+
<body>
|
|
51
|
+
<!-- This file lives in public/404.html -->
|
|
52
|
+
<div class="dialog">
|
|
53
|
+
<h1>The page you were looking for doesn't exist.</h1>
|
|
54
|
+
<p>You may have mistyped the address or the page may have moved.</p>
|
|
55
|
+
</div>
|
|
56
|
+
<p>If you are the application owner check the logs for more information.</p>
|
|
57
|
+
</body>
|
|
58
|
+
</html>
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>The change you wanted was rejected (422)</title>
|
|
5
|
+
<style>
|
|
6
|
+
body {
|
|
7
|
+
background-color: #EFEFEF;
|
|
8
|
+
color: #2E2F30;
|
|
9
|
+
text-align: center;
|
|
10
|
+
font-family: arial, sans-serif;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
div.dialog {
|
|
14
|
+
width: 25em;
|
|
15
|
+
margin: 4em auto 0 auto;
|
|
16
|
+
border: 1px solid #CCC;
|
|
17
|
+
border-right-color: #999;
|
|
18
|
+
border-left-color: #999;
|
|
19
|
+
border-bottom-color: #BBB;
|
|
20
|
+
border-top: #B00100 solid 4px;
|
|
21
|
+
border-top-left-radius: 9px;
|
|
22
|
+
border-top-right-radius: 9px;
|
|
23
|
+
background-color: white;
|
|
24
|
+
padding: 7px 4em 0 4em;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
h1 {
|
|
28
|
+
font-size: 100%;
|
|
29
|
+
color: #730E15;
|
|
30
|
+
line-height: 1.5em;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
body > p {
|
|
34
|
+
width: 33em;
|
|
35
|
+
margin: 0 auto 1em;
|
|
36
|
+
padding: 1em 0;
|
|
37
|
+
background-color: #F7F7F7;
|
|
38
|
+
border: 1px solid #CCC;
|
|
39
|
+
border-right-color: #999;
|
|
40
|
+
border-bottom-color: #999;
|
|
41
|
+
border-bottom-left-radius: 4px;
|
|
42
|
+
border-bottom-right-radius: 4px;
|
|
43
|
+
border-top-color: #DADADA;
|
|
44
|
+
color: #666;
|
|
45
|
+
box-shadow:0 3px 8px rgba(50, 50, 50, 0.17);
|
|
46
|
+
}
|
|
47
|
+
</style>
|
|
48
|
+
</head>
|
|
49
|
+
|
|
50
|
+
<body>
|
|
51
|
+
<!-- This file lives in public/422.html -->
|
|
52
|
+
<div class="dialog">
|
|
53
|
+
<h1>The change you wanted was rejected.</h1>
|
|
54
|
+
<p>Maybe you tried to change something you didn't have access to.</p>
|
|
55
|
+
</div>
|
|
56
|
+
<p>If you are the application owner check the logs for more information.</p>
|
|
57
|
+
</body>
|
|
58
|
+
</html>
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>We're sorry, but something went wrong (500)</title>
|
|
5
|
+
<style>
|
|
6
|
+
body {
|
|
7
|
+
background-color: #EFEFEF;
|
|
8
|
+
color: #2E2F30;
|
|
9
|
+
text-align: center;
|
|
10
|
+
font-family: arial, sans-serif;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
div.dialog {
|
|
14
|
+
width: 25em;
|
|
15
|
+
margin: 4em auto 0 auto;
|
|
16
|
+
border: 1px solid #CCC;
|
|
17
|
+
border-right-color: #999;
|
|
18
|
+
border-left-color: #999;
|
|
19
|
+
border-bottom-color: #BBB;
|
|
20
|
+
border-top: #B00100 solid 4px;
|
|
21
|
+
border-top-left-radius: 9px;
|
|
22
|
+
border-top-right-radius: 9px;
|
|
23
|
+
background-color: white;
|
|
24
|
+
padding: 7px 4em 0 4em;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
h1 {
|
|
28
|
+
font-size: 100%;
|
|
29
|
+
color: #730E15;
|
|
30
|
+
line-height: 1.5em;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
body > p {
|
|
34
|
+
width: 33em;
|
|
35
|
+
margin: 0 auto 1em;
|
|
36
|
+
padding: 1em 0;
|
|
37
|
+
background-color: #F7F7F7;
|
|
38
|
+
border: 1px solid #CCC;
|
|
39
|
+
border-right-color: #999;
|
|
40
|
+
border-bottom-color: #999;
|
|
41
|
+
border-bottom-left-radius: 4px;
|
|
42
|
+
border-bottom-right-radius: 4px;
|
|
43
|
+
border-top-color: #DADADA;
|
|
44
|
+
color: #666;
|
|
45
|
+
box-shadow:0 3px 8px rgba(50, 50, 50, 0.17);
|
|
46
|
+
}
|
|
47
|
+
</style>
|
|
48
|
+
</head>
|
|
49
|
+
|
|
50
|
+
<body>
|
|
51
|
+
<!-- This file lives in public/500.html -->
|
|
52
|
+
<div class="dialog">
|
|
53
|
+
<h1>We're sorry, but something went wrong.</h1>
|
|
54
|
+
</div>
|
|
55
|
+
<p>If you are the application owner check the logs for more information.</p>
|
|
56
|
+
</body>
|
|
57
|
+
</html>
|
|
File without changes
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# This file is copied to spec/ when you run 'rails generate rspec:install'
|
|
2
|
+
ENV["RAILS_ENV"] ||= 'test'
|
|
3
|
+
require File.expand_path("../../config/environment", __FILE__)
|
|
4
|
+
require 'rspec/rails'
|
|
5
|
+
require 'rspec/autorun'
|
|
6
|
+
|
|
7
|
+
# Requires supporting ruby files with custom matchers and macros, etc,
|
|
8
|
+
# in spec/support/ and its subdirectories.
|
|
9
|
+
Dir[Rails.root.join("spec/support/**/*.rb")].each { |f| require f }
|
|
10
|
+
# Checks for pending migrations before tests are run.
|
|
11
|
+
# If you are not using ActiveRecord, you can remove this line.
|
|
12
|
+
ActiveRecord::Migration.check_pending! if defined?(ActiveRecord::Migration)
|
|
13
|
+
|
|
14
|
+
RSpec.configure do |config|
|
|
15
|
+
# ## Mock Framework
|
|
16
|
+
#
|
|
17
|
+
# If you prefer to use mocha, flexmock or RR, uncomment the appropriate line:
|
|
18
|
+
#
|
|
19
|
+
# config.mock_with :mocha
|
|
20
|
+
# config.mock_with :flexmock
|
|
21
|
+
# config.mock_with :rr
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
config.formatters << RspecLogFormatter::Formatter::Factory.new.build
|
|
25
|
+
config.add_formatter(:progress)
|
|
26
|
+
config.formatters << RspecLogFormatter::AnalyzerFormatter::Factory.new.build($stdout)
|
|
27
|
+
|
|
28
|
+
# Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
|
|
29
|
+
config.fixture_path = "#{::Rails.root}/spec/fixtures"
|
|
30
|
+
|
|
31
|
+
# If you're not using ActiveRecord, or you'd prefer not to run each of your
|
|
32
|
+
# examples within a transaction, remove the following line or assign false
|
|
33
|
+
# instead of true.
|
|
34
|
+
config.use_transactional_fixtures = true
|
|
35
|
+
|
|
36
|
+
# If true, the base class of anonymous controllers will be inferred
|
|
37
|
+
# automatically. This will be the default behavior in future versions of
|
|
38
|
+
# rspec-rails.
|
|
39
|
+
config.infer_base_class_for_anonymous_controllers = false
|
|
40
|
+
|
|
41
|
+
# Run specs in random order to surface order dependencies. If you find an
|
|
42
|
+
# order dependency and want to debug it, you can fix the order by providing
|
|
43
|
+
# the seed, which is printed after each run.
|
|
44
|
+
# --seed 1234
|
|
45
|
+
config.order = "random"
|
|
46
|
+
end
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
ENV["RAILS_ENV"] ||= "test"
|
|
2
|
+
require File.expand_path('../../config/environment', __FILE__)
|
|
3
|
+
require 'rails/test_help'
|
|
4
|
+
|
|
5
|
+
class ActiveSupport::TestCase
|
|
6
|
+
ActiveRecord::Migration.check_pending!
|
|
7
|
+
|
|
8
|
+
# Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.
|
|
9
|
+
#
|
|
10
|
+
# Note: You'll currently still have to declare fixtures explicitly in integration tests
|
|
11
|
+
# -- they do not yet inherit this setting
|
|
12
|
+
fixtures :all
|
|
13
|
+
|
|
14
|
+
# Add more helper methods to be used by all tests here...
|
|
15
|
+
end
|
|
File without changes
|
|
File without changes
|
|
@@ -1,22 +1,29 @@
|
|
|
1
|
-
require 'csv'
|
|
2
1
|
|
|
3
2
|
module RspecLogFormatter
|
|
4
3
|
module Analysis
|
|
5
4
|
class Analyzer
|
|
6
|
-
def analyze(filepath, options = {})
|
|
7
|
-
window = options[:last_builds]
|
|
8
5
|
|
|
9
|
-
|
|
6
|
+
def initialize(history_provider, options={})
|
|
7
|
+
@builds_to_analyze = options[:builds_to_analyze]
|
|
8
|
+
@max_reruns = options[:max_reruns]
|
|
9
|
+
@limit_history = options[:limit_history]
|
|
10
|
+
@history_provider = history_provider
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def analyze
|
|
14
|
+
build_numbers = @history_provider.builds
|
|
15
|
+
results = @history_provider.results.reject do |res|
|
|
16
|
+
@builds_to_analyze && !build_numbers.last(@builds_to_analyze).include?(res.build_number.to_i)
|
|
17
|
+
end
|
|
10
18
|
|
|
11
19
|
scores = []
|
|
12
20
|
results.group_by(&:description).each do |description, results|
|
|
13
|
-
score = Score.new(description)
|
|
21
|
+
score = Score.new(description, max_reruns: @max_reruns)
|
|
14
22
|
|
|
15
23
|
results.group_by(&:build_number).each do |build_number, results|
|
|
16
|
-
next if (window && !build_numbers.last(window).include?(build_number))
|
|
17
24
|
next if results.all?(&:failure?) #not flaky
|
|
18
25
|
|
|
19
|
-
|
|
26
|
+
results.each{|r| score.absorb(r) }
|
|
20
27
|
score.runs += results.count
|
|
21
28
|
score.failures += results.count(&:failure?)
|
|
22
29
|
score.failure_messages += results.select(&:failure?).map { |r| "#{r.klass}\n #{r.message}" }
|
|
@@ -24,50 +31,8 @@ module RspecLogFormatter
|
|
|
24
31
|
scores << score if score.runs > 0
|
|
25
32
|
end
|
|
26
33
|
|
|
27
|
-
scores.sort.map(&:as_hash)
|
|
34
|
+
scores.select(&:flaky?).sort.map(&:as_hash)
|
|
28
35
|
end
|
|
29
|
-
|
|
30
|
-
def truncate(filepath, opts = {})
|
|
31
|
-
builds = opts.fetch(:keep_builds)
|
|
32
|
-
build_numbers, results = result_numbers(filepath, options = {})
|
|
33
|
-
sio = StringIO.new
|
|
34
|
-
|
|
35
|
-
File.open(filepath, 'r').each_line do |line|
|
|
36
|
-
result = parse_line(line)
|
|
37
|
-
next unless build_numbers.last(builds).include? result.build_number
|
|
38
|
-
sio.puts line
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
sio.rewind
|
|
42
|
-
File.open(filepath, 'w') do |f|
|
|
43
|
-
f.write sio.read
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
private
|
|
48
|
-
|
|
49
|
-
def parse_line(line)
|
|
50
|
-
Result.new(*CSV.parse_line(line, col_sep: "\t").first(7))
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def each_result(filepath, &block)
|
|
54
|
-
File.open(filepath, 'r').each_line do |line|
|
|
55
|
-
result = parse_line(line)
|
|
56
|
-
block.call(result)
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
def result_numbers(filepath, options = {})
|
|
61
|
-
build_numbers = []
|
|
62
|
-
results = []
|
|
63
|
-
each_result(filepath) do |result|
|
|
64
|
-
build_numbers << result.build_number
|
|
65
|
-
results << result
|
|
66
|
-
end
|
|
67
|
-
[build_numbers.uniq.sort, results]
|
|
68
|
-
end
|
|
69
|
-
|
|
70
36
|
end
|
|
71
|
-
|
|
72
37
|
end
|
|
73
38
|
end
|
|
@@ -7,20 +7,24 @@ module RspecLogFormatter
|
|
|
7
7
|
|
|
8
8
|
def to_s
|
|
9
9
|
results = @results.reject do |result|
|
|
10
|
-
result[:
|
|
10
|
+
result[:fraction] == 0.0
|
|
11
11
|
end.first(10)
|
|
12
12
|
|
|
13
13
|
header = if results.empty?
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
"None of the specs were flaky"
|
|
15
|
+
else
|
|
16
|
+
"Top #{results.size} flakiest examples\n"
|
|
17
|
+
end
|
|
18
18
|
header + results.each_with_index.map do |result, i|
|
|
19
|
-
title = " #{i+1}) #{result[:description]} -- #{result[:
|
|
19
|
+
title = " #{i+1}) #{result[:description]} -- #{(100.0*result[:fraction]).to_i}%#{cost_segment(result)}"
|
|
20
20
|
failure_messages = result[:failure_messages].map { |fm| " * #{fm}" }.join("\n")
|
|
21
21
|
title + "\n" + failure_messages
|
|
22
22
|
end.join("\n")
|
|
23
23
|
end
|
|
24
|
+
|
|
25
|
+
def cost_segment(result)
|
|
26
|
+
" (cost: #{result[:cost].to_i}s)" if result[:cost]
|
|
27
|
+
end
|
|
24
28
|
end
|
|
25
29
|
end
|
|
26
30
|
end
|
|
@@ -1,17 +1,21 @@
|
|
|
1
|
+
require 'time'
|
|
2
|
+
|
|
1
3
|
module RspecLogFormatter
|
|
2
4
|
module Analysis
|
|
3
5
|
class Result
|
|
4
|
-
def initialize(build_number, time, outcome, description, spec_path, message=nil, klass=nil)
|
|
5
|
-
@
|
|
6
|
+
def initialize(build_number, time, outcome, description, spec_path, message=nil, klass=nil,duration=nil)
|
|
7
|
+
@time = Time.parse(time)
|
|
8
|
+
@build_number = (build_number || -1).to_i
|
|
6
9
|
@description = description
|
|
7
10
|
@outcome = outcome
|
|
8
11
|
@spec_path = spec_path
|
|
9
12
|
@message = message
|
|
10
13
|
@klass = klass
|
|
14
|
+
@duration = duration.to_f
|
|
11
15
|
end
|
|
12
16
|
|
|
13
|
-
attr_accessor :build_number, :description
|
|
14
|
-
attr_reader :message, :klass
|
|
17
|
+
attr_accessor :build_number, :description, :duration
|
|
18
|
+
attr_reader :message, :klass, :time
|
|
15
19
|
|
|
16
20
|
def failure?
|
|
17
21
|
@outcome == "failed"
|
|
@@ -1,29 +1,65 @@
|
|
|
1
1
|
module RspecLogFormatter
|
|
2
2
|
module Analysis
|
|
3
3
|
class Score
|
|
4
|
-
def initialize(desc)
|
|
4
|
+
def initialize(desc, opts={})
|
|
5
5
|
@description = desc
|
|
6
6
|
@runs = 0
|
|
7
7
|
@failures = 0
|
|
8
8
|
@failure_messages = []
|
|
9
|
+
@last_fail_time = Time.at(0)
|
|
10
|
+
@last_pass_time = Time.at(0)
|
|
11
|
+
@max_reruns = opts[:max_reruns]
|
|
9
12
|
end
|
|
10
13
|
|
|
11
|
-
def
|
|
12
|
-
|
|
14
|
+
def fraction
|
|
15
|
+
@failures.to_f/@runs
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def cost
|
|
19
|
+
sum = 0.0
|
|
20
|
+
0.upto(max_reruns) do |i|
|
|
21
|
+
sum += (fraction**i)*(1.0-fraction)*(i*@fail_duration + @pass_duration)
|
|
22
|
+
end
|
|
23
|
+
sum + (fraction**(max_reruns+1.0))*@fail_duration
|
|
13
24
|
end
|
|
14
25
|
|
|
15
26
|
def <=>(other)
|
|
16
|
-
|
|
27
|
+
if max_reruns
|
|
28
|
+
other.cost <=> cost
|
|
29
|
+
else
|
|
30
|
+
other.fraction <=> fraction
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def flaky?
|
|
35
|
+
fraction > 0.0
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def absorb(result)
|
|
39
|
+
if result.failure? && result.time > @last_fail_time
|
|
40
|
+
@last_fail_time = result.time
|
|
41
|
+
@fail_duration = result.duration
|
|
42
|
+
elsif result.success? && result.time > @last_pass_time
|
|
43
|
+
@last_pass_time = result.time
|
|
44
|
+
@pass_duration = result.duration
|
|
45
|
+
end
|
|
17
46
|
end
|
|
18
47
|
|
|
19
48
|
def as_hash
|
|
20
|
-
{
|
|
49
|
+
h = {
|
|
21
50
|
description: @description,
|
|
22
|
-
|
|
51
|
+
fraction: fraction,
|
|
23
52
|
failure_messages: failure_messages,
|
|
24
53
|
}
|
|
54
|
+
if max_reruns
|
|
55
|
+
h.merge!({cost: cost})
|
|
56
|
+
end
|
|
57
|
+
h
|
|
25
58
|
end
|
|
26
59
|
attr_accessor :runs, :failures, :failure_messages
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
attr_reader :max_reruns
|
|
27
63
|
end
|
|
28
64
|
end
|
|
29
65
|
end
|
|
@@ -2,19 +2,31 @@ require "csv"
|
|
|
2
2
|
require "rspec/core/formatters/base_formatter"
|
|
3
3
|
|
|
4
4
|
module RspecLogFormatter
|
|
5
|
-
class AnalyzerFormatter
|
|
5
|
+
class AnalyzerFormatter
|
|
6
6
|
FILENAME = "rspec.history"
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
class Factory
|
|
9
|
+
def initialize(options={})
|
|
10
|
+
@options = options
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def build(output)
|
|
14
|
+
RspecLogFormatter::AnalyzerFormatter.new(output, {
|
|
15
|
+
builds_to_analyze: nil,
|
|
16
|
+
max_reruns: nil
|
|
17
|
+
}.merge(@options))
|
|
18
|
+
end
|
|
10
19
|
end
|
|
11
20
|
|
|
21
|
+
def initialize(output, options={})
|
|
22
|
+
@output = output
|
|
23
|
+
@options = options
|
|
24
|
+
end
|
|
12
25
|
|
|
13
26
|
def dump_summary(_,_,_,_)
|
|
14
|
-
output.puts RspecLogFormatter::Analysis::PrettyPrinter.new(
|
|
15
|
-
RspecLogFormatter::Analysis::Analyzer.new.
|
|
27
|
+
@output.puts RspecLogFormatter::Analysis::PrettyPrinter.new(
|
|
28
|
+
RspecLogFormatter::Analysis::Analyzer.new(HistoryManager.new(FILENAME), @options).analyze
|
|
16
29
|
)
|
|
17
30
|
end
|
|
18
|
-
|
|
19
31
|
end
|
|
20
32
|
end
|
|
@@ -2,20 +2,27 @@ require "csv"
|
|
|
2
2
|
require "rspec/core/formatters/base_formatter"
|
|
3
3
|
|
|
4
4
|
module RspecLogFormatter
|
|
5
|
-
class Formatter
|
|
6
|
-
FILENAME = "rspec.history"
|
|
5
|
+
class Formatter
|
|
7
6
|
|
|
8
|
-
class
|
|
9
|
-
def
|
|
10
|
-
|
|
7
|
+
class Factory
|
|
8
|
+
def initialize(options={})
|
|
9
|
+
@options = options
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def build
|
|
13
|
+
RspecLogFormatter::Formatter.new({
|
|
14
|
+
clock: Time,
|
|
15
|
+
build_number: ENV["BUILD_NUMBER"],
|
|
16
|
+
limit_history: nil
|
|
17
|
+
}.merge(@options))
|
|
11
18
|
end
|
|
12
19
|
end
|
|
13
|
-
Factory = Maker.new
|
|
14
20
|
|
|
15
|
-
def initialize(
|
|
16
|
-
@clock = clock
|
|
17
|
-
@build_number = opts[:build_number]
|
|
18
|
-
@
|
|
21
|
+
def initialize(opts={})
|
|
22
|
+
@clock = opts[:clock]
|
|
23
|
+
@build_number = opts[:build_number]
|
|
24
|
+
@limit_history = opts[:limit_history]
|
|
25
|
+
@history_manager = RspecLogFormatter::HistoryManager.new(FILENAME)
|
|
19
26
|
end
|
|
20
27
|
|
|
21
28
|
def example_started(example)
|
|
@@ -31,8 +38,8 @@ module RspecLogFormatter
|
|
|
31
38
|
end
|
|
32
39
|
|
|
33
40
|
def dump_summary(_,_,_,_)
|
|
34
|
-
return unless @
|
|
35
|
-
|
|
41
|
+
return unless @limit_history
|
|
42
|
+
@history_manager.truncate(@limit_history)
|
|
36
43
|
end
|
|
37
44
|
|
|
38
45
|
private
|