rspec_log_formatter 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -13
- data/.gitignore +1 -1
- data/.ruby-version +1 -1
- data/README.md +53 -7
- data/bin/rspec_log_formatter +30 -0
- data/dummy/.gitignore +16 -0
- data/dummy/.rspec +1 -0
- data/dummy/Gemfile +47 -0
- data/dummy/README.rdoc +28 -0
- data/dummy/Rakefile +6 -0
- data/dummy/app/assets/images/.keep +0 -0
- data/dummy/app/assets/javascripts/application.js +16 -0
- data/dummy/app/assets/stylesheets/application.css +13 -0
- data/dummy/app/controllers/application_controller.rb +5 -0
- data/dummy/app/controllers/concerns/.keep +0 -0
- data/dummy/app/helpers/application_helper.rb +2 -0
- data/dummy/app/mailers/.keep +0 -0
- data/dummy/app/models/.keep +0 -0
- data/dummy/app/models/concerns/.keep +0 -0
- data/dummy/app/views/layouts/application.html.erb +14 -0
- data/dummy/bin/bundle +3 -0
- data/dummy/bin/rails +4 -0
- data/dummy/bin/rake +4 -0
- data/dummy/config/application.rb +23 -0
- data/dummy/config/boot.rb +4 -0
- data/dummy/config/database.yml +25 -0
- data/dummy/config/environment.rb +5 -0
- data/dummy/config/environments/development.rb +29 -0
- data/dummy/config/environments/production.rb +80 -0
- data/dummy/config/environments/test.rb +36 -0
- data/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/dummy/config/initializers/filter_parameter_logging.rb +4 -0
- data/dummy/config/initializers/inflections.rb +16 -0
- data/dummy/config/initializers/mime_types.rb +5 -0
- data/dummy/config/initializers/secret_token.rb +12 -0
- data/dummy/config/initializers/session_store.rb +3 -0
- data/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/dummy/config/locales/en.yml +23 -0
- data/dummy/config/routes.rb +56 -0
- data/dummy/config.ru +4 -0
- data/dummy/db/schema.rb +16 -0
- data/dummy/db/seeds.rb +7 -0
- data/dummy/lib/assets/.keep +0 -0
- data/dummy/lib/tasks/.keep +0 -0
- data/dummy/log/.keep +0 -0
- data/dummy/public/404.html +58 -0
- data/dummy/public/422.html +58 -0
- data/dummy/public/500.html +57 -0
- data/dummy/public/favicon.ico +0 -0
- data/dummy/public/robots.txt +5 -0
- data/dummy/spec/dummy_spec.rb +7 -0
- data/dummy/spec/spec_helper.rb +46 -0
- data/dummy/test/controllers/.keep +0 -0
- data/dummy/test/fixtures/.keep +0 -0
- data/dummy/test/helpers/.keep +0 -0
- data/dummy/test/integration/.keep +0 -0
- data/dummy/test/mailers/.keep +0 -0
- data/dummy/test/models/.keep +0 -0
- data/dummy/test/test_helper.rb +15 -0
- data/dummy/vendor/assets/javascripts/.keep +0 -0
- data/dummy/vendor/assets/stylesheets/.keep +0 -0
- data/lib/rspec_log_formatter/analysis/analyzer.rb +15 -50
- data/lib/rspec_log_formatter/analysis/pretty_printer.rb +10 -6
- data/lib/rspec_log_formatter/analysis/result.rb +8 -4
- data/lib/rspec_log_formatter/analysis/score.rb +42 -6
- data/lib/rspec_log_formatter/analyzer_formatter.rb +18 -6
- data/lib/rspec_log_formatter/formatter.rb +19 -12
- data/lib/rspec_log_formatter/history_manager.rb +46 -0
- data/lib/rspec_log_formatter/javascripts/chartkick.js +678 -0
- data/lib/rspec_log_formatter/performance_analyzer.rb +40 -0
- data/lib/rspec_log_formatter/templates/charts.html.erb +11 -0
- data/lib/rspec_log_formatter/version.rb +1 -1
- data/lib/rspec_log_formatter.rb +2 -1
- data/rspec_log_formatter.gemspec +1 -0
- data/spec/fixtures/test_slowing_down_over_time.history +7 -0
- data/spec/fixtures/varying_flakiness.history +9 -9
- data/spec/lib/rspec_log_analyzer/analysis/analyzer_spec.rb +25 -21
- data/spec/lib/rspec_log_analyzer/analysis/pretty_printer_spec.rb +10 -8
- data/spec/lib/rspec_log_analyzer/analyzer_formatter_spec.rb +3 -3
- data/spec/lib/rspec_log_analyzer/formatter_spec.rb +16 -18
- data/spec/lib/rspec_log_analyzer/history_manager_spec.rb +16 -0
- data/spec/lib/rspec_log_analyzer/performance_analyzer_spec.rb +16 -0
- data/specs.sh +4 -0
- metadata +95 -12
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
Dummy::Application.routes.draw do
|
|
2
|
+
# The priority is based upon order of creation: first created -> highest priority.
|
|
3
|
+
# See how all your routes lay out with "rake routes".
|
|
4
|
+
|
|
5
|
+
# You can have the root of your site routed with "root"
|
|
6
|
+
# root 'welcome#index'
|
|
7
|
+
|
|
8
|
+
# Example of regular route:
|
|
9
|
+
# get 'products/:id' => 'catalog#view'
|
|
10
|
+
|
|
11
|
+
# Example of named route that can be invoked with purchase_url(id: product.id)
|
|
12
|
+
# get 'products/:id/purchase' => 'catalog#purchase', as: :purchase
|
|
13
|
+
|
|
14
|
+
# Example resource route (maps HTTP verbs to controller actions automatically):
|
|
15
|
+
# resources :products
|
|
16
|
+
|
|
17
|
+
# Example resource route with options:
|
|
18
|
+
# resources :products do
|
|
19
|
+
# member do
|
|
20
|
+
# get 'short'
|
|
21
|
+
# post 'toggle'
|
|
22
|
+
# end
|
|
23
|
+
#
|
|
24
|
+
# collection do
|
|
25
|
+
# get 'sold'
|
|
26
|
+
# end
|
|
27
|
+
# end
|
|
28
|
+
|
|
29
|
+
# Example resource route with sub-resources:
|
|
30
|
+
# resources :products do
|
|
31
|
+
# resources :comments, :sales
|
|
32
|
+
# resource :seller
|
|
33
|
+
# end
|
|
34
|
+
|
|
35
|
+
# Example resource route with more complex sub-resources:
|
|
36
|
+
# resources :products do
|
|
37
|
+
# resources :comments
|
|
38
|
+
# resources :sales do
|
|
39
|
+
# get 'recent', on: :collection
|
|
40
|
+
# end
|
|
41
|
+
# end
|
|
42
|
+
|
|
43
|
+
# Example resource route with concerns:
|
|
44
|
+
# concern :toggleable do
|
|
45
|
+
# post 'toggle'
|
|
46
|
+
# end
|
|
47
|
+
# resources :posts, concerns: :toggleable
|
|
48
|
+
# resources :photos, concerns: :toggleable
|
|
49
|
+
|
|
50
|
+
# Example resource route within a namespace:
|
|
51
|
+
# namespace :admin do
|
|
52
|
+
# # Directs /admin/products/* to Admin::ProductsController
|
|
53
|
+
# # (app/controllers/admin/products_controller.rb)
|
|
54
|
+
# resources :products
|
|
55
|
+
# end
|
|
56
|
+
end
|
data/dummy/config.ru
ADDED
data/dummy/db/schema.rb
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
# This file is auto-generated from the current state of the database. Instead
|
|
3
|
+
# of editing this file, please use the migrations feature of Active Record to
|
|
4
|
+
# incrementally modify your database, and then regenerate this schema definition.
|
|
5
|
+
#
|
|
6
|
+
# Note that this schema.rb definition is the authoritative source for your
|
|
7
|
+
# database schema. If you need to create the application database on another
|
|
8
|
+
# system, you should be using db:schema:load, not running all the migrations
|
|
9
|
+
# from scratch. The latter is a flawed and unsustainable approach (the more migrations
|
|
10
|
+
# you'll amass, the slower it'll run and the greater likelihood for issues).
|
|
11
|
+
#
|
|
12
|
+
# It's strongly recommended that you check this file into your version control system.
|
|
13
|
+
|
|
14
|
+
ActiveRecord::Schema.define(version: 0) do
|
|
15
|
+
|
|
16
|
+
end
|
data/dummy/db/seeds.rb
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# This file should contain all the record creation needed to seed the database with its default values.
|
|
2
|
+
# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
|
|
3
|
+
#
|
|
4
|
+
# Examples:
|
|
5
|
+
#
|
|
6
|
+
# cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }])
|
|
7
|
+
# Mayor.create(name: 'Emanuel', city: cities.first)
|
|
File without changes
|
|
File without changes
|
data/dummy/log/.keep
ADDED
|
File without changes
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>The page you were looking for doesn't exist (404)</title>
|
|
5
|
+
<style>
|
|
6
|
+
body {
|
|
7
|
+
background-color: #EFEFEF;
|
|
8
|
+
color: #2E2F30;
|
|
9
|
+
text-align: center;
|
|
10
|
+
font-family: arial, sans-serif;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
div.dialog {
|
|
14
|
+
width: 25em;
|
|
15
|
+
margin: 4em auto 0 auto;
|
|
16
|
+
border: 1px solid #CCC;
|
|
17
|
+
border-right-color: #999;
|
|
18
|
+
border-left-color: #999;
|
|
19
|
+
border-bottom-color: #BBB;
|
|
20
|
+
border-top: #B00100 solid 4px;
|
|
21
|
+
border-top-left-radius: 9px;
|
|
22
|
+
border-top-right-radius: 9px;
|
|
23
|
+
background-color: white;
|
|
24
|
+
padding: 7px 4em 0 4em;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
h1 {
|
|
28
|
+
font-size: 100%;
|
|
29
|
+
color: #730E15;
|
|
30
|
+
line-height: 1.5em;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
body > p {
|
|
34
|
+
width: 33em;
|
|
35
|
+
margin: 0 auto 1em;
|
|
36
|
+
padding: 1em 0;
|
|
37
|
+
background-color: #F7F7F7;
|
|
38
|
+
border: 1px solid #CCC;
|
|
39
|
+
border-right-color: #999;
|
|
40
|
+
border-bottom-color: #999;
|
|
41
|
+
border-bottom-left-radius: 4px;
|
|
42
|
+
border-bottom-right-radius: 4px;
|
|
43
|
+
border-top-color: #DADADA;
|
|
44
|
+
color: #666;
|
|
45
|
+
box-shadow:0 3px 8px rgba(50, 50, 50, 0.17);
|
|
46
|
+
}
|
|
47
|
+
</style>
|
|
48
|
+
</head>
|
|
49
|
+
|
|
50
|
+
<body>
|
|
51
|
+
<!-- This file lives in public/404.html -->
|
|
52
|
+
<div class="dialog">
|
|
53
|
+
<h1>The page you were looking for doesn't exist.</h1>
|
|
54
|
+
<p>You may have mistyped the address or the page may have moved.</p>
|
|
55
|
+
</div>
|
|
56
|
+
<p>If you are the application owner check the logs for more information.</p>
|
|
57
|
+
</body>
|
|
58
|
+
</html>
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>The change you wanted was rejected (422)</title>
|
|
5
|
+
<style>
|
|
6
|
+
body {
|
|
7
|
+
background-color: #EFEFEF;
|
|
8
|
+
color: #2E2F30;
|
|
9
|
+
text-align: center;
|
|
10
|
+
font-family: arial, sans-serif;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
div.dialog {
|
|
14
|
+
width: 25em;
|
|
15
|
+
margin: 4em auto 0 auto;
|
|
16
|
+
border: 1px solid #CCC;
|
|
17
|
+
border-right-color: #999;
|
|
18
|
+
border-left-color: #999;
|
|
19
|
+
border-bottom-color: #BBB;
|
|
20
|
+
border-top: #B00100 solid 4px;
|
|
21
|
+
border-top-left-radius: 9px;
|
|
22
|
+
border-top-right-radius: 9px;
|
|
23
|
+
background-color: white;
|
|
24
|
+
padding: 7px 4em 0 4em;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
h1 {
|
|
28
|
+
font-size: 100%;
|
|
29
|
+
color: #730E15;
|
|
30
|
+
line-height: 1.5em;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
body > p {
|
|
34
|
+
width: 33em;
|
|
35
|
+
margin: 0 auto 1em;
|
|
36
|
+
padding: 1em 0;
|
|
37
|
+
background-color: #F7F7F7;
|
|
38
|
+
border: 1px solid #CCC;
|
|
39
|
+
border-right-color: #999;
|
|
40
|
+
border-bottom-color: #999;
|
|
41
|
+
border-bottom-left-radius: 4px;
|
|
42
|
+
border-bottom-right-radius: 4px;
|
|
43
|
+
border-top-color: #DADADA;
|
|
44
|
+
color: #666;
|
|
45
|
+
box-shadow:0 3px 8px rgba(50, 50, 50, 0.17);
|
|
46
|
+
}
|
|
47
|
+
</style>
|
|
48
|
+
</head>
|
|
49
|
+
|
|
50
|
+
<body>
|
|
51
|
+
<!-- This file lives in public/422.html -->
|
|
52
|
+
<div class="dialog">
|
|
53
|
+
<h1>The change you wanted was rejected.</h1>
|
|
54
|
+
<p>Maybe you tried to change something you didn't have access to.</p>
|
|
55
|
+
</div>
|
|
56
|
+
<p>If you are the application owner check the logs for more information.</p>
|
|
57
|
+
</body>
|
|
58
|
+
</html>
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>We're sorry, but something went wrong (500)</title>
|
|
5
|
+
<style>
|
|
6
|
+
body {
|
|
7
|
+
background-color: #EFEFEF;
|
|
8
|
+
color: #2E2F30;
|
|
9
|
+
text-align: center;
|
|
10
|
+
font-family: arial, sans-serif;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
div.dialog {
|
|
14
|
+
width: 25em;
|
|
15
|
+
margin: 4em auto 0 auto;
|
|
16
|
+
border: 1px solid #CCC;
|
|
17
|
+
border-right-color: #999;
|
|
18
|
+
border-left-color: #999;
|
|
19
|
+
border-bottom-color: #BBB;
|
|
20
|
+
border-top: #B00100 solid 4px;
|
|
21
|
+
border-top-left-radius: 9px;
|
|
22
|
+
border-top-right-radius: 9px;
|
|
23
|
+
background-color: white;
|
|
24
|
+
padding: 7px 4em 0 4em;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
h1 {
|
|
28
|
+
font-size: 100%;
|
|
29
|
+
color: #730E15;
|
|
30
|
+
line-height: 1.5em;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
body > p {
|
|
34
|
+
width: 33em;
|
|
35
|
+
margin: 0 auto 1em;
|
|
36
|
+
padding: 1em 0;
|
|
37
|
+
background-color: #F7F7F7;
|
|
38
|
+
border: 1px solid #CCC;
|
|
39
|
+
border-right-color: #999;
|
|
40
|
+
border-bottom-color: #999;
|
|
41
|
+
border-bottom-left-radius: 4px;
|
|
42
|
+
border-bottom-right-radius: 4px;
|
|
43
|
+
border-top-color: #DADADA;
|
|
44
|
+
color: #666;
|
|
45
|
+
box-shadow:0 3px 8px rgba(50, 50, 50, 0.17);
|
|
46
|
+
}
|
|
47
|
+
</style>
|
|
48
|
+
</head>
|
|
49
|
+
|
|
50
|
+
<body>
|
|
51
|
+
<!-- This file lives in public/500.html -->
|
|
52
|
+
<div class="dialog">
|
|
53
|
+
<h1>We're sorry, but something went wrong.</h1>
|
|
54
|
+
</div>
|
|
55
|
+
<p>If you are the application owner check the logs for more information.</p>
|
|
56
|
+
</body>
|
|
57
|
+
</html>
|
|
File without changes
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# This file is copied to spec/ when you run 'rails generate rspec:install'
|
|
2
|
+
ENV["RAILS_ENV"] ||= 'test'
|
|
3
|
+
require File.expand_path("../../config/environment", __FILE__)
|
|
4
|
+
require 'rspec/rails'
|
|
5
|
+
require 'rspec/autorun'
|
|
6
|
+
|
|
7
|
+
# Requires supporting ruby files with custom matchers and macros, etc,
|
|
8
|
+
# in spec/support/ and its subdirectories.
|
|
9
|
+
Dir[Rails.root.join("spec/support/**/*.rb")].each { |f| require f }
|
|
10
|
+
# Checks for pending migrations before tests are run.
|
|
11
|
+
# If you are not using ActiveRecord, you can remove this line.
|
|
12
|
+
ActiveRecord::Migration.check_pending! if defined?(ActiveRecord::Migration)
|
|
13
|
+
|
|
14
|
+
RSpec.configure do |config|
|
|
15
|
+
# ## Mock Framework
|
|
16
|
+
#
|
|
17
|
+
# If you prefer to use mocha, flexmock or RR, uncomment the appropriate line:
|
|
18
|
+
#
|
|
19
|
+
# config.mock_with :mocha
|
|
20
|
+
# config.mock_with :flexmock
|
|
21
|
+
# config.mock_with :rr
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
config.formatters << RspecLogFormatter::Formatter::Factory.new.build
|
|
25
|
+
config.add_formatter(:progress)
|
|
26
|
+
config.formatters << RspecLogFormatter::AnalyzerFormatter::Factory.new.build($stdout)
|
|
27
|
+
|
|
28
|
+
# Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
|
|
29
|
+
config.fixture_path = "#{::Rails.root}/spec/fixtures"
|
|
30
|
+
|
|
31
|
+
# If you're not using ActiveRecord, or you'd prefer not to run each of your
|
|
32
|
+
# examples within a transaction, remove the following line or assign false
|
|
33
|
+
# instead of true.
|
|
34
|
+
config.use_transactional_fixtures = true
|
|
35
|
+
|
|
36
|
+
# If true, the base class of anonymous controllers will be inferred
|
|
37
|
+
# automatically. This will be the default behavior in future versions of
|
|
38
|
+
# rspec-rails.
|
|
39
|
+
config.infer_base_class_for_anonymous_controllers = false
|
|
40
|
+
|
|
41
|
+
# Run specs in random order to surface order dependencies. If you find an
|
|
42
|
+
# order dependency and want to debug it, you can fix the order by providing
|
|
43
|
+
# the seed, which is printed after each run.
|
|
44
|
+
# --seed 1234
|
|
45
|
+
config.order = "random"
|
|
46
|
+
end
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
ENV["RAILS_ENV"] ||= "test"
|
|
2
|
+
require File.expand_path('../../config/environment', __FILE__)
|
|
3
|
+
require 'rails/test_help'
|
|
4
|
+
|
|
5
|
+
class ActiveSupport::TestCase
|
|
6
|
+
ActiveRecord::Migration.check_pending!
|
|
7
|
+
|
|
8
|
+
# Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.
|
|
9
|
+
#
|
|
10
|
+
# Note: You'll currently still have to declare fixtures explicitly in integration tests
|
|
11
|
+
# -- they do not yet inherit this setting
|
|
12
|
+
fixtures :all
|
|
13
|
+
|
|
14
|
+
# Add more helper methods to be used by all tests here...
|
|
15
|
+
end
|
|
File without changes
|
|
File without changes
|
|
@@ -1,22 +1,29 @@
|
|
|
1
|
-
require 'csv'
|
|
2
1
|
|
|
3
2
|
module RspecLogFormatter
|
|
4
3
|
module Analysis
|
|
5
4
|
class Analyzer
|
|
6
|
-
def analyze(filepath, options = {})
|
|
7
|
-
window = options[:last_builds]
|
|
8
5
|
|
|
9
|
-
|
|
6
|
+
def initialize(history_provider, options={})
|
|
7
|
+
@builds_to_analyze = options[:builds_to_analyze]
|
|
8
|
+
@max_reruns = options[:max_reruns]
|
|
9
|
+
@limit_history = options[:limit_history]
|
|
10
|
+
@history_provider = history_provider
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def analyze
|
|
14
|
+
build_numbers = @history_provider.builds
|
|
15
|
+
results = @history_provider.results.reject do |res|
|
|
16
|
+
@builds_to_analyze && !build_numbers.last(@builds_to_analyze).include?(res.build_number.to_i)
|
|
17
|
+
end
|
|
10
18
|
|
|
11
19
|
scores = []
|
|
12
20
|
results.group_by(&:description).each do |description, results|
|
|
13
|
-
score = Score.new(description)
|
|
21
|
+
score = Score.new(description, max_reruns: @max_reruns)
|
|
14
22
|
|
|
15
23
|
results.group_by(&:build_number).each do |build_number, results|
|
|
16
|
-
next if (window && !build_numbers.last(window).include?(build_number))
|
|
17
24
|
next if results.all?(&:failure?) #not flaky
|
|
18
25
|
|
|
19
|
-
|
|
26
|
+
results.each{|r| score.absorb(r) }
|
|
20
27
|
score.runs += results.count
|
|
21
28
|
score.failures += results.count(&:failure?)
|
|
22
29
|
score.failure_messages += results.select(&:failure?).map { |r| "#{r.klass}\n #{r.message}" }
|
|
@@ -24,50 +31,8 @@ module RspecLogFormatter
|
|
|
24
31
|
scores << score if score.runs > 0
|
|
25
32
|
end
|
|
26
33
|
|
|
27
|
-
scores.sort.map(&:as_hash)
|
|
34
|
+
scores.select(&:flaky?).sort.map(&:as_hash)
|
|
28
35
|
end
|
|
29
|
-
|
|
30
|
-
def truncate(filepath, opts = {})
|
|
31
|
-
builds = opts.fetch(:keep_builds)
|
|
32
|
-
build_numbers, results = result_numbers(filepath, options = {})
|
|
33
|
-
sio = StringIO.new
|
|
34
|
-
|
|
35
|
-
File.open(filepath, 'r').each_line do |line|
|
|
36
|
-
result = parse_line(line)
|
|
37
|
-
next unless build_numbers.last(builds).include? result.build_number
|
|
38
|
-
sio.puts line
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
sio.rewind
|
|
42
|
-
File.open(filepath, 'w') do |f|
|
|
43
|
-
f.write sio.read
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
private
|
|
48
|
-
|
|
49
|
-
def parse_line(line)
|
|
50
|
-
Result.new(*CSV.parse_line(line, col_sep: "\t").first(7))
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def each_result(filepath, &block)
|
|
54
|
-
File.open(filepath, 'r').each_line do |line|
|
|
55
|
-
result = parse_line(line)
|
|
56
|
-
block.call(result)
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
def result_numbers(filepath, options = {})
|
|
61
|
-
build_numbers = []
|
|
62
|
-
results = []
|
|
63
|
-
each_result(filepath) do |result|
|
|
64
|
-
build_numbers << result.build_number
|
|
65
|
-
results << result
|
|
66
|
-
end
|
|
67
|
-
[build_numbers.uniq.sort, results]
|
|
68
|
-
end
|
|
69
|
-
|
|
70
36
|
end
|
|
71
|
-
|
|
72
37
|
end
|
|
73
38
|
end
|
|
@@ -7,20 +7,24 @@ module RspecLogFormatter
|
|
|
7
7
|
|
|
8
8
|
def to_s
|
|
9
9
|
results = @results.reject do |result|
|
|
10
|
-
result[:
|
|
10
|
+
result[:fraction] == 0.0
|
|
11
11
|
end.first(10)
|
|
12
12
|
|
|
13
13
|
header = if results.empty?
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
"None of the specs were flaky"
|
|
15
|
+
else
|
|
16
|
+
"Top #{results.size} flakiest examples\n"
|
|
17
|
+
end
|
|
18
18
|
header + results.each_with_index.map do |result, i|
|
|
19
|
-
title = " #{i+1}) #{result[:description]} -- #{result[:
|
|
19
|
+
title = " #{i+1}) #{result[:description]} -- #{(100.0*result[:fraction]).to_i}%#{cost_segment(result)}"
|
|
20
20
|
failure_messages = result[:failure_messages].map { |fm| " * #{fm}" }.join("\n")
|
|
21
21
|
title + "\n" + failure_messages
|
|
22
22
|
end.join("\n")
|
|
23
23
|
end
|
|
24
|
+
|
|
25
|
+
def cost_segment(result)
|
|
26
|
+
" (cost: #{result[:cost].to_i}s)" if result[:cost]
|
|
27
|
+
end
|
|
24
28
|
end
|
|
25
29
|
end
|
|
26
30
|
end
|
|
@@ -1,17 +1,21 @@
|
|
|
1
|
+
require 'time'
|
|
2
|
+
|
|
1
3
|
module RspecLogFormatter
|
|
2
4
|
module Analysis
|
|
3
5
|
class Result
|
|
4
|
-
def initialize(build_number, time, outcome, description, spec_path, message=nil, klass=nil)
|
|
5
|
-
@
|
|
6
|
+
def initialize(build_number, time, outcome, description, spec_path, message=nil, klass=nil,duration=nil)
|
|
7
|
+
@time = Time.parse(time)
|
|
8
|
+
@build_number = (build_number || -1).to_i
|
|
6
9
|
@description = description
|
|
7
10
|
@outcome = outcome
|
|
8
11
|
@spec_path = spec_path
|
|
9
12
|
@message = message
|
|
10
13
|
@klass = klass
|
|
14
|
+
@duration = duration.to_f
|
|
11
15
|
end
|
|
12
16
|
|
|
13
|
-
attr_accessor :build_number, :description
|
|
14
|
-
attr_reader :message, :klass
|
|
17
|
+
attr_accessor :build_number, :description, :duration
|
|
18
|
+
attr_reader :message, :klass, :time
|
|
15
19
|
|
|
16
20
|
def failure?
|
|
17
21
|
@outcome == "failed"
|
|
@@ -1,29 +1,65 @@
|
|
|
1
1
|
module RspecLogFormatter
|
|
2
2
|
module Analysis
|
|
3
3
|
class Score
|
|
4
|
-
def initialize(desc)
|
|
4
|
+
def initialize(desc, opts={})
|
|
5
5
|
@description = desc
|
|
6
6
|
@runs = 0
|
|
7
7
|
@failures = 0
|
|
8
8
|
@failure_messages = []
|
|
9
|
+
@last_fail_time = Time.at(0)
|
|
10
|
+
@last_pass_time = Time.at(0)
|
|
11
|
+
@max_reruns = opts[:max_reruns]
|
|
9
12
|
end
|
|
10
13
|
|
|
11
|
-
def
|
|
12
|
-
|
|
14
|
+
def fraction
|
|
15
|
+
@failures.to_f/@runs
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def cost
|
|
19
|
+
sum = 0.0
|
|
20
|
+
0.upto(max_reruns) do |i|
|
|
21
|
+
sum += (fraction**i)*(1.0-fraction)*(i*@fail_duration + @pass_duration)
|
|
22
|
+
end
|
|
23
|
+
sum + (fraction**(max_reruns+1.0))*@fail_duration
|
|
13
24
|
end
|
|
14
25
|
|
|
15
26
|
def <=>(other)
|
|
16
|
-
|
|
27
|
+
if max_reruns
|
|
28
|
+
other.cost <=> cost
|
|
29
|
+
else
|
|
30
|
+
other.fraction <=> fraction
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def flaky?
|
|
35
|
+
fraction > 0.0
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def absorb(result)
|
|
39
|
+
if result.failure? && result.time > @last_fail_time
|
|
40
|
+
@last_fail_time = result.time
|
|
41
|
+
@fail_duration = result.duration
|
|
42
|
+
elsif result.success? && result.time > @last_pass_time
|
|
43
|
+
@last_pass_time = result.time
|
|
44
|
+
@pass_duration = result.duration
|
|
45
|
+
end
|
|
17
46
|
end
|
|
18
47
|
|
|
19
48
|
def as_hash
|
|
20
|
-
{
|
|
49
|
+
h = {
|
|
21
50
|
description: @description,
|
|
22
|
-
|
|
51
|
+
fraction: fraction,
|
|
23
52
|
failure_messages: failure_messages,
|
|
24
53
|
}
|
|
54
|
+
if max_reruns
|
|
55
|
+
h.merge!({cost: cost})
|
|
56
|
+
end
|
|
57
|
+
h
|
|
25
58
|
end
|
|
26
59
|
attr_accessor :runs, :failures, :failure_messages
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
attr_reader :max_reruns
|
|
27
63
|
end
|
|
28
64
|
end
|
|
29
65
|
end
|
|
@@ -2,19 +2,31 @@ require "csv"
|
|
|
2
2
|
require "rspec/core/formatters/base_formatter"
|
|
3
3
|
|
|
4
4
|
module RspecLogFormatter
|
|
5
|
-
class AnalyzerFormatter
|
|
5
|
+
class AnalyzerFormatter
|
|
6
6
|
FILENAME = "rspec.history"
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
class Factory
|
|
9
|
+
def initialize(options={})
|
|
10
|
+
@options = options
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def build(output)
|
|
14
|
+
RspecLogFormatter::AnalyzerFormatter.new(output, {
|
|
15
|
+
builds_to_analyze: nil,
|
|
16
|
+
max_reruns: nil
|
|
17
|
+
}.merge(@options))
|
|
18
|
+
end
|
|
10
19
|
end
|
|
11
20
|
|
|
21
|
+
def initialize(output, options={})
|
|
22
|
+
@output = output
|
|
23
|
+
@options = options
|
|
24
|
+
end
|
|
12
25
|
|
|
13
26
|
def dump_summary(_,_,_,_)
|
|
14
|
-
output.puts RspecLogFormatter::Analysis::PrettyPrinter.new(
|
|
15
|
-
RspecLogFormatter::Analysis::Analyzer.new.
|
|
27
|
+
@output.puts RspecLogFormatter::Analysis::PrettyPrinter.new(
|
|
28
|
+
RspecLogFormatter::Analysis::Analyzer.new(HistoryManager.new(FILENAME), @options).analyze
|
|
16
29
|
)
|
|
17
30
|
end
|
|
18
|
-
|
|
19
31
|
end
|
|
20
32
|
end
|
|
@@ -2,20 +2,27 @@ require "csv"
|
|
|
2
2
|
require "rspec/core/formatters/base_formatter"
|
|
3
3
|
|
|
4
4
|
module RspecLogFormatter
|
|
5
|
-
class Formatter
|
|
6
|
-
FILENAME = "rspec.history"
|
|
5
|
+
class Formatter
|
|
7
6
|
|
|
8
|
-
class
|
|
9
|
-
def
|
|
10
|
-
|
|
7
|
+
class Factory
|
|
8
|
+
def initialize(options={})
|
|
9
|
+
@options = options
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def build
|
|
13
|
+
RspecLogFormatter::Formatter.new({
|
|
14
|
+
clock: Time,
|
|
15
|
+
build_number: ENV["BUILD_NUMBER"],
|
|
16
|
+
limit_history: nil
|
|
17
|
+
}.merge(@options))
|
|
11
18
|
end
|
|
12
19
|
end
|
|
13
|
-
Factory = Maker.new
|
|
14
20
|
|
|
15
|
-
def initialize(
|
|
16
|
-
@clock = clock
|
|
17
|
-
@build_number = opts[:build_number]
|
|
18
|
-
@
|
|
21
|
+
def initialize(opts={})
|
|
22
|
+
@clock = opts[:clock]
|
|
23
|
+
@build_number = opts[:build_number]
|
|
24
|
+
@limit_history = opts[:limit_history]
|
|
25
|
+
@history_manager = RspecLogFormatter::HistoryManager.new(FILENAME)
|
|
19
26
|
end
|
|
20
27
|
|
|
21
28
|
def example_started(example)
|
|
@@ -31,8 +38,8 @@ module RspecLogFormatter
|
|
|
31
38
|
end
|
|
32
39
|
|
|
33
40
|
def dump_summary(_,_,_,_)
|
|
34
|
-
return unless @
|
|
35
|
-
|
|
41
|
+
return unless @limit_history
|
|
42
|
+
@history_manager.truncate(@limit_history)
|
|
36
43
|
end
|
|
37
44
|
|
|
38
45
|
private
|