rspec-flaky 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (14) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +39 -0
  3. data/assets/layout.html.erb +149 -0
  4. data/bin/rspec-flaky +4 -0
  5. data/lib/rspec/flaky.rb +32 -0
  6. data/lib/rspec/flaky/cli.rb +64 -0
  7. data/lib/rspec/flaky/differ.rb +61 -0
  8. data/lib/rspec/flaky/drawer.rb +21 -0
  9. data/lib/rspec/flaky/dumper/db.rb +25 -0
  10. data/lib/rspec/flaky/dumper/json.rb +23 -0
  11. data/lib/rspec/flaky/pathes.rb +27 -0
  12. data/lib/rspec/flaky/tables.rb +22 -0
  13. data/lib/rspec/flaky/version.rb +5 -0
  14. metadata +71 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 4d80b7e5a54c2a9d25326907dc0ff92304f410041c55e24d030bc99451370273
4
+ data.tar.gz: 6bf0165957822e3af4e57f477c18fac6f37cd756f275f78126265b085d365696
5
+ SHA512:
6
+ metadata.gz: 43df5f4540f9cde2c791cd194068dcc4b912dd63f1429d4121166469bac2a216426ab896cc97e946903c098cf3100355bc2592f82f23abce2691112c60b35fa0
7
+ data.tar.gz: ccddd3c02724b590f8c2798ad99c0488670719676afa6dc6aaae4f87e274f9058b925b9c8ecc48417139ba988c27b2ef0a0152f8e64b68442c199bec0a619f6c
data/README.md ADDED
@@ -0,0 +1,39 @@
1
+ # RSpecFlaky
2
+
3
+ TODO: Write a gem description
4
+
5
+ ## Installation
6
+
7
+ Add this line to test group of your application's Gemfile:
8
+
9
+ gem 'rspec-flaky'
10
+
11
+ Install the gem:
12
+
13
+ $ bundle
14
+
15
+ And then add this line to your spec/spec_helper.rb:
16
+
17
+ require 'rspec/flaky'
18
+
19
+ Select the model whose attributes will be dumped:
20
+
21
+ it 'is flaky test', tables: [User, Post] do
22
+ expect([true, false]).to be true
23
+ end
24
+
25
+ ## Usage
26
+
27
+ Run the command to iteratively run flaky example:
28
+
29
+ rspec-flaky path/to/flaky_spec.rb:12 -i 10
30
+
31
+ If at least one example is failed you can compare pointed model's attributes dumped to tmp/flaky_test folder.
32
+
33
+ ## Contributing
34
+
35
+ 1. Fork it
36
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
37
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
38
+ 4. Push to the branch (`git push origin my-new-feature`)
39
+ 5. Create new Pull Request
data/assets/layout.html.erb ADDED
@@ -0,0 +1,149 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <link href="application.css" rel='stylesheet' type='text/css' />
7
+ <title>Flaky Result</title>
8
+ <style>
9
+ html {
10
+ line-height: 1.15;
11
+ }
12
+
13
+ body {
14
+ margin: auto;
15
+ width: 80%;
16
+ }
17
+
18
+
19
+ h1 {
20
+ font-size: 2em;
21
+ margin: .67em 0
22
+ }
23
+
24
+ pre {
25
+ font-family: monospace, monospace;
26
+ font-size: 1em
27
+ }
28
+
29
+ html {
30
+ font-family: sans-serif
31
+ }
32
+
33
+ .pure-table {
34
+ border-collapse: collapse;
35
+ border-spacing: 0;
36
+ empty-cells: show;
37
+ border: 1px solid #cbcbcb
38
+ }
39
+
40
+ table {
41
+ table-layout: fixed;
42
+ width: 100%
43
+ }
44
+
45
+ .pure-table caption {
46
+ color: #000;
47
+ font: italic 85%/1 arial, sans-serif;
48
+ padding: 1em 0;
49
+ text-align: center
50
+ }
51
+
52
+ .pure-table td,
53
+ .pure-table th {
54
+ border-left: 1px solid #cbcbcb;
55
+ border-width: 0 0 0 1px;
56
+ font-size: inherit;
57
+ margin: 0;
58
+ overflow: visible;
59
+ padding: .5em 1em
60
+ }
61
+
62
+ .pure-table thead {
63
+ background-color: #e0e0e0;
64
+ color: #000;
65
+ text-align: left;
66
+ vertical-align: bottom
67
+ }
68
+
69
+ .pure-table td {
70
+ background-color: transparent;
71
+ max-width: 600px;
72
+ overflow: hidden;
73
+ }
74
+
75
+ .pure-table-odd td {
76
+ background-color: #f2f2f2
77
+ }
78
+
79
+ .pure-table-striped tr:nth-child(2n-1) td {
80
+ background-color: #f2f2f2
81
+ }
82
+
83
+ .pure-table-bordered td {
84
+ border-bottom: 1px solid #cbcbcb
85
+ }
86
+
87
+ .pure-table-bordered tbody>tr:last-child>td {
88
+ border-bottom-width: 0
89
+ }
90
+
91
+ .pure-table-horizontal td,
92
+ .pure-table-horizontal th {
93
+ border-width: 0 0 1px 0;
94
+ border-bottom: 1px solid #cbcbcb
95
+ }
96
+
97
+ .pure-table-horizontal tbody>tr:last-child>td {
98
+ border-bottom-width: 0
99
+ }
100
+ </style>
101
+ </head>
102
+ <body>
103
+ <% @diffs.each do |diffs| -%>
104
+ <div class="example">
105
+ <h2>Location: <%= diffs[:location] -%></h2>
106
+ <h2>Table: <%= diffs[:table] %></h2>
107
+ <% if diffs[:result] == "no_content" -%>
108
+ There were no failed and passed tests during the testing process. Try to increase the number of iterations
109
+ <% elsif diffs[:result] == "empty_table" -%>
110
+ Table is empty
111
+ <% else -%>
112
+ <p>Diffs:</p>
113
+ <% diffs[:result].each do |diff| -%>
114
+ <div>
115
+ <table class="pure-table">
116
+ <thead>
117
+ <th>
118
+ <%=diffs[:table]%>'s Attribute
119
+ </th>
120
+ <th>
121
+ Passed value
122
+ </th>
123
+ <th>
124
+ Failed value
125
+ </th>
126
+ </thead>
127
+ <tbody>
128
+ <% diff.each do |attribute| -%>
129
+ <tr>
130
+ <td>
131
+ <span><%= attribute[1] || "Record"%></span>
132
+ </td>
133
+ <td style="background-color: lightgreen">
134
+ <span><pre><%= prettify attribute[2]%></pre></span>
135
+ </td>
136
+ <td style="background-color: rosybrown">
137
+ <span><pre><%= prettify attribute[3]%></pre></span>
138
+ </td>
139
+ </tr>
140
+ <% end -%>
141
+ </tbody>
142
+ </table>
143
+ </div>
144
+ <% end -%>
145
+ <% end -%>
146
+ </div>
147
+ <%end -%>
148
+ </body>
149
+ </html>
data/bin/rspec-flaky ADDED
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rspec/flaky/cli'
3
+
4
+ Flaky::CLI.new.run(ARGV)
data/lib/rspec/flaky.rb ADDED
@@ -0,0 +1,32 @@
1
+ require 'rspec/flaky/version'
2
+ require 'rspec/flaky/differ'
3
+ require 'rspec/flaky/tables'
4
+ require 'rspec/flaky/pathes'
5
+ require 'rspec/flaky/drawer'
6
+ require 'rspec'
7
+ require 'fileutils'
8
+ require 'open3'
9
+
10
+ module RSpec::Flaky
11
+
12
+ def self.run_spec locations, options
13
+ FileUtils.rm_rf(Pathes.summary_path)
14
+ rspec_options = options.delete(:rspec_options) || ""
15
+ options[:iterations].times do
16
+ if options[:silent]
17
+ Open3.capture2e("FLAKY_SPEC=1 rspec #{locations} #{rspec_options}")
18
+ else
19
+ system("FLAKY_SPEC=1 rspec #{locations} #{rspec_options}")
20
+ end
21
+ end
22
+ Differ.get_result
23
+ Pathes.summary_path.children.each do |child|
24
+ if child.basename.to_s.start_with?(".:")
25
+ FileUtils.rm_rf(child) unless options[:save_jsons]
26
+ elsif child.basename.to_s == 'database_dumps'
27
+ FileUtils.rm_rf(child) unless options[:dump_db]
28
+ end
29
+ end
30
+ end
31
+
32
+ end
data/lib/rspec/flaky/cli.rb ADDED
@@ -0,0 +1,64 @@
1
+ require 'rspec/flaky'
2
+ require 'optparse'
3
+
4
+ module Flaky
5
+ class CLI
6
+
7
+ DEFAULT_OPTIONS = {
8
+ iterations: 1,
9
+ silent: false,
10
+ save_jsons: false,
11
+ dump_db: false
12
+ }
13
+
14
+ def run argv
15
+ locations = get_location(argv) unless argv.any?{|arg| arg == '-h' || arg == '--help'}
16
+ options = parse_options(argv).reverse_merge(DEFAULT_OPTIONS)
17
+ options[:rspec_options] = extract_rspec_options argv
18
+ RSpec::Flaky.run_spec(locations, options)
19
+ end
20
+
21
+ private
22
+
23
+ def get_location(argv)
24
+ first_arg = argv.shift
25
+ raise 'You need to specify location first' unless Pathname.new(first_arg).exist?
26
+ return first_arg
27
+ end
28
+
29
+
30
+ def parse_options argv
31
+ options = {}
32
+ OptionParser.new do |opts|
33
+ opts.banner = "Usage: rspec-flaky path/to/flaky_spec.rb:12 [options] -- [rspec options]"
34
+
35
+ opts.on("-i", "--iterations [NUMBER]", Integer, "Execute spec a given number of times") do |v|
36
+ options[:iterations] = v
37
+ end
38
+
39
+ opts.on("--silent", "Silent mode (no output)") do |v|
40
+ options[:silent] = v
41
+ end
42
+ opts.on("-j", "--jsons", "Save pointed models attributes") do |v|
43
+ options[:save_jsons] = v
44
+ end
45
+ opts.on("-d", "--dump", "Dump database to sql-file after each example") do |v|
46
+ options[:dump_db] = v
47
+ end
48
+
49
+ opts.on('-h', '--help', 'Displays Help') do
50
+ puts opts
51
+ exit
52
+ end
53
+ end.parse!(argv)
54
+ options
55
+ end
56
+
57
+ def extract_rspec_options argv
58
+ idx = argv.index('--') || -1
59
+ rspec_options = argv[idx+1..-1]
60
+ return if rspec_options.empty?
61
+ rspec_options.compact.join(' ')
62
+ end
63
+ end
64
+ end
data/lib/rspec/flaky/differ.rb ADDED
@@ -0,0 +1,61 @@
1
+ require 'hashdiff'
2
+ require 'fileutils'
3
+
4
+ module Differ
5
+
6
+ EXPECTED_CONTENT = %w(failed.json passed.json)
7
+
8
+ class << self
9
+ def get_result
10
+ create_summary_folder
11
+ @diffs = []
12
+ Pathes.summary_path.children.each do |example_dir|
13
+ next unless example_dir.directory?
14
+ next unless example_dir.basename.to_s.start_with?(".:")
15
+ example_dir.children.select do |tables_dir|
16
+ next unless tables_dir.directory?
17
+ @diffs << {
18
+ location: Pathes.relative_path(example_dir),
19
+ table: tables_dir.basename.to_s,
20
+ result: get_diffs(tables_dir)
21
+ }
22
+ end
23
+ end
24
+ Drawer.draw @diffs
25
+ end
26
+
27
+ def create_summary_folder
28
+ FileUtils.mkdir_p(Pathes.summary_path)
29
+ end
30
+
31
+ def get_diffs tables_dir
32
+ return 'no_content' unless contains_passed_and_failed_jsons? tables_dir
33
+ result = []
34
+ jsons = tables_dir.children.select { |child| EXPECTED_CONTENT.include? child.basename.to_s }
35
+ jsons = read_jsons(jsons)
36
+
37
+ return 'empty_table' if jsons.values.all? &:empty?
38
+ jsons.values.map(&:length).max.times do |idx|
39
+ passed_record = jsons["passed"].try(:[], idx) || {}
40
+ failed_record = jsons["failed"].try(:[], idx) || {}
41
+ result << Hashdiff.diff(passed_record, failed_record)
42
+ end
43
+ result
44
+ end
45
+
46
+ def read_jsons pathes
47
+ {}.tap do |hash|
48
+ pathes.each do |path|
49
+ hash[path.basename.to_s.split('.')[0]] = JSON.parse(File.read(Pathes.base_path.join(path)))
50
+ end
51
+ end
52
+ end
53
+
54
+ def contains_passed_and_failed_jsons? tables_dir
55
+ actual_content = tables_dir.children.map(&:basename).map(&:to_s)
56
+ EXPECTED_CONTENT & actual_content == EXPECTED_CONTENT
57
+ end
58
+ end
59
+
60
+
61
+ end
data/lib/rspec/flaky/drawer.rb ADDED
@@ -0,0 +1,21 @@
1
+ module Drawer
2
+
3
+ class << self
4
+ def draw diffs
5
+ @diffs = diffs
6
+ erb_str = File.read(Pathes.template_path)
7
+ result = ERB.new(erb_str, nil, '-').result(binding)
8
+ File.open(Pathes.summary_path.join('result.html'), 'w') do |f|
9
+ f.write(result)
10
+ end
11
+ end
12
+
13
+ def prettify(data)
14
+ return '-' if data.nil?
15
+ return JSON.pretty_generate(data) if data.is_a? Hash
16
+ data
17
+ end
18
+
19
+ end
20
+
21
+ end
data/lib/rspec/flaky/dumper/db.rb ADDED
@@ -0,0 +1,25 @@
1
+ require 'rails'
2
+
3
+ module Dumper
4
+ class Db
5
+
6
+ def dump!(location, status)
7
+ path = dump_path(location)
8
+ FileUtils.mkdir_p(path) unless File.exists?(path)
9
+ #TODO adapter for mysql and sqlite3
10
+ #TODO username from config
11
+ system "pg_dump -U postgres -d #{db_name} > #{path}/#{status}.sql"
12
+ end
13
+
14
+ private
15
+
16
+ def db_name
17
+ Rails.configuration.database_configuration["test"]["database"]
18
+ end
19
+
20
+ def dump_path location
21
+ "tmp/flaky_tests/database_dumps/#{location}"
22
+ end
23
+
24
+ end
25
+ end
data/lib/rspec/flaky/dumper/json.rb ADDED
@@ -0,0 +1,23 @@
1
+ require 'json'
2
+ require 'fileutils'
3
+
4
+ module Dumper
5
+ class Json
6
+
7
+ def dump!(location, status, tables)
8
+ tables.each do |table|
9
+ json = JSON.pretty_generate(table.all.map(&:attributes))
10
+ FileUtils.mkdir_p(dump_path(location, table)) unless File.exists?(dump_path(location, table))
11
+ File.open("#{dump_path(location, table)}/#{status}.json", 'w') do |f|
12
+ f.write(json)
13
+ end
14
+ end
15
+ end
16
+
17
+ private
18
+
19
+ def dump_path location, table
20
+ "tmp/flaky_tests/#{location}/#{table}"
21
+ end
22
+ end
23
+ end
data/lib/rspec/flaky/pathes.rb ADDED
@@ -0,0 +1,27 @@
1
+ module Pathes
2
+
3
+ class << self
4
+
5
+ def template_path
6
+ assets_path.join("layout.html.erb")
7
+ end
8
+
9
+ def summary_path
10
+ base_path.join('tmp/flaky_tests')
11
+ end
12
+
13
+ def base_path
14
+ Pathname.new(FileUtils.pwd)
15
+ end
16
+
17
+ def relative_path path
18
+ path.relative_path_from(summary_path).to_s
19
+ end
20
+
21
+ def assets_path
22
+ Pathname.new(__dir__).join("../../../assets/")
23
+ end
24
+
25
+ end
26
+
27
+ end
data/lib/rspec/flaky/tables.rb ADDED
@@ -0,0 +1,22 @@
1
+ require 'rspec'
2
+ require 'rspec/flaky/dumper/json'
3
+ require 'rspec/flaky/dumper/db'
4
+
5
+ RSpec.configure do |config|
6
+ if ENV["FLAKY_SPEC"] == "1"
7
+ config.before(:suite, table: true) do
8
+ config.after(:each, tables: true) do |example|
9
+ location = example.metadata[:location].gsub('/', ':')
10
+ status = example.exception.nil? ? 'passed' : 'failed'
11
+ tables = Array.wrap(example.metadata[:tables].select{|t| t < ApplicationRecord})
12
+
13
+ Dumper::Json.new.dump!(location, status, tables)
14
+ Dumper::Db.new.dump!(location, status)
15
+ end
16
+
17
+ config.before(:each, tables: true) do
18
+ DatabaseCleaner.strategy = :truncation
19
+ end
20
+ end
21
+ end
22
+ end
data/lib/rspec/flaky/version.rb ADDED
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+ module RSpecFlaky
3
+ VERSION = '0.0.1'
4
+ end
5
+
metadata ADDED
@@ -0,0 +1,71 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: rspec-flaky
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - maratz
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-12-28 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: hashdiff
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 1.0.1
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 1.0.1
27
+ description: Gem wraps every runned example to dump pointed database tables to json
28
+ files
29
+ email:
30
+ - mzasorinwd@gmail.com
31
+ executables:
32
+ - rspec-flaky
33
+ extensions: []
34
+ extra_rdoc_files: []
35
+ files:
36
+ - README.md
37
+ - assets/layout.html.erb
38
+ - bin/rspec-flaky
39
+ - lib/rspec/flaky.rb
40
+ - lib/rspec/flaky/cli.rb
41
+ - lib/rspec/flaky/differ.rb
42
+ - lib/rspec/flaky/drawer.rb
43
+ - lib/rspec/flaky/dumper/db.rb
44
+ - lib/rspec/flaky/dumper/json.rb
45
+ - lib/rspec/flaky/pathes.rb
46
+ - lib/rspec/flaky/tables.rb
47
+ - lib/rspec/flaky/version.rb
48
+ homepage:
49
+ licenses:
50
+ - MIT
51
+ metadata: {}
52
+ post_install_message:
53
+ rdoc_options: []
54
+ require_paths:
55
+ - lib
56
+ required_ruby_version: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ">="
59
+ - !ruby/object:Gem::Version
60
+ version: '0'
61
+ required_rubygems_version: !ruby/object:Gem::Requirement
62
+ requirements:
63
+ - - ">="
64
+ - !ruby/object:Gem::Version
65
+ version: '0'
66
+ requirements: []
67
+ rubygems_version: 3.0.3
68
+ signing_key:
69
+ specification_version: 4
70
+ summary: Gem for catching flaky specs
71
+ test_files: []