flakey_spec_catcher 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +4 -0
- data/Gemfile +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +156 -0
- data/Rakefile +10 -0
- data/flakey_spec_catcher.gemspec +38 -0
- data/lib/flakey_spec_catcher/version.rb +1 -1
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 71dd2fe6a58fc01c2647d06f6aa20f9c904f6ced4d1a54278d96344c40534713
|
4
|
+
data.tar.gz: cc060a637c1e14e04e7fd914b84c0b6c854912512a649fcf95696f57069444e1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 28114d12131611ae6405bd0c3d066b44c16c3efece45633e46b1c8f5a263eec7189c45b46efbf112897e5e7cbeecf110f493ee499c79d76647f64e2da4a99abe
|
7
|
+
data.tar.gz: be57cbc9ab2052f770507effd1b2e985124607bc5bfa11b0c659279726969aea9390df6b352501bdd31a1e179cb3c554fd448dc24c705f4974087772f502bd00
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2019 Instructure, Inc.
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,156 @@
|
|
1
|
+
# Flakey Spec Catcher (FSC)
|
2
|
+
|
3
|
+
### About:
|
4
|
+
|
5
|
+
This gem is intended to catch and prevent the merging of flakey ruby specs.
|
6
|
+
Currently, it will run the equivalent of a git diff between the current branch's
|
7
|
+
commit and origin/master to detect any changes to files that contain `_spec.rb`.
|
8
|
+
For these files, it will re-run either the changed tests or the whole spec
|
9
|
+
file's test suite a specified number of times.
|
10
|
+
|
11
|
+
Note that `flakey_spec_catcher` will only detect committed changes, so make sure
|
12
|
+
to add any changed spec files to the commit and amend it if you're testing
|
13
|
+
locally.
|
14
|
+
|
15
|
+
All that's required is to install the gem and then call its executable,
|
16
|
+
`flakey_spec_catcher`.
|
17
|
+
|
18
|
+
### Install:
|
19
|
+
|
20
|
+
```sh
|
21
|
+
gem install flakey_spec_catcher
|
22
|
+
```
|
23
|
+
|
24
|
+
Or add it to your Gemfile
|
25
|
+
|
26
|
+
```ruby
|
27
|
+
# Gemfile
|
28
|
+
gem 'flakey_spec_catcher'
|
29
|
+
```
|
30
|
+
|
31
|
+
and install with Bundler:
|
32
|
+
|
33
|
+
```sh
|
34
|
+
bundle install
|
35
|
+
```
|
36
|
+
|
37
|
+
### Configuration:
|
38
|
+
|
39
|
+
In some cases, certain environment variables can be configured:
|
40
|
+
|
41
|
+
- `FSC_REPEAT_FACTOR`: Number of times the examples in each file will be run.
|
42
|
+
Defaults to 20.
|
43
|
+
|
44
|
+
- `FSC_IGNORE_FILES`: Specify changed files that will be exempt from FSC
|
45
|
+
re-runs. Regex matching is allowed.
|
46
|
+
|
47
|
+
- `FSC_IGNORE_BRANCHES`: Ignore any committed changes related to the specified
|
48
|
+
branch(es).
|
49
|
+
|
50
|
+
- `FSC_SILENT_MODE`: If 'true', FSC will return back with a exit status 0
|
51
|
+
regardless of the result. If 'false', FSC will return a non-zero exit status
|
52
|
+
if flakey specs are detected.
|
53
|
+
|
54
|
+
- `FSC_RERUN_FILE_ONLY`: If 'true', FSC will re-run the whole spec file that
|
55
|
+
contains changes rather than only re-running each individual, changed example
|
56
|
+
or test.
|
57
|
+
|
58
|
+
- `FSC_USAGE_PATTERNS`: Specify RSpec re-run usage patterns for specific
|
59
|
+
directories or paths. For example, if you want FSC to re-run your changes in
|
60
|
+
'spec/ui' with `bundle exec rspec` and want your 'spec/api' changes to be
|
61
|
+
re-run with `parallel_rspec`, you could specify the following value:
|
62
|
+
`FSC_USAGE_PATTERNS='{ spec/ui/** => bundle exec rspec }, { spec/api => parallel_rspec }'`
|
63
|
+
|
64
|
+
Any of these environment variables can be overriden in the commit message by adding the environment variable key and
|
65
|
+
usage/value according to the accepted values specified above.
|
66
|
+
|
67
|
+
Example:
|
68
|
+
|
69
|
+
```
|
70
|
+
Commit Message
|
71
|
+
|
72
|
+
FSC_REPEAT_FACTOR='10'
|
73
|
+
FSC_USAGE_PATTERNS = '{ spec/ui => bundle exec rspec }, { spec/api => parallel_rspec }'
|
74
|
+
...
|
75
|
+
|
76
|
+
```
|
77
|
+
|
78
|
+
### How FSC works:
|
79
|
+
|
80
|
+
1. GitController runs a git diff and creates a ChangeSummary object to
|
81
|
+
represent the git diff for each changed spec file.
|
82
|
+
2. A ChangeCapsule uses the ChangeSummary object to go through each changed
|
83
|
+
file and identify ChangeContext objects.
|
84
|
+
3. After representing all changed code blocks as ChangeContext objects within
|
85
|
+
a ChangeCapsule (one per file), ChangeCapsules are stored in the
|
86
|
+
CapsuleManager.
|
87
|
+
4. User Configurations are parsed using the UserConfig class and an instance
|
88
|
+
is stored as an attribute in the Runner class.
|
89
|
+
5. UserConfig and CapsuleManager objects are used to initialize a RerunManager
|
90
|
+
object which creates a RerunCapsule object for each ChangeContext and pairs it
|
91
|
+
with an rspec usage based on user configurations.
|
92
|
+
6. RerunManager flattens the number of re-runs based on user settings and
|
93
|
+
passes back the identified file/test changes to the Runner along with their
|
94
|
+
specified usages.
|
95
|
+
7. Runner will re-run each of the changes at a file level or the specific
|
96
|
+
context-level of each test and will exit with a 0 if all changes passed all of
|
97
|
+
their re-runs or a non-zero exit status (passed from RSpec).
|
98
|
+
|
99
|
+
See class documentation for more details.
|
100
|
+
|
101
|
+
### Release:
|
102
|
+
|
103
|
+
We release `flakey_spec_catcher` to [RubyGems.org].
|
104
|
+
|
105
|
+
To release a new gem version:
|
106
|
+
|
107
|
+
1) Bump the gem version as appropriate in `lib/flakey_spec_catcher/version.rb`.
|
108
|
+
Follow [semantic versioning] principles. Commit, push to Gerrit, and merge.
|
109
|
+
|
110
|
+
2) Pull master and install dependencies for good measure:
|
111
|
+
|
112
|
+
```sh
|
113
|
+
git checkout master
|
114
|
+
git pull
|
115
|
+
bundle install
|
116
|
+
```
|
117
|
+
|
118
|
+
3) Now release the gem!
|
119
|
+
|
120
|
+
```sh
|
121
|
+
bundle exec rake release
|
122
|
+
```
|
123
|
+
|
124
|
+
This will build the gem, `git tag` HEAD with the current gem version, and push
|
125
|
+
the gem to [RubyGems.org].
|
126
|
+
|
127
|
+
To clean up after yourself:
|
128
|
+
|
129
|
+
```sh
|
130
|
+
bundle exec rake clobber
|
131
|
+
```
|
132
|
+
|
133
|
+
### Testing FSC:
|
134
|
+
|
135
|
+
Since FSC relies on git commit history to correctly identify the specs that it
|
136
|
+
will re-run, we have several specs that will create a temporary commit with
|
137
|
+
staged changes, run the corresponding model specs to verify functionality, and
|
138
|
+
then remove the temporary commit and staged changes. Upon merging a commmit,
|
139
|
+
code health and code coverage are both assessed in SonarQube.
|
140
|
+
|
141
|
+
See: <https://sonarqube.core.inseng.net/dashboard?id=tab%3Aflakey_spec_catcher>
|
142
|
+
|
143
|
+
To test your FSC changes, we've setup some scripts that will build docker images
|
144
|
+
and run tests in the latest ruby versions.
|
145
|
+
|
146
|
+
```sh
|
147
|
+
bash bin/build
|
148
|
+
bash bin/test
|
149
|
+
```
|
150
|
+
|
151
|
+
### License:
|
152
|
+
|
153
|
+
MIT. See LICENSE.txt for details.
|
154
|
+
|
155
|
+
[RubyGems.org]: https://rubygems.org/
|
156
|
+
[semantic versioning]: https://semver.org/
|
data/Rakefile
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# rubocop:disable Layout/ExtraSpacing, Layout/SpaceAroundOperators
|
4
|
+
require './lib/flakey_spec_catcher/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = 'flakey_spec_catcher'
|
8
|
+
gem.version = FlakeySpecCatcher::VERSION
|
9
|
+
gem.license = 'MIT'
|
10
|
+
gem.authors = [
|
11
|
+
'Brian Watson',
|
12
|
+
'Mikey Hargiss',
|
13
|
+
'Ben Nelson'
|
14
|
+
]
|
15
|
+
gem.email = [
|
16
|
+
'bwatson@instructure.com',
|
17
|
+
'mhargiss@instructure.com',
|
18
|
+
'bnelson@instructure.com'
|
19
|
+
]
|
20
|
+
gem.summary = 'Run new or changed specs many times to prevent unreliable specs'
|
21
|
+
|
22
|
+
gem.files = `git ls-files -z`
|
23
|
+
.split("\x0")
|
24
|
+
.reject { |f| f.match(%r{^(test|spec|features|fixture_specs|bin)/}) }
|
25
|
+
.reject { |f| f.match(/^(Jenkinsfile|Dockerfile|.dockerignore|.rspec|.rubocop)/) }
|
26
|
+
gem.bindir = 'bin'
|
27
|
+
gem.executables = 'flakey_spec_catcher'
|
28
|
+
gem.require_paths = ['lib']
|
29
|
+
|
30
|
+
gem.metadata['allowed_push_host'] = 'https://rubygems.org'
|
31
|
+
gem.required_ruby_version = '>= 2.3'
|
32
|
+
|
33
|
+
gem.add_dependency 'rspec', '~> 3.8'
|
34
|
+
gem.add_development_dependency 'climate_control', '~> 0.2'
|
35
|
+
gem.add_development_dependency 'rake', '~> 12.3'
|
36
|
+
gem.add_development_dependency 'simplecov', '~> 0.17'
|
37
|
+
end
|
38
|
+
# rubocop:enable Layout/ExtraSpacing, Layout/SpaceAroundOperators
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flakey_spec_catcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brian Watson
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2019-10-
|
13
|
+
date: 2019-10-11 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rspec
|
@@ -78,7 +78,13 @@ executables:
|
|
78
78
|
extensions: []
|
79
79
|
extra_rdoc_files: []
|
80
80
|
files:
|
81
|
+
- ".gitignore"
|
82
|
+
- Gemfile
|
83
|
+
- LICENSE.txt
|
84
|
+
- README.md
|
85
|
+
- Rakefile
|
81
86
|
- bin/flakey_spec_catcher
|
87
|
+
- flakey_spec_catcher.gemspec
|
82
88
|
- lib/flakey_spec_catcher.rb
|
83
89
|
- lib/flakey_spec_catcher/capsule_manager.rb
|
84
90
|
- lib/flakey_spec_catcher/change_capsule.rb
|