flakey_spec_catcher 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27339d7daba0c74bb7d57aec6a18685f1aa6034f11158a99e055e971eaebebe4
-  data.tar.gz: cc738a7d26cc97a63dec0becca3cc9a13244c4cae9906a5645174d8277357f84
+  metadata.gz: 71dd2fe6a58fc01c2647d06f6aa20f9c904f6ced4d1a54278d96344c40534713
+  data.tar.gz: cc060a637c1e14e04e7fd914b84c0b6c854912512a649fcf95696f57069444e1
 SHA512:
-  metadata.gz: cad188a9621417d13d104d0b26fe379e875daa2c4c571e64d6429acb04bd20898e8e727724e2916506e33d6a5bca8f9ca27175f3b154eebb5e3c096c92aaa7ce
-  data.tar.gz: 17dcab2f7900196b7bc388ab6e37e86fbbd394b4cffd5d8a29fa7f751b61b55ce58523f59420c3f5ac9c3c2c706a7ceed6fd003aebb17a320551c578233cabff
+  metadata.gz: 28114d12131611ae6405bd0c3d066b44c16c3efece45633e46b1c8f5a263eec7189c45b46efbf112897e5e7cbeecf110f493ee499c79d76647f64e2da4a99abe
+  data.tar.gz: be57cbc9ab2052f770507effd1b2e985124607bc5bfa11b0c659279726969aea9390df6b352501bdd31a1e179cb3c554fd448dc24c705f4974087772f502bd00

data/.gitignore

@@ -0,0 +1,4 @@
+/coverage
+/Gemfile.lock
+/flakey_spec_catcher-*.gem
+/pkg

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Instructure, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,156 @@
+# Flakey Spec Catcher (FSC)
+
+### About:
+
+This gem is intended to catch and prevent the merging of flakey ruby specs.
+Currently, it will run the equivalent of a git diff between the current branch's
+commit and origin/master to detect any changes to files that contain `_spec.rb`.
+For these files, it will re-run either the changed tests or the whole spec
+file's test suite a specified number of times.
+
+Note that `flakey_spec_catcher` will only detect committed changes, so make sure
+to add any changed spec files to the commit and amend it if you're testing
+locally.
+
+All that's required is to install the gem and then call its executable,
+`flakey_spec_catcher`.
+
+### Install:
+
+```sh
+gem install flakey_spec_catcher
+```
+
+Or add it to your Gemfile
+
+```ruby
+# Gemfile
+gem 'flakey_spec_catcher'
+```
+
+and install with Bundler:
+
+```sh
+bundle install
+```
+
+### Configuration:
+
+In some cases, certain environment variables can be configured:
+
+  - `FSC_REPEAT_FACTOR`: Number of times the examples in each file will be run.
+  Defaults to 20.
+
+  - `FSC_IGNORE_FILES`: Specify changed files that will be exempt from FSC
+  re-runs. Regex matching is allowed.
+
+  - `FSC_IGNORE_BRANCHES`: Ignore any committed changes related to the specified
+  branch(es).
+
+  - `FSC_SILENT_MODE`: If 'true', FSC will return back with a exit status 0
+  regardless of the result. If 'false', FSC will return a non-zero exit status
+  if flakey specs are detected.
+
+  - `FSC_RERUN_FILE_ONLY`: If 'true', FSC will re-run the whole spec file that
+  contains changes rather than only re-running each individual, changed example
+  or test.
+
+  - `FSC_USAGE_PATTERNS`: Specify RSpec re-run usage patterns for specific
+  directories or paths. For example, if you want FSC to re-run your changes in
+  'spec/ui' with `bundle exec rspec` and want your 'spec/api' changes to be
+  re-run with `parallel_rspec`, you could specify the following value:
+  `FSC_USAGE_PATTERNS='{ spec/ui/** => bundle exec rspec }, { spec/api => parallel_rspec }'`
+
+Any of these environment variables can be overriden in the commit message by adding the environment variable key and
+usage/value according to the accepted values specified above.
+
+Example:
+
+```
+Commit Message
+
+FSC_REPEAT_FACTOR='10'
+FSC_USAGE_PATTERNS = '{ spec/ui => bundle exec rspec }, { spec/api => parallel_rspec }'
+...
+
+```
+
+### How FSC works:
+
+  1. GitController runs a git diff and creates a ChangeSummary object to
+  represent the git diff for each changed spec file.
+  2. A ChangeCapsule uses the ChangeSummary object to go through each changed
+  file and identify ChangeContext objects.
+  3. After representing all changed code blocks as ChangeContext objects within
+  a ChangeCapsule (one per file), ChangeCapsules are stored in the
+  CapsuleManager.
+  4. User Configurations are parsed using the UserConfig class and an instance
+  is stored as an attribute in the Runner class.
+  5. UserConfig and CapsuleManager objects are used to initialize a RerunManager
+  object which creates a RerunCapsule object for each ChangeContext and pairs it
+  with an rspec usage based on user configurations.
+  6. RerunManager flattens the number of re-runs based on user settings and
+  passes back the identified file/test changes to the Runner along with their
+  specified usages.
+  7. Runner will re-run each of the changes at a file level or the specific
+  context-level of each test and will exit with a 0 if all changes passed all of
+  their re-runs or a non-zero exit status (passed from RSpec).
+
+See class documentation for more details.
+
+### Release:
+
+We release `flakey_spec_catcher` to [RubyGems.org].
+
+To release a new gem version:
+
+1) Bump the gem version as appropriate in `lib/flakey_spec_catcher/version.rb`.
+Follow [semantic versioning] principles. Commit, push to Gerrit, and merge.
+
+2) Pull master and install dependencies for good measure:
+
+```sh
+git checkout master
+git pull
+bundle install
+```
+
+3) Now release the gem!
+
+```sh
+bundle exec rake release
+```
+
+This will build the gem, `git tag` HEAD with the current gem version, and push
+the gem to [RubyGems.org].
+
+To clean up after yourself:
+
+```sh
+bundle exec rake clobber
+```
+
+### Testing FSC:
+
+Since FSC relies on git commit history to correctly identify the specs that it
+will re-run, we have several specs that will create a temporary commit with
+staged changes, run the corresponding model specs to verify functionality, and
+then remove the temporary commit and staged changes. Upon merging a commmit,
+code health and code coverage are both assessed in SonarQube.
+
+See: <https://sonarqube.core.inseng.net/dashboard?id=tab%3Aflakey_spec_catcher>
+
+To test your FSC changes, we've setup some scripts that will build docker images
+and run tests in the latest ruby versions.
+
+```sh
+bash bin/build
+bash bin/test
+```
+
+### License:
+
+MIT. See LICENSE.txt for details.
+
+[RubyGems.org]: https://rubygems.org/
+[semantic versioning]: https://semver.org/

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+
+CLOBBER.include(
+  'coverage/',
+  'pkg/',
+  'flakey_spec_catcher-*.gem',
+  'Gemfile.lock'
+)

data/flakey_spec_catcher.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# rubocop:disable Layout/ExtraSpacing, Layout/SpaceAroundOperators
+require './lib/flakey_spec_catcher/version'
+
+Gem::Specification.new do |gem|
+  gem.name    = 'flakey_spec_catcher'
+  gem.version = FlakeySpecCatcher::VERSION
+  gem.license = 'MIT'
+  gem.authors = [
+    'Brian Watson',
+    'Mikey Hargiss',
+    'Ben Nelson'
+  ]
+  gem.email   = [
+    'bwatson@instructure.com',
+    'mhargiss@instructure.com',
+    'bnelson@instructure.com'
+  ]
+  gem.summary = 'Run new or changed specs many times to prevent unreliable specs'
+
+  gem.files = `git ls-files -z`
+              .split("\x0")
+              .reject { |f| f.match(%r{^(test|spec|features|fixture_specs|bin)/}) }
+              .reject { |f| f.match(/^(Jenkinsfile|Dockerfile|.dockerignore|.rspec|.rubocop)/) }
+  gem.bindir        = 'bin'
+  gem.executables   = 'flakey_spec_catcher'
+  gem.require_paths = ['lib']
+
+  gem.metadata['allowed_push_host'] = 'https://rubygems.org'
+  gem.required_ruby_version         = '>= 2.3'
+
+  gem.add_dependency 'rspec', '~> 3.8'
+  gem.add_development_dependency 'climate_control', '~> 0.2'
+  gem.add_development_dependency 'rake', '~> 12.3'
+  gem.add_development_dependency 'simplecov', '~> 0.17'
+end
+# rubocop:enable Layout/ExtraSpacing, Layout/SpaceAroundOperators

data/lib/flakey_spec_catcher/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FlakeySpecCatcher
-  VERSION = '0.3.0'
+  VERSION = '0.3.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flakey_spec_catcher
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Brian Watson
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-10 00:00:00.000000000 Z
+date: 2019-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -78,7 +78,13 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
 - bin/flakey_spec_catcher
+- flakey_spec_catcher.gemspec
 - lib/flakey_spec_catcher.rb
 - lib/flakey_spec_catcher/capsule_manager.rb
 - lib/flakey_spec_catcher/change_capsule.rb