spec_tiller 1.1.3 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e4d10c226695ce9372ea1e73d7922b7840cd5c7
-  data.tar.gz: 99de02747309e0a2702d7065c4738a90151fe6f3
+  metadata.gz: d9860619a722afce5c9d9fd35042c134bad7e4a3
+  data.tar.gz: 7409ba1e7a22b25c7f1b51b6122cf7349412b998
 SHA512:
-  metadata.gz: 5365c4ec86b284ab4b826fab9439646b54683038d4cc5718dc787beb7c048fcaf27757f74173aff0e71728e7f25f225b09ee40c34ea49e8daa2003477dce7d57
-  data.tar.gz: 4b4b6f852f476291a52eb01ccaf042d090bc0e6ee8ec053c746918b0d0331623226e24835e73e90cd5875ff30885e87c22789ce799977dae80792e20cf71db02
+  metadata.gz: 22d2ed32885a2e3a48c160e8a4fbecc7d2ede996a9ba51d946d57f5422e91e7328e94bf2e8ae0883927b5179730e79f6952e435f40eb587c2bb541a9c6825df8
+  data.tar.gz: 7a36f2b751515ce1734e6ede356758e983cfd94e98278d6d0eaecd66c694499fb0ad7393687537880ab8c73d1d66151c6278352f4d4663fda895000c0da54912

data/.travis.yml

@@ -0,0 +1,22 @@
+---
+language: ruby
+rvm:
+- 2.1.2
+branches:
+  only:
+  - develop
+  - master
+  - "/^release\\/.*$/"
+  - "/^hotfix\\/.*$/"
+  - "/^feature\\/.*$/"
+cache: bundler
+script:
+- RSPEC_RETRY_COUNT=2 bundle exec rspec $TEST_SUITE --tag ~local_only
+- if [[ $RUN_JS == "true" ]]; then bundle exec rake jasmine:ci; fi
+env:
+  global:
+  - IGNORE_SPECS="spec/documents/*"
+  matrix:
+  - TEST_SUITE="spec/tasks/spec_tiller_rake_spec.rb"
+  - TEST_SUITE="spec/build_matrix_parser_spec.rb spec/distribute_spec_files_spec.rb spec/sync_spec_file_list_spec.rb"
+num_builds: 2

data/README.md

@@ -17,7 +17,10 @@ And then execute:
 
 ## Set Up
 
-#### Git Hooks  
+#### Git Hooks
+
+###### Sync
+
 In order to make sure that the travis file remains up to date with any newly added or removed spec files, you'll have to include a rake task in your pre-commit and post-merge git hooks. Create two files in .git/hooks/ --> ``pre-commit`` and ``post-merge`` (no file extension). **Please make sure that these files are executable** (more information on [git hooks](http://git-scm.com/book/en/Customizing-Git-Git-Hooks)). In each of those two files, add the following rake task:  
 
     #!/bin/sh  
@@ -29,6 +32,17 @@ Upon setting this up, any time you commit or merge, you'll notice the following
     Removed: [spec/file/removed_spec.rb]  
     Added:   [spec/file/added_spec.rb, spec/file/another_added_spec.rb]
 
+###### Redistribute
+
+You can also set up a hook to redistribute spec files with each commit. Because this task looks at previous travis builds for a specified branch, it's best to either add it to your post merge hook or point it to a branch that you know has builds on travis. Follow the instructions above and create a file that looks like this:
+
+    #!/bin/sh
+    rake spec_tiller:redistribute [ BRANCH='branch_name' ]
+
+If you don't specify a branch_name, 'develop' will be used. You shouldn't use the 'local' option (see below) in a git hook - it may take a long time to run.
+
+The sync task is called from the redistribute task, so you shouldn't need both hooks.
+
 ####.travis.yml
 In your ``.travis.yml`` file, create a top-level variable **num_builds** and set it to the number of builds you want the spec files distributed over (default value is 5 builds). You must also make sure that the base of your script is as follows:
 
@@ -72,11 +86,77 @@ Here is an example of what a ``.travis.yml`` file may look like after all is sai
     num_builds: 5
 
 #### Redistributing Files
-Initially, and every so often, you will have to redistribute the spec files (make sure everything is properly set up before running this rake task). The *spec_tiller:sync* rake task adds any new spec files to the last bucket, so after a little while, the timing of the build may not be optimal. In order to redistribute the spec files in order to optimize test suite run time, run the following rake task:
+In order to keep your jobs relatively even in length (*spec_tiller:sync* rake task adds new files to a random job), you should run redistribute occasionally. Redistribute uses the rspec profile results to distribute specs into jobs that will run in roughly the same amount of time. You can call the task as specified below:
+
+    rake spec_tiller:redistribute [ BRANCH='branch_name']
+
+where the 'branch_name argument is optional. There are two ways to set up the redistribute task.
+
+######Local Redistribute
+This option is the most simple - it doesn't require any extra setup. The task will run your whole test suite locally and use the profile results to redistribute your files. However, because it needs to be run locally, it works best for projects with a relatively small test suite. Larger projects should use the travis setup. To run 'local' redistribute, specify 'local' as the branch name:
+
+    rake spec_tiller:redistribute BRANCH='local'
+
+######Travis Redistribute
+This option uses profile results from past travis builds to redistribute your test suite. After it is finished running, the sync task is invoked to ensure any new files were also added. There are a few extra pieces of setup required:
+
+  * Add the --profile 1000000000 rspec option to your travis script. This will cause travis to print all profile results in the build log. It should look something like this:
+
+      bundle exec rspec $TEST_SUITE --tag ~local_only --profile 1000000000
+
+  * Create a github auth key for travis and add it to your ~./bash_profile. The key should be called GITHUB_TOKEN_FOR_TRAVIS_API and should have a value of the key. You can create a key on this page: [https://github.com/settings/applications](https://github.com/settings/applications). Generate a new personal access token for travis with the following scopes:
+
+      repo, public_repo, repo:status, read:org, read:public_key
+
+  * Optionally, you can set up your travis script so that profile results can be folded. This will prevent your build log from getting cluttered. One approach is to store profile results in another file, then call it in an after script. There are two ways to do this, but either way you'll need to include this after script in your travis file:
+
+      ``.travis.yml``:
+
+        after_script:
+        - cat /tmp/profile_results.txt
+
+  * The best way is to use a custom rspec formatter. An example exists in this project at spec/support/formatters/profile_to_file_formatter.rb. You can tell rspec to use the formatter in your spec_helper or rails_helper:
+
+      ``spec_helper.rb``:
+
+        RSpec.configure do |config|
+
+          if ENV['CI_ENV'] == 'travis'
+            config.add_formatter('progress')
+            config.add_formatter(ProfileToFileFormatter)
+          end
+        end
+
+  * The other option is to read stdout and watch for the profile results. You'll most likely want to move your script into a separate file, then update your travis script to run from that file. See examples below:
+
+      ``.travis.yml``:
+
+        script: ./scripts/travis_script.sh
+
+      ``scripts/travis_script.sh``:
+
+        #!/bin/bash
+
+        set -v
+
+        bundle exec rspec $TEST_SUITE --tag ~local_only --profile 1000000000 --tty | while read line
+        do
+          if [[ $line =~ Top.[0-9]+.slowest.examples ]]; then
+            INPROF=true
+            echo "$line" > /tmp/profile_results.txt
+          elif [[ $line =~ Coverage.report.generated ]]; then
+            INPROF=false
+          elif [[ $INPROF == true ]]; then
+            echo "$line" >> /tmp/profile_results.txt
+          else
+            echo -e "$line"
+          fi
+        done
+
 
-    spec_tiller:redistribute
+To call travis redistribute, specify the branch that should be used as the base. The most recent build for the branch must have the '--profile 1000000000' option specified. If no branch is specified, 'develop' will be used. Below is an example call:
 
-This will run your whole test suite, keeping track of how long each spec file takes to run, and the will distribute the spec files in order to maximize your test suite's run time, over the number of builds you've designated (with a default value of 5). **This rake task will not print the output of the rake task until it is complete.**
+    rake spec_tiller:redistribute BRANCH='develop'
 
 #### Ignoring Files
 By default, both sync and redistribute will look for any tests following the "spec/**/*_spec.rb" pattern. If you want the tasks to ignore any specs, you can add the IGNORE_SPECS variable to your global variables. The value should be the patterns or specs you want to exclude, separated by spaces.

data/lib/spec_tiller/sync_spec_file_list.rb

@@ -26,14 +26,29 @@ module SyncSpecFiles
 
   module_function :rewrite_travis_content
 
+  def self.sync
+    content = YAML::load(File.open('.travis.yml'))
+    current_file_list = Dir.glob('spec/**/*_spec.rb').map { |file_path| file_path.slice(/(spec\/\S+$)/) }
+
+    puts "\nSyncing list of spec files..."
+
+    SyncSpecFiles.rewrite_travis_content(content, current_file_list) do |content, original, current_file_list|
+      File.open('.travis.yml', 'w') { |file| file.write(content.to_yaml(:line_width => -1)) }
+      puts SyncSpecFiles.file_diff(original, current_file_list)
+    end
+
+    `git add .travis.yml`
+  end
+
   def self.get_ignored_specs(content)
     ignore_specs = []
     content['env']['global'].each do |row|
-      next unless row.is_a?(String)
-      # Input: IGNORE_SPECS="spec/a.rb spec/b.rb"
-      # Output: ['spec/a.rb spec/b.rb']
-      matches = row.match(/IGNORE_SPECS="\s*([^"]+)"/)
-      ignore_specs << matches[1].split(' ') unless matches.nil?
+      if row.is_a?(String)
+        # Input: IGNORE_SPECS="spec/a.rb spec/b.rb"
+        # Output: ['spec/a.rb spec/b.rb']
+        matches = row.match(/IGNORE_SPECS="\s*([^"]+)"/)
+        ignore_specs << matches[1].split(' ') unless matches.nil?
+      end
     end
     ignore_specs.flatten
   end
@@ -57,9 +72,11 @@ module SyncSpecFiles
 
     def self.add_new_files(original, buckets, current_file_list)
       buckets_clone = buckets.map(&:dup)
+      num_buckets = buckets.length
 
       added_files(original, current_file_list).each do |spec_file|
-        buckets_clone.last << spec_file
+        bucket_index = rand(num_buckets)
+        buckets_clone[bucket_index] << spec_file
       end
 
       buckets_clone

data/lib/spec_tiller/travis_api.rb

@@ -0,0 +1,47 @@
+require 'travis'
+
+class TravisAPI
+
+  def self.get_logs(branch)
+    client = Travis::Client.new('https://api.travis-ci.com')
+    client.github_auth(ENV.fetch('GITHUB_TOKEN_FOR_TRAVIS_API'))
+    repository = client.repo(current_repo)
+
+    raise 'Repository not found. Ensure Fetch URL of "git remote show origin" points to your repository.' if repository.nil?
+
+    raise "Branch #{branch} not found in current repository." unless repository.branches.key?(branch)
+
+    last_build = most_recent_build_for(repository, branch)
+
+    logs_for(last_build)
+  end
+
+
+
+    def self.current_repo
+      # Input:
+      #   ...
+      #   Fetch URL: git@github.com:grnhse/spec-tiller.git
+      #   ...
+      # Output: grnhse/spec-tiller
+      `git remote show -n origin`.match(/Fetch URL: .*:(.+).git/)[1]
+    end
+
+    def self.most_recent_build_for(repository, branch)
+      repository.each_build do |build|
+        if build.commit.branch == branch && build.state == 'passed'
+          return build
+        end
+      end
+
+      raise "No passing builds found for #{branch}."
+    end
+
+    def self.logs_for(build)
+      build.
+        jobs.
+        map{ |j| j.log.body }.
+        compact.
+        join('\n')
+    end
+end

data/lib/spec_tiller/version.rb

@@ -1,3 +1,3 @@
 module SpecTiller
-  VERSION = '1.1.3'
+  VERSION = '1.2.0'
 end

data/lib/tasks/spec_tiller.rake

@@ -1,36 +1,37 @@
+require 'spec_tiller/travis_api'
+
 namespace :spec_tiller do
   desc 'Compares spec files in travis.yml to current list of spec files, and syncs accordingly'
   task :sync do
-    content = YAML::load(File.open('.travis.yml'))
-    current_file_list = Dir.glob('spec/**/*_spec.rb').map { |file_path| file_path.slice(/(spec\/\S+$)/) }
-
-    puts "\nSyncing list of spec files..."
-
-    SyncSpecFiles.rewrite_travis_content(content, current_file_list) do |content, original, current_file_list|
-      File.open('.travis.yml', 'w') { |file| file.write(content.to_yaml(:line_width => -1)) }
-      puts SyncSpecFiles.file_diff(original, current_file_list)
-    end
-
-    `git add .travis.yml`
+    SyncSpecFiles.sync
   end
 
   desc 'Runs whole test suite and redistributes spec files across builds according to file run time'
   task :redistribute => :environment do
+
+    branch = ENV['BRANCH'] ? ENV['BRANCH'] : 'develop'
+
+    `echo "#{branch}"`
     travis_yml_file = YAML::load(File.open('.travis.yml'))
-    env_variables = travis_yml_file['env']['global']
-    script = travis_yml_file['script'].first.gsub('$TEST_SUITE ', '')
 
-    ignore_specs = SyncSpecFiles.get_ignored_specs(travis_yml_file).map { |spec| %Q("#{spec}") }
-    script += %Q( --exclude-pattern #{ignore_specs.join(',')}) unless ignore_specs.empty?
+    if branch == 'local'
+      env_variables = travis_yml_file['env']['global']
+      script = travis_yml_file['script'].first.gsub('$TEST_SUITE ', '')
 
-    profile_results = `#{env_variables.join(' ')} #{script} --profile 1000000000`
+      ignore_specs = SyncSpecFiles.get_ignored_specs(travis_yml_file).map { |spec| %Q("#{spec}") }
+      script += %Q( --exclude-pattern #{ignore_specs.join(',')}) unless ignore_specs.empty?
+
+      profile_results = `#{env_variables.join(' ')} #{script} --profile 1000000000`
+    else
+      profile_results = TravisAPI.get_logs(branch)
+    end
 
-    `echo "#{profile_results}" > spec/log/rspec_profile_output.txt`
     TravisBuildMatrix::SpecDistributor.new(travis_yml_file, profile_results) do |content|
       File.open('.travis.yml', 'w') { |file| file.write(content.to_yaml(:line_width => -1)) }
     end
 
-    puts profile_results
+    SyncSpecFiles.sync
+
   end
 
-end
+end

data/spec/spec_helper.rb

@@ -3,6 +3,12 @@ Bundler.setup
 
 require 'spec_tiller' # and any other gems you need
 
+Dir["./spec/support/**/*.rb"].sort.each { |f| require f }
+
 RSpec.configure do |config|
-  # some (optional) config here
+
+  if ENV['CI_ENV'] == 'travis'
+    config.add_formatter('progress')
+    config.add_formatter(ProfileToFileFormatter)
+  end
 end

data/spec/support/formatters/profile_to_file_formatter.rb

@@ -0,0 +1,17 @@
+require 'rspec'
+require 'rspec/core/formatters/profile_formatter'
+
+class ProfileToFileFormatter < RSpec::Core::Formatters::ProfileFormatter
+  RSpec::Core::Formatters.register(self, :dump_profile)
+
+  def initialize(_)
+    @output = File.new('/tmp/profile_results.txt', 'w')
+
+  end
+
+  # This class really just changes @output,
+  # everything else can function as it does in ProfileFormatter
+  def dump_profile(profile)
+    super(profile)
+  end
+end

data/spec/sync_spec_file_list_spec.rb

@@ -18,19 +18,8 @@ describe 'SyncSpecFiles' do
       SyncSpecFiles.rewrite_travis_content(travis_yaml, current_file_list)
     end
 
-    it 'adds new files to the last line' do
-      last_line = travis_yaml['env']['matrix'].last
-
-      expect(last_line).to include('spec/test/new1.rb','spec/test2/new2.rb','spec/test/new3.rb')
-    end
-
-    it "doesn't add new files to other lines" do
-      travis_yaml['env']['matrix'][0..-2].each do |bucket|
-        expect(bucket).not_to include('spec/test/new1.rb')
-        expect(bucket).not_to include('spec/test2/new2.rb')
-        expect(bucket).not_to include('spec/test/new3.rb')
-      end
-
+    it 'adds new files to random line' do
+      expect(travis_yaml['env']['matrix'].join(' ')).to include('spec/test/new1.rb','spec/test2/new2.rb','spec/test/new3.rb')
     end
 
     it 'removes unused specs' do
@@ -40,8 +29,10 @@ describe 'SyncSpecFiles' do
 
     end
 
-    it 'removes unused buckets' do
-      expect(travis_yaml['env']['matrix'].length).to eq(2)
+    it 'removes lines without a TEST_SUITE' do
+      travis_yaml['env']['matrix'].each do |bucket|
+        expect(bucket).to include('TEST_SUITE="')
+      end
     end
 
     it 'does not include ignored specs' do

data/spec/tasks/spec_tiller_rake_spec.rb

@@ -12,14 +12,14 @@ describe 'spec_tiller:sync' do
     end
   end
 
-  it 'adds new files to last line' do
+  it 'adds new files to random line' do
     temp_file_name = 'spec/documents/new_added_spec.rb'
     File.new(temp_file_name, 'w')
     content_at_start = YAML::load(File.open('.travis.yml'))
 
     task.invoke
     content_at_end = YAML::load(File.open('.travis.yml'))
-    expect(content_at_end['env']['matrix'].last).to include(temp_file_name)
+    expect(content_at_end['env']['matrix'].join(' ')).to include(temp_file_name)
 
     # Clean up added file and travis.yml
     File.delete(temp_file_name)
@@ -45,9 +45,13 @@ end
 
 describe 'spec_tiller:redistribute' do
   include_context('rake')
+  before do
+    allow(SyncSpecFiles).to receive(:sync)
+  end
 
   it 'distributes evenly based on run time' do
     content_at_start = YAML::load(File.open('.travis.yml'))
+    ENV['BRANCH'] = 'local'
     task.invoke
     content_at_end = YAML::load(File.open('.travis.yml'))
 

data/spec_tiller.gemspec

@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "railties"
+  spec.add_dependency "travis", ">= 1.6.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spec_tiller
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.2.0
 platform: ruby
 authors:
 - Matt Schmaus
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-07 00:00:00.000000000 Z
+date: 2015-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -67,6 +67,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: travis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
 description: |2
       This gem will parse the output of calling "rspec --perform", then will redistribute
       the spec files evenly, based on file run time, across all builds established
@@ -79,6 +93,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -88,6 +103,7 @@ files:
 - lib/spec_tiller/distribute_spec_files.rb
 - lib/spec_tiller/railtie.rb
 - lib/spec_tiller/sync_spec_file_list.rb
+- lib/spec_tiller/travis_api.rb
 - lib/spec_tiller/version.rb
 - lib/tasks/spec_tiller.rake
 - spec/build_matrix_parser_spec.rb
@@ -101,6 +117,7 @@ files:
 - spec/documents/iterator5_spec.rb
 - spec/documents/rspec_profile_results.txt
 - spec/spec_helper.rb
+- spec/support/formatters/profile_to_file_formatter.rb
 - spec/support/shared_contexts/rake.rb
 - spec/sync_spec_file_list_spec.rb
 - spec/tasks/spec_tiller_rake_spec.rb
@@ -141,6 +158,7 @@ test_files:
 - spec/documents/iterator5_spec.rb
 - spec/documents/rspec_profile_results.txt
 - spec/spec_helper.rb
+- spec/support/formatters/profile_to_file_formatter.rb
 - spec/support/shared_contexts/rake.rb
 - spec/sync_spec_file_list_spec.rb
 - spec/tasks/spec_tiller_rake_spec.rb