wolfpack 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 931aae52e2cabdbbc6f1e861eabb13477ce114a1
+  data.tar.gz: 2aa1c44933e7fc4735c32707a55f4d5db1ad956e
+SHA512:
+  metadata.gz: 1fb21574f5ea8a345c1803cafb2535c1b4baf2b64c6c1f83a210fa21ce81ace162887a222ad14b459cd775fddc51ee0012008f445c61ebd712947d85739e67cf
+  data.tar.gz: 9912bf0f401922b0d00d7f9bf40feb98bc7f8efa02b95e23ea1e4daf42bb54aa40b5eee5a827dcd32931248bf962d20c934e5ee1b94e5e8ca4874b919cbfac36

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wolfpack.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Brad Gessler
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,91 @@
+    @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@@@@@@@@@@@@@'~~~     ~~~`@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@@@@@@@@'                     `@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@@@@@'                           `@@@@@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@@@'                               `@@@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@'            Wolfpack               `@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@'                                     `@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@'     A really stupid way to run        `@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@      a bunch of tasks in parallel.      @@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@'                                         `@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@                                           @@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@                                           @@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@                       n,  Awhoooooooo!    @@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@                     _/ | _                @@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@                    /'  `'/                @@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@a                 <~    .'                a@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@                 .'    |                 @@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@a              _/      |                a@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@a           _/      `.`.              a@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@a     ____/ '   \__ | |______       a@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@@@a__/___/      /__\ \ \     \___.a@@@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@@@/  (___.'\_______)\_|_|        \@@@@@@@@@@@@@@@@@@@@@@@@
+    @@@@@@@@@@@@@@@@@@|\________                       ~~~~~\@@@@@@@@@@@@@@@@@@
+    ~~~\@@@@@@@@@@@@@@||       |\___________________________/|@/~~~~~~~~~~~\@@@
+        |~~~~\@@@@@@@/ |  |    | | by: S.C.E.S.W.          | ||\____________|@@
+
+## Why Wolfpack?
+
+One day you realize your CI server takes over an hour to run 2000 specs. Since you're an awesome developer and have fully embraced [The Twelve Factor App](http://12factor.net/) all you want to do is break this massive rspec runner into a bunch of sub-processes that can get the job done in parallel. Wouldn't be awesome if there was a little tool that would let you setup multiple environments of your app on the same machine and run tests against them? You're in luck, because that's what Wolfpack is all about!
+
+## Usage
+
+Wolfpack can be used for more than just a Rails app, but making specs run faster is such a great example to demonstrate how the whole thing works, so create a config file in your rails app.
+
+```ruby
+# config/wolfpack.rb
+after_fork do |n, args|
+  # This will create n different database variables. Rails `rake db:create db:schema:load`
+  # should pick this up for test setup.
+  ENV['DATABASE_URL'] = "mysql2://localhost/app_test_#{n}"
+
+  # Using redis? Cool! But lets partition that too. The bummer here is that
+  # redis limits us to 16 databases, so we should throw an exception just
+  # to make that clear.
+  raise 'Redis only supports 0-15 databases' unless n < 16
+  ENV['REDIS_URL'] = "redis://localhost/#{n}"
+
+  # We'll feed in a list of specs that we want to run from stdin,
+  # concat with a space, and output into an ENV var that rspec will pick up.
+  ENV['SPEC_FILES'] = args.join(' ')
+
+  # And whatever else you need to deal with...
+end
+```
+
+Then run wolfpack with the config file.
+
+```sh
+# Running the wolfpack command on a 4 processor machine. Your machine might 
+# run with a different number of workers depending on processors.
+$ find spec/**/**_spec.rb | wolfpack run 'echo "Using $DATABASE_URL"; rake db:create db:schema:load; rspec $SPEC_FILES;' -c config/wolfpack.rb
+Using mysql2://localhost/app_test_0
+Using mysql2://localhost/app_test_1
+Using mysql2://localhost/app_test_2
+Using mysql2://localhost/app_test_3
+# All of the spec output will be displayed here..
+```
+
+Pretty cool huh? As you can imagine, Wolfpack can be used to parallelize the execution of any unix program.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'wolfpack'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install wolfpack
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/wolfpack

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 'wolfpack'
+Wolfpack::CLI.start

data/examples/env_var.rb

@@ -0,0 +1,8 @@
+# This example sets environmental variables that mutates the child process.
+after_fork do |n, args|
+  # Set the WOLF var to the instance of the wolf.
+  ENV['WOLF'] = "Wolf #{n}"
+
+  # Space the arguments passed into wolfpack with spaces.
+  ENV['ARGS'] = args.join(' ')
+end

data/examples/rails_specs.rb

@@ -0,0 +1,29 @@
+# So, you're a http://12factor.net/ hotshot? You're in luck because
+# it will be pretty easy to get wolfpack working with your specs.
+
+after_fork do |n, args|
+  # This will create n different database variables. Rails `rake db:create db:schema:load`
+  # should pick this up for test setup.
+  ENV['DATABASE_URL'] = "mysql2://localhost/app_test_#{n}"
+
+  # Using redis? Cool! But lets partition that too. The bummer here is that
+  # redis limits us to 16 databases, so we should throw an exception just
+  # to make that clear.
+  raise 'Redis only supports 0-15 databases' unless n < 16
+  ENV['REDIS_URL'] = "redis://localhost/#{n}"
+
+  # We'll feed in a list of specs that we want to run from stdin,
+  # concat with a space, and output into an ENV var that rspec will pick up.
+  ENV['SPEC_FILES'] = args.join(' ')
+
+  # And whatever else you need to deal with...
+end
+
+__END__
+
+Now you're ready to run this! Try something like this on your CLI:
+
+$ bundle exec wolfpack exec "rake db:setup rake db:schema:load; rspec $SPEC_FILES;"
+
+and you're off to the races! In practice its probably not that easy though because
+specs are generally not developed initially with concurrency in mind.

data/lib/wolfpack/version.rb

@@ -0,0 +1,3 @@
+module Wolfpack
+  VERSION = "0.0.1"
+end

data/lib/wolfpack.rb

@@ -0,0 +1,118 @@
+require "wolfpack/version"
+require "thor"
+require "parallel"
+
+module Wolfpack
+  # Gets the number of "processors" on the current machine. This does not map
+  # directly to physical cores. For example, a hyperthreaded Intel chip may 
+  # have 2 physical cores but show up as 4 cores.
+  def self.processor_count
+    @processor_count ||= Parallel.processor_count
+  end
+
+  # Configure a runner instance
+  class Configurator
+    attr_reader :runner
+
+    # Configures the instance of a given runner with a file.
+    def initialize(runner, config_path)
+      @runner = runner
+      instance_eval(File.read(config_path), config_path)
+    end
+
+    # Replace the instances after_fork hook with after_fork 
+    # from the configuration file.
+    #
+    # @example
+    # after_fork do |n|
+    #   ENV['DATABASE_URL'] = "#{ENV['DATABASE_URL']}_#{n}"
+    # end
+    def after_fork(&block)
+      runner.after_fork = block
+    end
+  end
+
+  # Encapsulates a command that is te be split up and run.
+  class Runner
+    attr_accessor :after_fork, :command, :args
+
+    # Create a command that will run with the give arguments. Optionally
+    # a path may be given to a configuration file that sets up a runner.
+    def initialize(command, args = [], config_path = nil)
+      @command, @args = command, args
+      configure(config_path) if config_path
+    end
+
+    # Run the command `n` number of times. Default is the number of processes
+    # on the local machine.
+    def run(processes = nil)
+      # Sometimes a nil will make it here because of the CLI. This will
+      # make sure we have a number to work with.
+      processes ||= Wolfpack.processor_count
+
+      # Split args into groups of n processes.
+      partions = partition(args, processes)
+
+      # Now run the command with the processes.
+      Parallel.each_with_index(partions, :in_processes => processes) do |args, n|
+        after_fork.call(n, args) if after_fork
+        system @command
+      end
+    end
+
+    # Configures the runner's by reading a configuration file and eval-ing
+    # the ruby.
+    def configure(config_path)
+      Configurator.new(self, config_path)
+    end
+
+  private
+    # Take an array and break it into `n` partitions
+    def partition(arr, n)
+      result = Array.new(n){ [] }
+      arr.each_with_index do |el, idx|
+        result[idx % n].push el
+      end
+      result
+    end
+  end
+
+  class CLI < Thor
+    desc "exec COMMAND", "Runs many tasks in parallel"
+    method_options %w( config -c ) => :string
+    method_options %w( processes -n ) => :integer
+    method_options %w( args -a ) => :array
+    def exec(command)
+      # Parse out an integer for the # of processors the user specifies since
+      # thor doesn't return an integer for its params.
+      processes = options[:processes].to_i if options[:processes]
+
+      # Process stdin that's piped in.
+      args = if $stdin.tty?
+        options[:args] || []
+      else
+        # Read from the pipe
+        buffer = ""
+        until $stdin.eof? do
+          buffer << $stdin.read
+        end
+        buffer.lines
+      end
+
+      # If args are passed in, read those and split, otherwise read stdin 
+      # from a pipe and split by lines.
+
+      Wolfpack::Runner.new(command, args, options[:config]).run(processes)
+    end
+
+    desc "version", "Wolfpack version"
+    def version
+      puts Wolfpack::VERSION
+    end
+
+    desc "processors", "Count of processors on machine"
+    def processors
+      puts Wolfpack.processor_count
+    end
+  end
+end

data/spec/lib/wolfpack.rb

@@ -0,0 +1 @@
+require 'spec_helper'

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+require 'wolfpack'
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

data/wolfpack.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'wolfpack/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "wolfpack"
+  spec.version       = Wolfpack::VERSION
+  spec.authors       = ["Brad Gessler"]
+  spec.email         = ["brad@polleverywhere.com"]
+  spec.description   = %q{Run stuff in parallel}
+  spec.summary       = %q{Run ruby tasks in parallel}
+  spec.homepage      = "https://github.com/polleverywhere/wolfpack"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "guard-rspec"
+
+  spec.add_dependency "thor"
+  spec.add_dependency "parallel"
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: wolfpack
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Brad Gessler
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Run stuff in parallel
+email:
+- brad@polleverywhere.com
+executables:
+- wolfpack
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/wolfpack
+- examples/env_var.rb
+- examples/rails_specs.rb
+- lib/wolfpack.rb
+- lib/wolfpack/version.rb
+- spec/lib/wolfpack.rb
+- spec/spec_helper.rb
+- wolfpack.gemspec
+homepage: https://github.com/polleverywhere/wolfpack
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Run ruby tasks in parallel
+test_files:
+- spec/lib/wolfpack.rb
+- spec/spec_helper.rb