woodhouse 0.1.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.swp
+.bundle
+.binstubs
+Gemfile.lock
+html/

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - jruby-19mode
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+platforms :jruby do
+  gem 'hot_bunnies'
+end
+
+gemspec

data/Guardfile

@@ -0,0 +1,4 @@
+guard 'rspec', :version => 1 do
+  watch(%r{spec/.+_spec\.rb})
+  watch(%r{lib/(.+?)/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
+end

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2008 play/type GmbH
+Copyright (c) 2012 CrowdCompass, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/PROGRESS-NOTES.txt

@@ -0,0 +1,5 @@
+# Progress Notifications
+
+* Jobs have a _progress_id added to the arguments
+* A direct exchange 'woodhouse.jobs' is created
+* This _progress_id is used as the routing key

data/README.markdown

@@ -0,0 +1,152 @@
+# Woodhouse
+
+[<img src="https://secure.travis-ci.org/mboeh/woodhouse.png?branch=master" alt="Build Status" />](http://travis-ci.org/mboeh/woodhouse)
+
+An AMQP-based background worker system for Ruby designed to make managing heterogenous tasks relatively easy.
+
+The use case for Woodhouse is for reliable and sane performance in situations where jobs on a single queue may vary significantly
+in length. The goal is to permit large numbers of quick jobs to be serviced even when many slow jobs are in the queue. A secondary
+goal is to provide a sane way for jobs on a given queue to be given special priority or dispatched to a server more suited to them.
+
+Woodhouse 0.0.x is production-ready for Rails 2 and Ruby 1.8, while 0.1.x is in active development for Ruby 1.9.
+
+## Usage
+
+### Rails
+
+Add
+
+      gem 'woodhouse', github: 'mboeh/woodhouse'
+
+to your Gemfile.
+
+Run
+      
+      % rails generate woodhouse
+
+to create script/woodhouse and config/initializers/woodhouse.rb.
+
+### Basic Usage
+
+The simplest way to set up a worker class is to include Woodhouse::Worker and define public methods.
+
+      class IsisWorker
+        include Woodhouse::Worker
+
+        def pam_gossip(job)
+          puts "Pam gossips about #{job[:who]}."
+        end
+
+        def sterling_insult(job)
+          puts "Sterling insults #{job[:who]}."
+        end
+      end
+
+Jobs are dispatched asynchronously to a worker by adding `async_` to the method name:
+
+      IsisWorker.async_pam_gossip :who => "Cyril"
+
+Woodhouse jobs always take a hash of arguments. The worker receives a Woodhouse::Job, which acts like a hash
+but also supplies additional functionality.
+
+### Dispatchers
+
+The dispatcher used for sending out jobs can be set in the Woodhouse config block:
+
+      Woodhouse.configure do |woodhouse|
+        woodhouse.dispatcher_type = :local # :local_pool | :amqp
+      end
+      
+Calling the `async` version of a job method sends it to the currently configured dispatcher. The default dispatcher
+type is `:local`, which simply executes the job synchronously (although still passing it through middleware; see below).
+
+If you want `girl_friday` style in-process threaded backgrounding, you can get that by selecting the `:local_pool`
+dispatcher.
+
+Finally, if you want to run your jobs in a background process, you'll need to set up the `:amqp` dispatcher. This will
+use either the Hot Bunnies library (on JRuby) or the Bunny library (on all other Ruby engines). Bunny is suitable for
+dispatch but can be a little bit CPU-hungry in the background process. Hot Bunnies works great for both. You don't have
+to use the same Ruby version for your background process as for the dispatching application -- we use Woodhouse in production
+with a JRuby background process and MRI frontend processes.
+
+You'll also need to have RabbitMQ running. If it's running with the default (open) permissions on the local server, you don't
+need to configure it at all. Otherwise, you'll have to set the server connection info. You have two options for this. On Rails,
+you can create a config/woodhouse.yml file, formatted similar to config/database.yml:
+
+      production:
+        host: myrabbitmq.server.local
+        vhost: /some-vhost
+
+(The parameters accepted here are the same used for Bunny.connect; I promise to document them here soon.)
+
+Otherwise, you can do it in the Woodhouse config block:
+
+      Woodhouse.configure do |woodhouse|
+        woodhouse.server_info = { :host => "myrabbitmq.server.local" }
+      end
+
+### Running The Background Process
+
+All you have to do is run `script/woodhouse`. It'll load your Rails environment and start the server process. It responds to QUIT
+and INT signals correctly; I'm working on seeing if I can get it to restart worker processes with HUP and to dump/load the current
+layout with USR1/USR2.
+
+`script/woodhouse` logs job execution and results to `log/woodhouse.log`.
+
+### Performance Errata
+
+If you're using JRuby with a large application, I've found that the JVM's permanent generation can get exhausted. If you have
+plenty of heap but still get GC overhead errors, try bumping up the PermGen by including this on the JRuby command line:
+
+      -J-XX:MaxPermSize=128m
+
+Performance will generally be better with the `--server` flag:
+
+      --server
+
+A lot of the jobs I run tend to allocate and dispose a lot of memory very quickly, and Woodhouse will be a long-running process.
+I've gotten good results from enabling aggressive heap tuning:
+
+      -J-XX:+AggressiveHeap
+
+## Features
+
+* Configurable worker sets per server
+* Configurable number of threads per worker
+* Segmenting a single queue among multiple workers based on job characteristics (using AMQP header exchanges)
+* Progress reporting on jobs
+* New Relic background job reporting
+* Job dispatch and execution middleware stacks
+
+## Upcoming 
+
+* Live reconfiguration of workers -- add or remove workers across one or more nodes without restarting
+* Persistent configuration changes -- configuration changes saved to a data store and kept across deploys
+* Watchdog/status workers on every node
+* Web interface
+
+## To Do
+
+* Examples and guides
+* More documentation
+* Watchdog system
+
+## Supported Versions
+
+### woodhouse 0.1.x
+
+* bunny 0.9.x, RabbitMQ 2.x or later
+* ruby 1.9
+* MRI, JRuby, Rubinius 2
+
+### woodhouse 0.0.x
+
+* ruby 1.8
+* MRI, JRuby, Rubinius
+
+### Acknowledgements
+
+Woodhouse originated in a substantially modified version of the Workling background worker system, although all code has since
+been replaced.
+
+This library was developed for [CrowdCompass](http://crowdcompass.com) and was released as open source with their permission.

data/Rakefile

@@ -0,0 +1,35 @@
+require 'rubygems'
+require 'rake'
+require 'bundler/setup'
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'spec/rake/spectask'
+namespace :spec do
+  
+  Spec::Rake::SpecTask.new(:server) do |t|
+    t.spec_files = FileList["spec/**/*_spec.rb"] - FileList["spec/integration/*_spec.rb"]
+  end
+
+  Spec::Rake::SpecTask.new(:client) do |t|
+    t.spec_files = %w[spec/layout_spec.rb spec/middleware_stack_spec.rb spec/mixin_registry_spec.rb]
+  end
+
+end
+
+# Full server specs are supported on Ruby 1.9 or JRuby.
+if RUBY_VERSION.to_f >= 1.9 or %w[jruby rbx].include?(RUBY_ENGINE)
+  task :spec => "spec:server"
+else
+  task :spec => "spec:client"
+end
+
+task :default => :spec
+
+if ENV['RDOC']
+  require 'rdoc/task'
+  Rake::RDocTask.new(:rdoc) do |t|
+    t.main = "README.rdoc"
+    t.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+  end
+end

data/THOUGHTS

@@ -0,0 +1,84 @@
+How I want Ganymede to work:
+
+# app/workers/foo_bar_worker.rb
+class FooBarWorker < Ganymede::Worker
+
+  def foo(options)
+
+  end
+
+  def bar(options)
+
+  end
+
+end
+
+# somewhere that jobs are dispatched
+
+def do_a_thing(because)
+  @because = because
+  FooBarWorker.foo(self)
+end
+
+def to_ganymede
+  { 
+    :id => id,
+    :class => self.class.name,
+    :event_name => self.event.name,
+    :size => estimate_job_size,
+    :trigger => @because
+  }
+end
+
+# config/initializers/ganymede.rb
+
+# Default configuration: all workers listen to all jobs, one thread per job,
+# no filtering.
+Ganymede.configure do |config|
+  config.layout do |layout|
+    # layout.no_default_node! if you do not want a default node type
+    layout.node :bigjobs do |bigjobs|
+      bigjobs.worker :FooBarWorker, :only => { :size => "big" }, :threads => 2
+      bigjobs.worker :FooBarWorker, :only => { :size => "big", :trigger => "scheduled" }, :threads => 1
+    end
+    layout.default.worker :FooBarWorker, :only => { :size => "small", :trigger => "scheduled" }, :threads => 2
+    layout.default.worker :FooBarWorker, :only => { :size => "small", :trigger => "on-demand" }, :threads => 4
+    layout.default.worker :FooBarWorker, :only => { :size => "small", :trigger => "requested" }, :threads => 1
+  end
+  # config.middleware_out << Ganymede::Middleware::AutoSplatObjects
+end
+
+class Ganymede::Middleware::AutoSplatObjects
+
+  def call(*opts)
+    if opts.length == 1 and !opts.first.kind_of?(Hash)
+      if opts.respond_to?(:to_ganymede)
+        opts = opts.to_ganymede
+      end
+    end
+    yield opts
+  end
+
+end
+
+# equivalent to:
+layout = Ganymede::Layout.new
+default = Ganymede::Layout::Node.new(:default)
+default.default_configuration!
+default.add_worker Ganymede::Layout::Worker.new(:FooBarWorker, :foo, :only => { :size => "small", :trigger => "scheduled" }, :threads => 2)
+# ...
+layout.add_node default
+bigjobs = Ganymede::Layout::Node.new(:bigjobs)
+bigjobs.add_worker Ganymede::Layout::Worker.new(:FooBarWorker, :foo, :only => { :size => "big", :trigger => "scheduled" }, :threads => 1)
+
+server = Ganymede::Server.new(layout, :default)
+server.start # FIAT LUX
+
+# ... later ...
+# ... We really need a special worker just allocated for HugeEvent
+
+default.add_worker Ganymede::Worker.new(:FooBarWorker, :foo, :only => { :event_name => "HugeEvent" }, :threads => 1
+
+# Loads the new layout and starts up new workers as required
+server.layout = layout
+server.reload

data/doc/example/script-woodhouse

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+require File.dirname(__FILE__) + '/../config/environment'
+
+logg = Logger.new(File.dirname(__FILE__) + "/../log/woodhouse.log")
+logg.level = Logger::DEBUG
+Woodhouse.global_configuration.logger = logg
+Celluloid.logger = logg
+
+Woodhouse::Process.new.execute

data/doc/example/woodhouse-initializer.rb

@@ -0,0 +1,12 @@
+Woodhouse.configure do |woodhouse|
+  woodhouse.runner_middleware << Woodhouse::Middleware::AirbrakeExceptions
+end
+
+Woodhouse.layout do |layout|
+  layout.node(:default) do |node|
+    node.all_workers
+    node.remove :ImportWorker
+    node.add :ImportWorker, :threads => 2, :only => { :format => "csv" }
+    node.add :ImportWorker, :threads => 3, :only => { :format => "xml" }
+  end
+end

data/lib/generators/woodhouse_generator.rb

@@ -0,0 +1,38 @@
+class WoodhouseGenerator < Rails::Generators::Base
+  desc "Creates initializer and script files for Woodhouse."
+
+  def create_initializer
+    initializer "woodhouse.rb", <<-EOF
+Woodhouse.configure do |woodhouse|
+  # woodhouse.dispatcher_type = :amqp
+  # woodhouse.extension :progress
+  # woodhouse.extension :new_relic
+end
+
+Woodhouse.layout do |layout|
+  layout.node(:default) do |node|
+    node.all_workers
+  end
+end
+    EOF
+  end
+
+  def create_script
+    create_file "script/woodhouse", <<-EOF
+#!/usr/bin/env ruby
+require File.expand_path(File.dirname(__FILE__) + '/../config/environment')
+
+logg = Logger.new(File.dirname(__FILE__) + "/../log/woodhouse.log")
+logg.level = Logger::DEBUG
+logg.formatter = Logger::Formatter.new
+
+Celluloid.logger = logg
+Woodhouse.global_configuration.logger = logg
+
+Woodhouse.global_configuration.dispatcher_type = :amqp
+
+Woodhouse::Process.new.execute
+    EOF
+  end
+
+end

data/lib/woodhouse/dispatcher.rb

@@ -0,0 +1,37 @@
+class Woodhouse::Dispatcher
+
+  def initialize(config, opts = {}, &blk)
+    @config = config
+    after_initialize(config, opts = {}, &blk)
+  end
+
+  def dispatch(class_name, job_method, arguments)
+    dispatch_job Woodhouse::Job.new(class_name, job_method, arguments)
+  end
+
+  def dispatch_job(job)
+    @config.dispatcher_middleware.call(job) {|job|
+      deliver_job(job)
+    }
+    job
+  end
+
+  def update_job(job, data = {})
+    deliver_job_update(job, data)
+  end
+
+  private
+
+  def after_initialize
+
+  end
+
+  def deliver_job(job)
+    raise NotImplementedError, "implement #deliver_job in a subclass of Woodhouse::Dispatcher"
+  end
+
+  def deliver_job_update(job, data)
+    raise NotImplementedError, "implement #deliver_job_update in a subclass of Woodhouse::Dispatcher"
+  end
+
+end

data/lib/woodhouse/dispatchers/bunny_dispatcher.rb

@@ -0,0 +1,48 @@
+require 'bunny'
+require 'connection_pool'
+require 'woodhouse/dispatchers/common_amqp_dispatcher'
+
+class Woodhouse::Dispatchers::BunnyDispatcher < Woodhouse::Dispatchers::CommonAmqpDispatcher
+
+  def initialize(config)
+    super
+    @pool = new_pool
+  end
+
+  private
+
+  def publish_job(job, exchange)
+    exchange.publish(job.payload, :headers => job.arguments)
+  end
+
+  def run
+    retried = false
+    @pool.with do |conn|
+      yield conn
+    end
+  rescue Bunny::ClientTimeout
+    if retried
+      raise
+    else
+      new_pool!
+      retried = true
+      retry
+    end
+  end
+
+  private
+
+  def new_pool!
+    @pool = new_pool
+  end
+
+  def new_pool
+    @bunny.stop if @bunny
+
+    bunny = @bunny = Bunny.new(@config.server_info || {})
+    @bunny.start
+
+    ConnectionPool.new { bunny.create_channel }
+  end
+
+end

data/lib/woodhouse/dispatchers/common_amqp_dispatcher.rb

@@ -0,0 +1,28 @@
+# Provides common behavior shared by the Bunny and HotBunnies dispatchers.
+class Woodhouse::Dispatchers::CommonAmqpDispatcher < Woodhouse::Dispatcher
+
+  private
+
+  # Yields an AMQP channel to the block, doing any error handling or synchronization
+  # necessary.
+  def run(&blk)
+    raise NotImplementedError
+  end
+
+  def deliver_job(job)
+    run do |client|  
+      exchange = client.exchange(job.exchange_name, :type => :headers)
+      publish_job(job, exchange)
+    end
+  end
+
+  def deliver_job_update(job, data)
+    run do |client|
+      exchange = client.exchange("woodhouse.progress", :type => :direct)
+      # establish durable queue to pick up updates
+      client.queue(job.job_id, :durable => true).bind(exchange, :routing_key => job.job_id)
+      exchange.publish(data.to_json, :routing_key => job.job_id)
+    end
+  end
+
+end

data/lib/woodhouse/dispatchers/hot_bunnies_dispatcher.rb

@@ -0,0 +1,48 @@
+#
+# A Dispatcher implementation that uses hot_bunnies, a JRuby AMQP client using the
+# Java client for RabbitMQ. This class can be loaded if hot_bunnies is not
+# available, but it will fail upon initialization. If you want to use this
+# runner (it's currently the only one that works very well), make sure to
+# add
+#
+#   gem 'hot_bunnies'
+#
+# to your Gemfile.
+#
+
+require 'woodhouse/dispatchers/common_amqp_dispatcher'
+
+class Woodhouse::Dispatchers::HotBunniesDispatcher < Woodhouse::Dispatchers::CommonAmqpDispatcher
+
+  begin
+    require 'hot_bunnies'
+  rescue LoadError => err
+    define_method(:initialize) {|*args|
+      raise err
+    }
+  else
+    def initialize(config)
+      super
+      new_connection 
+      @mutex = Mutex.new 
+    end
+  end
+
+  private
+  
+  def run
+    @mutex.synchronize do
+      yield @channel
+    end
+  end
+
+  def publish_job(job, exchange)
+    exchange.publish(job.payload, :properties => { :headers => job.arguments })
+  end
+
+  def new_connection
+    @connection = HotBunnies.connect(@config.server_info)
+    @channel = @connection.create_channel
+  end
+
+end

data/lib/woodhouse/dispatchers/local_dispatcher.rb

@@ -0,0 +1,13 @@
+class Woodhouse::Dispatchers::LocalDispatcher < Woodhouse::Dispatcher
+
+  private
+
+  def deliver_job(job)
+    Woodhouse::JobExecution.new(@config, job).execute 
+  end
+
+  def deliver_job_update(job, data)
+    @config.logger.info "[Woodhouse job update] #{job.job_id} -- #{data.inspect}"
+  end
+
+end

data/lib/woodhouse/dispatchers/local_pool_dispatcher.rb

@@ -0,0 +1,25 @@
+class Woodhouse::Dispatchers::LocalPoolDispatcher < Woodhouse::Dispatcher
+
+  class Worker
+    include Celluloid
+
+    def execute(executor)
+      executor.execute
+    end
+  end
+
+  private
+
+  def after_initialize(config, opts = {}, &blk)
+    @pool = Worker.pool(size: opts[:size] || 10)
+  end
+
+  def deliver_job(job)
+    @pool.async.execute Woodhouse::JobExecution.new(@config, job)
+  end
+
+  def deliver_job_update(job, data)
+    @config.logger.info "[Woodhouse job update] #{job.job_id} -- #{data.inspect}"
+  end
+
+end

data/lib/woodhouse/dispatchers.rb

@@ -0,0 +1,19 @@
+module Woodhouse::Dispatchers
+
+  def self.default_amqp_dispatcher
+    if RUBY_ENGINE =~ /jruby/
+      Woodhouse::Dispatchers::HotBunniesDispatcher
+    else
+      Woodhouse::Dispatchers::BunnyDispatcher
+    end
+  end
+
+end
+
+require 'woodhouse/dispatcher'
+require 'woodhouse/dispatchers/local_dispatcher'
+require 'woodhouse/dispatchers/bunny_dispatcher'
+require 'woodhouse/dispatchers/hot_bunnies_dispatcher'
+require 'woodhouse/dispatchers/local_pool_dispatcher'
+
+Woodhouse::Dispatchers::AmqpDispatcher = Woodhouse::Dispatchers.default_amqp_dispatcher 

data/lib/woodhouse/extension.rb

@@ -0,0 +1,24 @@
+# Implements a very basic registry for Woodhouse extensions. This is a Class
+# rather than a Module because it will eventually be used to provide a more
+# structured approach than the one Woodhouse::Progress uses.
+class Woodhouse::Extension
+  
+  class << self
+
+    attr_accessor :registry
+
+    def register(name, extension)
+      registry[name] = extension
+    end
+
+    def install_extension(name, configuration, opts = {}, &blk)
+      if ext = registry[name]
+        ext.install_extension(configuration, opts, &blk)
+      end
+    end
+
+  end
+
+  self.registry = {}
+
+end

data/lib/woodhouse/extensions/new_relic/instrumentation_middleware.rb

@@ -0,0 +1,10 @@
+class Woodhouse::NewRelic::InstrumentationMiddleware < Woodhouse::Middleware
+  include NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+  def call(job, worker)
+    perform_action_with_newrelic_trace(:name => job.job_method, :class_name => job.worker_class_name, :params => job.arguments, :category => :task, :path => job.queue_name) do
+      yield job, worker
+    end
+  end
+
+end

data/lib/woodhouse/extensions/new_relic.rb

@@ -0,0 +1,23 @@
+require 'woodhouse'
+
+module Woodhouse::NewRelic
+  
+  class << self
+
+    def install_extension(configuration, opts = {}, &blk)
+      require 'woodhouse/extensions/new_relic/instrumentation_middleware'
+      configuration.runner_middleware << Woodhouse::NewRelic::InstrumentationMiddleware
+      configuration.at(:server_start) do
+        ::NewRelic::Agent.manual_start
+        configuration.logger.info "New Relic agent started."
+      end
+      configuration.at(:server_end) do
+        ::NewRelic::Agent.shutdown
+        configuration.logger.info "New Relic agent shut down."
+      end
+    end
+  end
+
+end
+
+Woodhouse::Extension.register :new_relic, Woodhouse::NewRelic