woodhouse 0.1.1

data/lib/woodhouse/runners/hot_bunnies_runner.rb

@@ -0,0 +1,79 @@
+#
+# A Runner implementation that uses hot_bunnies, a JRuby AMQP client using the
+# Java client for RabbitMQ. This class can be loaded if hot_bunnies is not
+# available, but it will fail upon initialization. If you want to use this
+# runner (it's currently the only one that works very well), make sure to
+# add
+#
+#   gem 'hot_bunnies'
+#
+# to your Gemfile. This runner will automatically be used in JRuby.
+#
+class Woodhouse::Runners::HotBunniesRunner < Woodhouse::Runner
+  begin
+    require 'hot_bunnies'
+  rescue LoadError => err
+    define_method(:initialize) {|*args|
+      raise err
+    }
+  end
+
+  def subscribe
+    client = HotBunnies.connect(@config.server_info)
+    channel = client.create_channel
+    channel.prefetch = 1
+    queue = channel.queue(@worker.queue_name)
+    exchange = channel.exchange(@worker.exchange_name, :type => :headers)
+    queue.bind(exchange, :arguments => @worker.criteria.amqp_headers)
+    worker = Celluloid.current_actor
+    queue.subscribe(:ack => true).each(:blocking => false) do |headers, payload|
+      begin
+        job = make_job(headers, payload)
+        if can_service_job?(job)
+          if service_job(job)
+            headers.ack
+          else
+            headers.reject
+          end
+        else
+          @config.logger.error("Cannot service job #{job.describe} in queue for #{@worker.describe}")
+          headers.reject
+        end
+      rescue => err
+        begin
+          @config.logger.error("Error bubbled up out of worker. This shouldn't happen. #{err.message}")
+          err.backtrace.each do |btr|
+            @config.logger.error("  #{btr}")
+          end
+          headers.reject
+        ensure
+          worker.bail_out(err)
+        end
+      end
+    end
+    wait :spin_down
+  ensure
+    client.close
+  end
+
+  def spin_down
+    signal :spin_down
+  end
+
+  def bail_out(err)
+    raise Woodhouse::BailOut, "#{err.class}: #{err.message}"
+  end
+
+  private
+
+  def make_job(message, payload)
+    Woodhouse::Job.new(@worker.worker_class_name, @worker.job_method) do |job|
+      job.arguments = message.properties.headers.inject({}) {|h,(k,v)|
+        h[k.to_s] = v.to_s
+        h
+      }
+      job.payload = payload
+    end
+  end
+
+end

data/lib/woodhouse/runners.rb

@@ -0,0 +1,16 @@
+module Woodhouse::Runners
+  
+  def self.guess
+    if defined? ::JRUBY_VERSION
+      Woodhouse::Runners::HotBunniesRunner
+    else
+      Woodhouse::Runners::BunnyRunner
+    end
+  end
+
+end
+
+require 'woodhouse/runner'
+require 'woodhouse/runners/bunny_runner'
+require 'woodhouse/runners/hot_bunnies_runner'
+require 'woodhouse/runners/dummy_runner'

data/lib/woodhouse/scheduler.rb

@@ -0,0 +1,113 @@
+class Woodhouse::Scheduler
+  include Woodhouse::Util
+  include Celluloid
+
+  class SpunDown < StandardError
+
+  end
+
+  class WorkerSet
+    include Woodhouse::Util
+    include Celluloid
+
+    attr_reader :worker
+    trap_exit :worker_died
+
+    def initialize(scheduler, worker, config)
+      expect_arg_or_nil :worker, Woodhouse::Layout::Worker, worker
+      @scheduler = scheduler
+      @worker_def = worker
+      @config = config
+      @threads = []
+      spin_up
+    end
+
+    def spin_down
+      @spinning_down = true
+      @threads.each_with_index do |thread, idx|
+        @config.logger.debug "Spinning down thread #{idx} for worker #{@worker_def.describe}"
+        thread.spin_down
+        thread.terminate
+      end
+      @scheduler.remove_worker(@worker_def)
+      signal :spun_down
+    end
+
+    def wait_until_done
+      wait :spun_down
+    end
+
+    private
+
+    def worker_died(actor, reason)
+      if reason
+        @config.logger.info "Worker died (#{reason.class}: #{reason.message}). Spinning down."
+        @threads.delete actor
+        raise Woodhouse::BailOut
+      end
+    end
+
+    def spin_up
+      @worker_def.threads.times do |idx|
+        @config.logger.debug "Spinning up thread #{idx} for worker #{@worker_def.describe}"
+        worker = @config.runner_type.new_link(@worker_def, @config)
+        @threads << worker
+        worker.subscribe!
+      end
+    end
+
+  end
+
+  trap_exit :worker_set_died
+
+  def initialize(config)
+    @config = config
+    @worker_sets = {}
+  end
+
+  def start_worker(worker)
+    @config.logger.debug "Starting worker #{worker.describe}"
+    unless @worker_sets.has_key?(worker)
+      @worker_sets[worker] = WorkerSet.new_link(Celluloid.current_actor, worker, @config)
+      true
+    else
+      false
+    end
+  end
+
+  def stop_worker(worker, wait = false)
+    if set = @worker_sets[worker]
+      @config.logger.debug "Spinning down worker #{worker.describe}"
+      set.spin_down
+    end
+  end
+  
+  def running_worker?(worker)
+    @worker_sets.has_key?(worker)
+  end
+
+  def spin_down
+    @spinning_down = true
+    @config.logger.debug "Spinning down all workers"
+    @worker_sets.each do |worker, set|
+      set.spin_down
+      set.terminate
+    end
+  end
+
+  def remove_worker(worker)
+    @worker_sets.delete(worker)
+  end
+
+  def worker_set_died(actor, reason)
+    if reason
+      @config.logger.info "Worker set died (#{reason.class}: #{reason.message}). Spinning down."
+      begin
+        spin_down
+      ensure
+        raise reason
+      end
+    end
+  end
+
+end

data/lib/woodhouse/server.rb

@@ -0,0 +1,80 @@
+module Woodhouse
+
+  class Server
+    include Celluloid
+    include Woodhouse::Util
+
+    attr_reader :layout, :node
+    attr_accessor :configuration
+
+    trap_exit :scheduler_died
+
+    def initialize(keyw = {})
+      self.layout        = keyw[:layout]
+      self.node          = keyw[:node]
+      self.configuration = keyw[:configuration] || Woodhouse.global_configuration
+    end
+
+    def layout=(value)
+      expect_arg_or_nil :value, Woodhouse::Layout, value
+      @previous_layout = @layout
+      @layout = value ? value.frozen_clone : nil
+    end
+
+    def node=(value)
+      @node = value || :default
+    end
+
+    def start
+      # TODO: don't pass global config
+      @scheduler ||= Woodhouse::Scheduler.new_link(configuration)
+      return false unless ready_to_start?
+      configuration.triggers.trigger :server_start
+      dispatch_layout_changes
+      true
+    end
+
+    def reload
+      dispatch_layout_changes!
+    end
+    
+    def ready_to_start?
+      @node and @layout and @layout.node(@node)
+    end
+
+    # TODO: do this better
+    def shutdown
+      @scheduler.spin_down
+      @scheduler.terminate
+      configuration.triggers.trigger :server_end
+      signal :shutdown
+    end
+
+    private
+
+    def scheduler_died(actor, reason)
+      signal :shutdown
+    end
+
+    def dispatch_layout_changes
+      if @layout.nil?
+        shutdown
+      else
+        apply_layout_changes @layout.changes_from(@previous_layout, @node)
+      end
+    end
+
+    def apply_layout_changes(changes)
+      if @scheduler
+        changes.adds.each do |add|
+          @scheduler.start_worker(add)
+        end
+        changes.drops.each do |drop|
+          @scheduler.stop_worker(drop)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/woodhouse/trigger_set.rb

@@ -0,0 +1,19 @@
+class Woodhouse::TriggerSet
+
+  def initialize
+    @triggers = {}
+  end
+
+  def add(event_name, &blk)
+    @triggers[event_name.to_sym] ||= []
+    @triggers[event_name.to_sym] << blk
+  end
+
+
+  def trigger(event_name, *args)
+    (@triggers[event_name.to_sym] || []).each do |trigger|
+      trigger.call(*args)
+    end
+  end
+
+end

data/lib/woodhouse/version.rb

@@ -0,0 +1,3 @@
+module Woodhouse
+  VERSION = "0.1.1"
+end

data/lib/woodhouse/worker.rb

@@ -0,0 +1,86 @@
+#
+# Classes which include this module become automatically visible to
+# MixinRegistry (the default way of finding jobs in Woodhouse).
+# All public methods of the class are automatically made available as
+# jobs.
+#
+# Classes including Woodhouse::Worker also get access to the +logger+
+# method, which will be the same logger globally configured for the current
+# Layout.
+#
+# Classes including Woodhouse::Worker also have convenience shortcuts
+# for dispatching jobs. Any job defined on the class can be dispatched
+# asynchronously by calling ClassName.async_job_name(options).
+#
+# == Example
+#
+#   class PamPoovey
+#     include Woodhouse::Worker
+#
+#     # This is available as the job PamPoovey#do_hr
+#     def do_hr(options)
+#       logger.info "Out comes the dolphin puppet"
+#     end
+#
+#     private
+#     
+#     # This is not picked up as a job
+#     def fight_club
+#       # ...
+#     end
+#   end
+#
+#   # later ...
+#   
+#   Woodhouse::MixinRegistry.new[:PamPoovey] # => PamPoovey
+#   PamPoovey.async_do_hr(:employee => "Lana")
+#
+module Woodhouse::Worker
+  
+  def self.included(into)
+    into.extend ClassMethods
+    into.set_worker_name into.name unless into.name.nil?
+  end
+
+  # The current Woodhouse logger. Set by the runner. Don't expect it to be set
+  # if you create the object yourself. If you want to be able to run job methods
+  # directly, you should account for setting +logger+.
+  attr_accessor :logger
+
+  module ClassMethods
+    
+    def worker_name
+      @worker_name
+    end
+
+    # Sets the name for this worker class if not already set (i.e., if it's
+    # an anonymous class). The first time the name for the worker is set,
+    # it becomes registered with MixinRegistry. After that, attempting to
+    # change the worker class will raise ArgumentError.
+    def set_worker_name(name)
+      if @worker_name
+        raise ArgumentError, "cannot change worker name"
+      else
+        if name and !name.empty?
+          @worker_name = name.to_sym
+          Woodhouse::MixinRegistry.register self
+        end
+      end
+    end
+
+    # You can dispatch a job +baz+ on class +FooBar+ by calling FooBar.async_baz.
+    def method_missing(method, *args, &block)
+      if method.to_s =~ /^asynch?_(.*)/
+        if instance_methods(false).detect{|meth| meth.to_s == $1 }
+          Woodhouse.dispatch(@worker_name, $1, args.first)
+        else
+          super
+        end
+      else
+        super
+      end
+    end
+
+  end
+
+end

data/lib/woodhouse.rb

@@ -0,0 +1,99 @@
+module Woodhouse
+  WoodhouseError = Class.new(StandardError)
+  WorkerNotFoundError = Class.new(WoodhouseError)
+  ConnectionError = Class.new(WoodhouseError)
+  ConfigurationError = Class.new(WoodhouseError)
+  FatalError = Class.new(WoodhouseError)
+  BailOut = Class.new(FatalError)
+
+  module Util
+    
+    private
+
+    def expect_arg(name, klass, value)
+      unless value.kind_of?(klass)
+        raise ArgumentError, "expected #{name} to be a #{klass.name}, got #{value.class}"
+      end
+    end
+
+    def expect_arg_or_nil(name, klass, value)
+      expect_arg(name, klass, value) unless value.nil?
+    end
+  
+    # Cheap knockoff, suffices for my simple purposes
+    def camelize(string)
+      string.split(/_/).map{ |word| word.capitalize }.join('')
+    end
+
+  end
+
+  # TODO: hate keeping global state in this class. I need to push
+  # some of this down into NodeConfiguration or something like it.
+  module GlobalMethods 
+
+    def logger
+      global_configuration.logger
+    end
+
+    def global_configuration
+      @global_configuration ||= Woodhouse::NodeConfiguration.default
+    end
+
+    def configure
+      @global_configuration ||= Woodhouse::NodeConfiguration.default
+      yield @global_configuration
+    end
+  
+    def global_layout
+      @global_layout ||= Woodhouse::Layout.default
+    end
+
+    def layout
+      @global_layout ||= Woodhouse::Layout.new
+      yield Woodhouse::LayoutBuilder.new(Woodhouse.global_configuration, @global_layout)
+    end
+
+    # Returns +true+ on JRuby, Rubinius, or MRI 1.9. +false+ otherwise.
+    def threading_safe?
+      RUBY_VERSION.to_f >= 1.9 or %w[jruby rbx].include?(RUBY_ENGINE)
+    end
+
+    def dispatch(*a)
+      global_configuration.dispatcher.dispatch(*a)
+    end
+
+    def update_job(*a)
+      global_configuration.dispatcher.update_job(*a)
+    end
+
+  end
+
+  extend GlobalMethods
+
+end
+
+require 'fiber18'
+require 'celluloid'
+require 'woodhouse/job'
+require 'woodhouse/layout'
+require 'woodhouse/layout_builder'
+require 'woodhouse/scheduler'
+require 'woodhouse/server'
+require 'woodhouse/queue_criteria'
+require 'woodhouse/node_configuration'
+require 'woodhouse/registry'
+require 'woodhouse/mixin_registry'
+require 'woodhouse/worker'
+require 'woodhouse/job_execution'
+require 'woodhouse/runners'
+require 'woodhouse/dispatchers'
+require 'woodhouse/middleware_stack'
+require 'woodhouse/middleware'
+require 'woodhouse/rails'
+require 'woodhouse/process'
+require 'woodhouse/layout_serializer'
+require 'woodhouse/trigger_set'
+
+require 'woodhouse/extension'
+require 'woodhouse/extensions/progress'
+require 'woodhouse/extensions/new_relic'

data/spec/integration/bunny_worker_process_spec.rb

@@ -0,0 +1,32 @@
+require 'woodhouse'
+require File.dirname(File.expand_path(__FILE__)) + '/../shared_contexts'
+
+describe Woodhouse::Runners::BunnyRunner do
+  it_should_behave_like "common"
+
+  let(:scheduler) {
+    common_config.runner_type = :bunny
+    Woodhouse::Scheduler.new(common_config)
+  }
+
+  let(:worker) {
+    Woodhouse::Layout::Worker.new(:FooBarWorker, :foo, :only => { :orz => "*happy campers*" })
+  }
+
+  it "should pull jobs off a queue" do
+    scheduler.start_worker worker
+    sleep 0.5
+    # TODO: this should use the bunny dispatcher, once I write it
+    bunny = Bunny.new
+    bunny.start
+    exchange = bunny.exchange(worker.exchange_name, :type => :headers)
+    exchange.publish("hi", :headers => { :orz => "*happy campers*" })
+    exchange.publish("hi", :headers => { :orz => "*silly cows*" })
+    bunny.stop
+    sleep 0.2
+    FakeWorker.jobs.should_not be_empty
+    FakeWorker.jobs.last[:orz].should == "*happy campers*"
+    scheduler.spin_down
+  end
+
+end

data/spec/layout_builder_spec.rb

@@ -0,0 +1,55 @@
+require 'woodhouse'
+require File.dirname(File.expand_path(__FILE__)) + '/shared_contexts'
+
+describe Woodhouse::LayoutBuilder do
+  it_should_behave_like "common"
+
+  it "should provide a DSL to set up layouts" do
+    registry = {
+      :Pam   => FakeWorker,
+      :Cyril => FakeWorker,
+      :Ray   => FakeWorker,
+      :Lana  => FakeWorker,
+    }
+    common_config.registry = registry
+    builder = Woodhouse::LayoutBuilder.new(common_config) do |layout|
+      layout.node(:default) do |default|
+        # Eight workers...
+        default.all_workers :threads => 2
+        # Six workers...
+        default.remove :Cyril
+        # Five workers...
+        default.remove :Ray, :foo
+        # Six workers.
+        default.add    :Ray, :bar, :only => { :baz => "bat" }
+      end
+      layout.node(:odin) do |odin|
+        # Two workers.
+        odin.add :Lana, :threads => 2
+        # Still two workers
+        odin.add :Lana, :threads => 5
+      end
+    end
+    layout = builder.layout
+    layout.nodes.should have(2).nodes
+    default = layout.node(:default)
+    default.workers.should have(6).workers
+    default.workers.first.threads.should == 2
+    default.workers.map(&:worker_class_name).should_not include(:Cyril)
+    default.workers.map(&:worker_class_name).should include(:Ray)
+    default.workers.select{|wk|
+      wk.worker_class_name == :Ray
+    }.map(&:job_method).should_not include(:foo)
+    ray = default.workers.detect{|wk|
+      wk.worker_class_name == :Ray && wk.criteria.criteria
+    }
+    ray.should_not be_nil
+    ray.criteria.matches?("baz" => "bat").should be_true
+    odin = layout.node(:odin)
+    odin.workers.should have(2).workers
+    odin.workers.first.threads.should == 5
+
+    Woodhouse::Layout.load(layout.dump).dump.should == layout.dump
+  end
+
+end