woodhouse 0.1.1

data/lib/woodhouse/layout_serializer.rb

@@ -0,0 +1,82 @@
+require 'json'
+
+class Woodhouse::LayoutSerializer
+
+  def initialize(layout)
+    @layout = layout
+  end
+
+  def as_hash
+    { 
+      :nodes => node_list(layout.nodes)
+    }
+  end
+
+  def to_json
+    as_hash.to_json
+  end
+
+  def self.dump(layout)
+    new(layout).to_json
+  end
+
+  def self.load(json)
+    LayoutLoader.new(json).layout
+  end
+
+  class LayoutLoader
+
+    def initialize(json)
+      @entries = JSON.parse(json)
+    end
+
+    def layout
+      Woodhouse::Layout.new.tap do |layout|
+        @entries['nodes'].each do |node|
+          new_node = layout.add_node(node['name'])
+          node['workers'].each do |worker|
+            new_node.add_worker Woodhouse::Layout::Worker.new(worker['worker_class_name'], worker['job_method'], :threads => worker['threads'], :only => worker['criteria'])
+          end
+        end
+      end
+    end
+
+  end
+
+  private
+
+  attr_reader :layout
+
+  def node_list(nodes)
+    nodes.map{|node|
+      node_hash(node)
+    }
+  end
+
+  def node_hash(node)
+    {
+      :name => node.name,
+      :workers => worker_list(node.workers)
+    }
+  end
+
+  def worker_list(workers)
+    workers.map{|worker|
+      worker_hash(worker)
+    }
+  end
+
+  def worker_hash(worker)
+    {
+      :worker_class_name => worker.worker_class_name,
+      :job_method        => worker.job_method,
+      :threads           => worker.threads,
+      :criteria          => criteria_hash(worker.criteria)
+    }
+  end
+
+  def criteria_hash(criteria)
+    criteria.criteria
+  end
+
+end

data/lib/woodhouse/middleware/airbrake_exceptions.rb

@@ -0,0 +1,12 @@
+class Woodhouse::Middleware::AirbrakeExceptions < Woodhouse::Middleware
+
+  def call(job, worker)
+    begin
+      yield job, worker
+    rescue => err
+      Airbrake.notify(err)
+      raise err
+    end
+  end
+
+end

data/lib/woodhouse/middleware/assign_logger.rb

@@ -0,0 +1,10 @@
+class Woodhouse::Middleware::AssignLogger < Woodhouse::Middleware
+
+  def call(job, worker)
+    if @config.logger and worker.respond_to?(:logger)
+      worker.logger = @config.logger
+    end
+    yield job, worker
+  end
+
+end

data/lib/woodhouse/middleware/log_dispatch.rb

@@ -0,0 +1,21 @@
+class Woodhouse::Middleware::LogDispatch < Woodhouse::Middleware
+  
+  def call(job)
+    begin
+      yield job
+    rescue => err
+      log "#{job.describe} could not be dispatched: #{err.inspect}"
+      raise err
+    end
+    log "#{job.describe} dispatched"
+  end
+
+  private
+
+  def log(msg)
+    if @config.logger
+      @config.logger.info msg
+    end
+  end
+
+end

data/lib/woodhouse/middleware/log_jobs.rb

@@ -0,0 +1,22 @@
+class Woodhouse::Middleware::LogJobs < Woodhouse::Middleware
+
+  def call(job, worker)
+    log "#{job.describe} starting"
+    begin
+      yield job, worker
+    rescue => err
+      log "#{job.describe} failed: #{err.inspect}"
+      raise err
+    end
+    log "#{job.describe} done"
+  end
+
+  private
+
+  def log(msg)
+    if @config.logger
+      @config.logger.info msg
+    end
+  end
+
+end

data/lib/woodhouse/middleware.rb

@@ -0,0 +1,16 @@
+class Woodhouse::Middleware
+
+  def initialize(config)
+    @config = config
+  end
+
+  def call(*args)
+    yield *args
+  end
+
+end
+
+require 'woodhouse/middleware/log_jobs'
+require 'woodhouse/middleware/log_dispatch'
+require 'woodhouse/middleware/assign_logger'
+require 'woodhouse/middleware/airbrake_exceptions'

data/lib/woodhouse/middleware_stack.rb

@@ -0,0 +1,35 @@
+class Woodhouse::MiddlewareStack < Array
+
+  def initialize(config)
+    @config = config
+  end
+
+  def call(*args, &final)
+    stack = make_stack.dup
+    next_step = lambda {|*args|
+      next_item = stack.shift
+      if next_item.nil?
+        final.call(*args)
+      else
+        next_item.call(*args, &next_step)
+      end
+    }
+    next_step.call(*args)
+  end
+
+  private
+
+  def make_stack
+    @stack ||= 
+      map do |item|
+        if item.respond_to?(:call)
+          item
+        elsif item.respond_to?(:new)
+          item.new(@config)
+        else
+          raise ArgumentError, "bad entry #{item.inspect} in middleware stack"
+        end
+      end
+  end
+
+end

data/lib/woodhouse/mixin_registry.rb

@@ -0,0 +1,27 @@
+class Woodhouse::MixinRegistry < Woodhouse::Registry 
+
+  class << self
+    
+    def classes
+      @classes ||= {}
+    end
+
+    def register(klass)
+      register_worker klass.name, klass
+    end
+
+    def register_worker(class_name, klass)
+      classes[class_name.to_s] = klass
+    end
+
+  end
+  
+  def [](worker)
+    Woodhouse::MixinRegistry.classes[worker.to_s]
+  end
+
+  def each(&blk)
+    Woodhouse::MixinRegistry.classes.each &blk
+  end
+
+end

data/lib/woodhouse/node_configuration.rb

@@ -0,0 +1,80 @@
+class Woodhouse::NodeConfiguration
+  include Woodhouse::Util
+
+  attr_accessor :registry, :server_info, :runner_type, :dispatcher_type, :logger, :default_threads
+  attr_accessor :dispatcher_middleware, :runner_middleware
+  attr_accessor :triggers
+
+  def initialize
+    self.default_threads = 1
+    self.dispatcher_middleware = Woodhouse::MiddlewareStack.new(self)
+    self.runner_middleware = Woodhouse::MiddlewareStack.new(self)
+    self.server_info = {}
+    self.triggers = Woodhouse::TriggerSet.new
+    yield self if block_given?
+  end
+
+  def at(event_name, &blk)
+    triggers.add(event_name, &blk)
+  end
+
+  def dispatcher
+    @dispatcher ||= dispatcher_type.new(self)
+  end
+
+  def dispatcher_type=(value)
+    if value.respond_to?(:to_sym)
+      value = lookup_key(value, :Dispatcher)
+    end
+    @dispatcher = nil
+    @dispatcher_type = value
+  end
+
+  def runner_type=(value)
+    if value.respond_to?(:to_sym)
+      value = lookup_key(value, :Runner)
+    end
+    @dispatcher = nil
+    @runner_type = value
+  end
+
+  def server_info=(hash)
+    @server_info = hash ? symbolize_keys(hash) : {}
+  end
+
+  def extension(name, opts = {}, &blk)
+    Woodhouse::Extension.install_extension(name, self, opts, &blk)
+  end
+
+  private
+
+  def lookup_key(key, namespace)
+    const = Woodhouse.const_get("#{namespace}s").const_get("#{camelize(key.to_s)}#{namespace}")
+    unless const
+      raise NameError, "couldn't find Woodhouse::#{namespace}s::#{camelize(key.to_s)}#{namespace} (from #{key})"
+    end
+    const
+  end
+
+  def symbolize_keys(hash)
+    hash.inject({}){|h,(k,v)|
+      h[k.to_sym] = v
+      h
+    }
+  end
+
+  # TODO: detect defaults based on platform
+  def self.default
+    new do |config|
+      config.registry         = Woodhouse::MixinRegistry.new
+      config.server_info      = nil
+      config.runner_type      = Woodhouse::Runners.guess
+      config.dispatcher_type  = :local
+      config.logger           = Logger.new("/dev/null") 
+      config.dispatcher_middleware << Woodhouse::Middleware::LogDispatch
+      config.runner_middleware     << Woodhouse::Middleware::LogJobs
+      config.runner_middleware     << Woodhouse::Middleware::AssignLogger
+    end
+  end
+
+end

data/lib/woodhouse/process.rb

@@ -0,0 +1,41 @@
+# TODO: take arguments. Also consider using thor.
+class Woodhouse::Process
+
+  def initialize(keyw = {})
+    @server = keyw[:server] || build_default_server(keyw)
+  end
+  
+  def execute
+    # Borrowed this from sidekiq. https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
+    trap "INT" do
+      Thread.main.raise Interrupt
+    end
+
+    trap "TERM" do
+      Thread.main.raise Interrupt
+    end
+
+    begin
+      @server.start!
+      puts "Woodhouse serving as of #{Time.now}. Ctrl-C to stop."
+      @server.wait(:shutdown) 
+    rescue Interrupt
+      puts "Shutting down."
+      @server.shutdown!
+      @server.wait(:shutdown)
+    ensure
+      @server.terminate
+      exit
+    end
+  end
+
+  private
+
+  def build_default_server(keyw)
+    Woodhouse::Server.new(
+      :layout => keyw[:layout] || Woodhouse.global_layout,
+      :node   => keyw[:node]   || :default
+    )
+  end
+
+end

data/lib/woodhouse/queue_criteria.rb

@@ -0,0 +1,52 @@
+module Woodhouse
+
+  class QueueCriteria
+    attr_reader :criteria
+
+    def initialize(opts = {})
+      if opts.kind_of?(self.class)
+        opts = opts.criteria
+      end
+      unless opts.nil?
+        @criteria = stringify_values(opts).freeze
+      end
+    end
+
+    def ==(other)
+      @criteria == other.criteria
+    end
+
+    def describe
+      @criteria.inspect
+    end
+
+    def amqp_headers
+      # TODO: needs to be smarter
+      @criteria ? @criteria.merge('x-match' => 'all') : {}
+    end
+
+    def queue_key
+      @criteria ? @criteria.map{|k,v|
+        "#{k.downcase}_#{v.downcase}"
+      }.join("_") : ""
+    end
+
+    def matches?(args)
+      return true if @criteria.nil?
+      @criteria.all? do |key, val|
+        args[key] == val
+      end
+    end
+
+    private
+
+    def stringify_values(hash)
+      hash.inject({}) {|h,(k,v)|
+        h[k.to_s] = v.to_s
+        h
+      }
+    end
+
+  end
+
+end

data/lib/woodhouse/rails.rb

@@ -0,0 +1,46 @@
+if defined?(Rails::Railtie)
+  module Woodhouse::RailsExtensions
+    def layout(&blk)
+      unless @delay_finished
+        @delayed_layout = blk
+      else
+        super
+      end
+    end
+
+    def finish_loading_layout!
+      @delay_finished = true
+      if @delayed_layout
+        layout &@delayed_layout
+      end
+    end
+  end
+
+  Woodhouse.extend Woodhouse::RailsExtensions
+ 
+  class Woodhouse::Rails < Rails::Engine
+#    config.autoload_paths << Rails.root.join("app/workers")
+
+    initializer 'woodhouse' do
+      config_paths = %w[woodhouse.yml workling.yml].map{|file|
+        Rails.root.join("config/" + file)
+      }
+      # Preload everything in app/workers so default layout includes them 
+      Rails.root.join("app/workers").tap do |workers|
+        Pathname.glob(workers.join("**/*.rb")).each do |worker_path|
+          worker_path.relative_path_from(workers).basename(".rb").to_s.camelize.constantize
+        end
+      end
+      # Set up reasonable defaults
+      Woodhouse.configure do |config|
+        config.logger = ::Rails.logger
+        config_paths.each do |path|
+          if File.exist?(path)
+            config.server_info = YAML.load(File.read(path))[::Rails.env]
+          end
+        end
+      end
+      Woodhouse.finish_loading_layout!
+    end
+  end
+end

data/lib/woodhouse/rails2.rb

@@ -0,0 +1,21 @@
+require 'woodhouse'
+
+ActiveSupport::Dependencies.autoload_paths << RAILS_ROOT + "/app/workers"
+
+Woodhouse.configure do |config|
+  config_paths = %w[woodhouse.yml workling.yml].map{|file|
+    RAILS_ROOT + "/config/" + file
+  }
+  config.logger = ::Rails.logger
+  if ::Rails.env =~ /development|test/
+    config.dispatcher_type = :local
+  else
+    config.dispatcher_type = :bunny
+  end
+  config_paths.each do |path|
+    if File.exist?(path)
+      config.server_info = YAML.load(File.read(path))[::Rails.env]
+    end
+  end
+  config.runner_type = Woodhouse::Runners.guess
+end

data/lib/woodhouse/registry.rb

@@ -0,0 +1,12 @@
+class Woodhouse::Registry
+  include Woodhouse::Util
+
+  def [](worker)
+    raise NotImplementedError, "subclass Woodhouse::Registry and override #[]"
+  end
+
+  def each
+    raise NotImplementedError, "subclass Woodhouse::Registry and override #each"
+  end
+
+end

data/lib/woodhouse/runner.rb

@@ -0,0 +1,60 @@
+#
+# The abstract base class for actors in charge of finding and running jobs
+# of a given type. Runners will be allocated for each Woodhouse::Layout::Worker
+# in a layout. Woodhouse::Layout::Worker#threads indicates how many Runners should
+# be spawned for each job type.
+#
+# The lifecycle of a Runner is to be created by Woodhouse::Scheduler::WorkerSet,
+# and to automatically begin subscribing as soon as it is initialized. At some
+# point, the actor will receive the +spin_down+ message, at which case it must
+# cease all work and return from +subscribe+.
+#
+# Whenever a Runner receives a job on its queue, it should convert it into a 
+# Workling::Job and pass it to +service_job+ after confirming with
+# +can_service_job?+ that this is an appropriate job for this queue.
+#
+# Runners should always subscribe to queues in ack mode. Messages should be
+# acked after they finish, and rejected if the job is inappropriate for this
+# worker or if it raises an exception.
+#
+# TODO: document in more detail the contract between Runner and Dispatcher over
+# AMQP exchanges, and how Woodhouse uses AMQP to distribute jobs.
+#
+class Woodhouse::Runner
+  include Woodhouse::Util
+  include Celluloid
+
+  def initialize(worker, config)
+    @worker = worker
+    @config = config
+    @config.logger.debug "Thread for #{@worker.describe} ready and waiting for jobs"
+  end
+
+  # Implement this in a subclass. When this message is received by an actor, it should
+  # finish whatever job it is currently doing, gracefully disconnect from AMQP, and
+  # stop the subscribe loop.
+  def spin_down
+    raise NotImplementedError, "implement #spin_down in a subclass of Woodhouse::Runner"
+  end
+
+  private
+
+  # Implement this in a subclass. When this message is received by an actor, it should
+  # connect to AMQP and start pulling jobs off the queue. This method should not finish
+  # until spin_down is called.
+  def subscribe # :doc:
+    raise NotImplementedError, "implement #subscribe in a subclass of Woodhouse::Runner"
+  end
+  
+  # Returns +true+ if the Job's arguments match this worker's QueueCriteria, else +false+.
+  def can_service_job?(job) # :doc:
+    @worker.accepts_job?(job)
+  end
+
+  # Executes a Job. See Woodhouse::JobExecution.
+  def service_job(job) # :doc:
+    @config.logger.debug "Servicing job for #{@worker.describe}"
+    Woodhouse::JobExecution.new(@config, job).execute
+  end
+
+end

data/lib/woodhouse/runners/bunny_runner.rb

@@ -0,0 +1,60 @@
+require 'bunny'
+
+class Woodhouse::Runners::BunnyRunner < Woodhouse::Runner
+  include Celluloid
+
+  def subscribe
+    bunny = Bunny.new(@config.server_info)
+    bunny.start
+    channel = bunny.create_channel
+    channel.prefetch(1)
+    queue = channel.queue(@worker.queue_name)
+    exchange = channel.exchange(@worker.exchange_name, :type => :headers)
+    queue.bind(exchange, :arguments => @worker.criteria.amqp_headers)
+    worker = Celluloid.current_actor
+    queue.subscribe(:ack => true, :block => false) do |delivery, props, payload|
+      begin
+        job = make_job(props, payload)
+        if can_service_job?(job)
+          if service_job(job)
+            channel.acknowledge(delivery.delivery_tag, false)
+          else
+            channel.reject(delivery.delivery_tag, false)
+          end
+        else
+          @config.logger.error("Cannot service job #{job.describe} in queue for #{@worker.describe}")
+          channel.reject(delivery.delivery_tag, false) 
+        end
+      rescue => err
+        begin
+          @config.logger.error("Error bubbled up out of worker. This shouldn't happen. #{err.message}")
+          err.backtrace.each do |btr|
+            @config.logger.error("  #{btr}")
+          end
+          # Don't risk grabbing this job again. 
+          channel.reject(delivery.delivery_tag, false)
+        ensure
+          worker.bail_out(err)
+        end
+      end
+    end
+    wait :spin_down
+  end
+
+  def bail_out(err)
+    raise Woodhouse::BailOut, "#{err.class}: #{err.message}"
+  end
+
+  def spin_down
+    signal :spin_down
+  end
+
+  def make_job(properties, payload)
+    Woodhouse::Job.new(@worker.worker_class_name, @worker.job_method) do |job|
+      args = properties.headers
+      job.arguments = args
+      job.payload = payload
+    end
+  end
+
+end

data/lib/woodhouse/runners/dummy_runner.rb

@@ -0,0 +1,11 @@
+class Woodhouse::Runners::DummyRunner < Woodhouse::Runner
+
+  def subscribe
+    wait :spin_down
+  end
+
+  def spin_down
+    signal :spin_down
+  end
+
+end