woodhouse 0.1.1

data/lib/woodhouse/extensions/progress.rb

@@ -0,0 +1,165 @@
+require 'woodhouse'
+require 'json'
+require 'digest/sha1'
+
+module Woodhouse::Progress
+
+  class << self
+
+    attr_accessor :client
+
+    def install_extension(configuration, opts = {}, &blk)
+      install!(configuration)
+    end
+
+    def install!(configuration = Woodhouse.global_configuration)
+      self.client = Woodhouse::Progress::BunnyProgressClient
+      configuration.runner_middleware << Woodhouse::Progress::InjectProgress
+    end
+
+    def pull(job_id)
+      client.new(Woodhouse.global_configuration).pull(job_id)
+    end
+
+    def pull_raw(job_id)
+      client.new(Woodhouse.global_configuration).pull_raw(job_id)
+    end
+
+  end
+
+  class ProgressClient
+    attr_accessor :config
+
+    def initialize(config)
+      self.config = config
+    end
+
+    def pull(job_id)
+      progress = pull_raw(job_id)
+      if progress
+        JSON.parse(progress)
+      end
+    end
+
+    def pull_raw(job_id)
+      pull_progress(job_id)
+    end
+
+    protected
+
+    def pull_progress(job_id)
+      raise NotImplementedError
+    end
+
+  end
+
+  class BunnyProgressClient < ProgressClient
+    
+    protected
+
+    def pull_progress(job_id)
+      bunny = Bunny.new(config.server_info)
+      
+      bunny.start
+      begin
+        channel = bunny.create_channel
+        exchange = channel.direct("woodhouse.progress")
+        queue = channel.queue(job_id, :durable => true)
+        queue.bind(exchange, :routing_key => job_id)
+        payload = nil
+        queue.message_count.times do
+          _, _, next_payload = queue.pop
+          payload = next_payload if next_payload
+        end
+        payload
+      ensure
+        bunny.stop
+      end
+    end
+
+  end
+
+
+  class StatusTicker
+    attr_accessor :top
+    attr_accessor :current
+    attr_accessor :status
+
+    def initialize(job, name, keyw = {})
+      self.job  = job
+      self.name = name
+      self.top  = keyw[:top]
+      self.current = keyw.fetch(:start, 0)
+      self.status = keyw[:status]
+    end
+
+    def to_hash
+      { name => count_attributes.merge( "status" => status ) }
+    end
+
+    def count_attributes
+      { "current" => current }.tap do |h|
+        h["top"] = top if top
+      end
+    end
+
+    def tick(keyw = {})
+      status = keyw[:status]
+      count  = keyw[:count]
+      by     = keyw[:by] || 1
+      new_top = keyw[:top]
+
+      if status
+        self.status = status
+      end
+      
+      if current
+        next_tick = count || current + by
+
+        self.current = next_tick
+      end
+
+      self.top = new_top if new_top
+       
+      job.update_progress(to_hash)
+    end
+
+    alias call tick
+
+    protected
+
+    attr_accessor :job, :name
+
+  end
+  
+  module JobWithProgress
+    
+    attr_accessor :progress_sink
+
+    def status_ticker(name, keyw = {})
+      StatusTicker.new(self, name, keyw)
+    end
+
+    def update_progress(data)
+      job = self
+      Celluloid::Future.new { progress_sink.update_job(job, data) }
+    end
+
+    def progress_sink
+      @progress_sink ||= Woodhouse
+    end
+
+  end
+
+  class InjectProgress < Woodhouse::Middleware
+    
+    def call(job, worker)
+      job.extend JobWithProgress
+      yield job, worker
+    end
+
+  end
+
+end
+
+Woodhouse::Extension.register :progress, Woodhouse::Progress

data/lib/woodhouse/job.rb

@@ -0,0 +1,76 @@
+require 'securerandom'
+require 'forwardable'
+
+class Woodhouse::Job
+  attr_accessor :worker_class_name, :job_method, :arguments, :payload
+  extend Forwardable
+
+  def_delegators :arguments, :each
+
+  def initialize(class_name = nil, method = nil, args = nil)
+    self.worker_class_name = class_name
+    self.job_method = method
+    self.arguments = args
+    unless arguments["_id"]
+      arguments["_id"] = generate_id
+    end
+    if arguments["payload"]
+      self.payload = arguments.delete("payload")
+    end
+    yield self if block_given?
+  end
+
+  def job_id
+    arguments["_id"]
+  end
+
+  def to_hash
+    {
+      "worker_class_name" => worker_class_name,
+      "job_method"        => job_method,
+    }.merge(arguments)
+  end
+
+  def job_method=(value)
+    @job_method = value ? value.to_sym : nil
+  end
+
+  def arguments=(h)
+    @arguments = (h || {}).inject({}){|args,(k,v)|
+      args[k.to_s] = v.to_s
+      args
+    }
+  end
+
+  def [](key)
+    arguments[key.to_s]
+  end
+
+  def maybe(meth, *args, &blk)
+    if respond_to?(meth)
+      send(meth, *args, &blk)
+    end
+  end
+
+  # TODO: copypasted from Woodhouse::Layout::Worker. Fix that
+  def exchange_name
+    "#{worker_class_name}_#{job_method}".downcase
+  end
+
+  def queue_name
+    exchange_name
+  end
+
+  def describe
+    "#{worker_class_name}##{job_method}(#{arguments.inspect})"
+  end
+  
+  def generate_id
+    SecureRandom.hex(16)
+  end
+
+  def payload
+    @payload || " "
+  end
+
+end

data/lib/woodhouse/job_execution.rb

@@ -0,0 +1,60 @@
+class Woodhouse::JobExecution
+
+  class << self
+    attr_accessor :fatal_error_proc
+  end
+
+  memory_error_rx = /((OutOf|NoMemory)Error|Java heap space)/
+  self.fatal_error_proc = lambda do |err|
+    err.class.name =~ memory_error_rx or err.message =~ memory_error_rx
+  end
+  
+  def initialize(config, job)
+    @config = config
+    @job = job
+  end
+
+  # Looks up the correct worker class for a job and executes it, running it
+  # through the runner middleware stack first. Returns true if the job finishes
+  # without an exception, false otherwise.
+  #
+  # If you need to keep track of exceptions raised by jobs, add middleware to
+  # handle them, like Woodhouse::Middleware::AirbrakeExceptions.
+  def execute
+    worker = @config.registry[@job.worker_class_name]
+    unless worker
+      raise Woodhouse::WorkerNotFoundError, "couldn't find job class #{@job.worker_class_name}"
+    end
+    work_object = worker.new
+    begin
+      @config.runner_middleware.call(@job, work_object) {|job, work_object|
+        work_object.send(job.job_method, job)
+      }
+      return true
+    rescue Woodhouse::FatalError
+      raise
+    rescue => err
+      if fatal_error?(err)
+        raise err
+      else
+        # Ignore the exception
+        return false
+      end
+    end
+  end
+
+  private
+
+  # TODO: lots of similar methods scattered around. Should refactor.
+  def symbolize_keys(hash)
+    hash.inject({}) {|h,(k,v)|
+      h[k.to_sym] = v
+      h
+    }
+  end
+
+  def fatal_error?(err)
+    self.class.fatal_error_proc.call(err)
+  end
+
+end

data/lib/woodhouse/layout.rb

@@ -0,0 +1,290 @@
+module Woodhouse
+
+  # 
+  # A Layout describes the configuration of a set of Woodhouse Server instances.
+  # Each Server runs all of the workers assigned to a single Node.
+  #
+  # Layouts and their contents (Node and Worker instances) are all plain data,
+  # suitable to being serialized, saved out, passed around, etc.
+  #
+  # Woodhouse clients do not need to know anything about the Layout to dispatch
+  # jobs, but servers rely on the Layout to know which jobs to serve. The basic
+  # process of setting up a Woodhouse server is to create a layout with one or
+  # more nodes and then pass it to Woodhouse::Server to serve.
+  # 
+  # There is a default layout suitable for many applications, available as
+  # Woodhouse::Layout.default. It has a single node named :default, which has
+  # the default node configuration -- one worker for every job. If you do not
+  # need to distribute different sets of jobs to different workers, the default
+  # layout should serve you.
+  # 
+  # TODO: A nicer DSL for creating and tweaking Layouts.
+  #
+  class Layout
+    include Woodhouse::Util
+
+    def initialize
+      @nodes = []
+    end
+
+    # Returns a frozen list of the nodes assigned to this layout.
+    def nodes
+      @nodes.frozen? ? @nodes : @nodes.dup.freeze
+    end
+
+    # Adds a Node to this layout. If +node+ is a Symbol, a Node will be
+    # automatically created with that name.
+    #
+    #   # Example:
+    #
+    #   layout.add_node Woodhouse::Layout::Node.new(:isis)
+    #
+    #   # Is equivalent to
+    #
+    #   layout.add_node :isis
+    #
+    def add_node(node)
+      if node.respond_to?(:to_sym)
+        node = Woodhouse::Layout::Node.new(node.to_sym)  
+      end
+      expect_arg :node, Woodhouse::Layout::Node, node
+      @nodes << node
+      node
+    end
+
+    # Looks up a Node by name and returns it.
+    def node(name)
+      name = name.to_sym
+      @nodes.detect{|node|
+        node.name == name
+      }
+    end
+
+    # Returns a frozen copy of this Layout and all of its child Node and
+    # Worker objects. Woodhouse::Server always takes a frozen copy of the
+    # layout it is given. It is thus safe to modify the same layout
+    # subsequently, and the changes only take effect when the layout is
+    # passed to the server again and Woodhouse::Server#reload is called.
+    def frozen_clone
+      clone.tap do |cloned|
+        cloned.nodes = @nodes.map{|node| node.frozen_clone }.freeze
+        cloned.freeze
+      end
+    end
+
+    # Returns a set of Changes necessary to move from +other_layout+ to this
+    # layout. This is used to permit live reconfiguration of servers by only
+    # spinning up and down nodes/workers which have changed.
+    def changes_from(other_layout, node)
+      Woodhouse::Layout::Changes.new(self, other_layout, node)
+    end
+
+    def dump(serializer = Woodhouse::LayoutSerializer)
+      serializer.dump(self)
+    end
+
+    def self.load(dumped, serializer = Woodhouse::LayoutSerializer)
+      serializer.load(dumped)
+    end
+
+    # The default layout, for convenience purposes. Has one node +:default+,
+    # which has the default configuration (see Woodhouse::Layout::Node#default_configuration!)
+    def self.default
+      new.tap do |layout|
+        layout.add_node :default
+        layout.node(:default).default_configuration!(Woodhouse.global_configuration)
+      end
+    end
+
+    protected
+
+    attr_writer :nodes
+    
+    # 
+    # A Node describes the set of workers present on a single Server.
+    #
+    # More information about Woodhouse's layout system can be found in the
+    # documentation for Woodhouse::Layout.
+    #
+    class Node
+      include Woodhouse::Util
+
+      attr_reader :name
+
+      def initialize(name)
+        @name = name.to_sym
+        @workers = []
+      end
+
+      # Returns a frozen list of workers assigned to this node.
+      def workers
+        @workers.frozen? ? @workers : @workers.dup.freeze
+      end
+      
+      # Adds a Worker to this node.
+      def add_worker(worker)
+        expect_arg :worker, Woodhouse::Layout::Worker, worker
+        @workers << worker
+      end
+
+      def remove_worker(worker)
+        expect_arg :worker, Woodhouse::Layout::Worker, worker
+        @workers.delete(worker)
+      end
+
+      def worker_for_job(job)
+        @workers.detect {|worker|
+          worker.accepts_job?(job)
+        }
+      end
+
+      def clear
+        @workers.clear
+      end
+
+      # Configures this node with one worker per job (jobs obtained 
+      # from Registry#each). The +default_threads+ value of the given
+      # +config+ is used to determine how many threads should be
+      # assigned to each worker.
+      def default_configuration!(config, options = {})
+        options[:threads] ||= config.default_threads
+        config.registry.each do |name, klass|
+          klass.public_instance_methods(false).each do |method|
+            add_worker Woodhouse::Layout::Worker.new(name, method, options)
+          end
+        end
+      end
+      
+      # Used by Layout#frozen_clone
+      def frozen_clone # :nodoc:
+        clone.tap do |cloned|
+          cloned.workers = @workers.map{|worker| worker.frozen_clone }.freeze
+          cloned.freeze
+        end
+      end
+
+      protected
+
+      attr_writer :workers
+    end
+
+    # 
+    # A Worker describes a single job that is performed on a Server.
+    # One or more Runner actors are created for every Worker in a Node.
+    #
+    # Any Worker has three parameters used to route jobs to it:
+    #
+    # +worker_class_name+:: 
+    #   This is generally a class name. It's looked up
+    #   in a Registry and used to instantiate a job object.
+    # +job_method+:: 
+    #   This is a method on the object called up with +worker_class_name+.
+    # +criteria+:: 
+    #   A hash of values (actually, a QueueCriteria object) used
+    #   to filter only specific jobs to this worker. When a job is dispatched,
+    #   its +arguments+ are compared with a worker's +criteria+. This is
+    #   done via an AMQP headers exchange (TODO: need to have a central document
+    #   to reference on how Woodhouse uses AMQP and jobs are dispatched)
+    #
+    class Worker
+      attr_reader :worker_class_name, :job_method, :threads, :criteria
+
+      def initialize(worker_class_name, job_method, opts = {})
+        opts = opts.clone
+        self.worker_class_name = worker_class_name
+        self.job_method = job_method
+        self.threads = opts.delete(:threads) || 1
+        self.criteria = opts.delete(:only)
+        unless opts.keys.empty? 
+          raise ArgumentError, "unknown option keys: #{opts.keys.inspect}"
+        end
+      end
+      
+      def exchange_name
+        "#{worker_class_name}_#{job_method}".downcase
+      end
+      
+      def queue_name
+        exchange_name + criteria.queue_key
+      end
+
+      def worker_class_name=(value)
+        @worker_class_name = value.to_sym
+      end
+
+      def job_method=(value)
+        @job_method = value.to_sym
+      end
+
+      def threads=(value)
+        @threads = value.to_i
+      end
+
+      def criteria=(value)
+        @criteria = Woodhouse::QueueCriteria.new(value).freeze
+      end
+
+      def frozen_clone
+        clone.freeze
+      end
+
+      def describe
+        "#@worker_class_name##@job_method(#{@criteria.describe})"
+      end
+
+      def accepts_job?(job)
+        criteria.matches?(job.arguments)
+      end
+
+      # TODO: want to recognize increases and decreases in numbers of
+      # threads and make minimal changes
+      def ==(other)
+        [worker_class_name, job_method, 
+          threads, criteria] ==
+        [other.worker_class_name, other.job_method,
+          other.threads, other.criteria]
+      end
+
+    end
+    
+    # 
+    # A diff between two Layouts, used to determine what workers need to be
+    # spun up and down when a layout change is sent to a Server.
+    #
+    class Changes
+
+      def initialize(new_layout, old_layout, node_name)
+        @new_layout = new_layout
+        @new_node = @new_layout && @new_layout.node(node_name)
+        @old_layout = old_layout
+        @old_node = @old_layout && @old_layout.node(node_name)
+        @node_name = node_name
+      end
+
+      def adds
+        new_workers.reject{|worker|
+          old_workers.member? worker
+        }
+      end
+
+      def drops
+        old_workers.reject{|worker|
+          new_workers.member? worker
+        }
+      end
+      
+      private
+
+      def old_workers
+        @old_workers ||= @old_node ? @old_node.workers : []
+      end
+
+      def new_workers
+        @new_workers ||= @new_node ? @new_node.workers : []
+      end
+
+    end
+
+  end
+
+end

data/lib/woodhouse/layout_builder.rb

@@ -0,0 +1,55 @@
+class Woodhouse::LayoutBuilder
+
+  attr_reader :layout
+
+  class NodeBuilder
+    
+    def initialize(config, node)
+      @config = config
+      @node = node
+    end
+
+    def all_workers(options = {})
+      @node.default_configuration! @config, options
+    end
+
+    def add(class_name, job_method, opts = {})
+      if job_method.kind_of?(Hash)
+        # Two-argument invocation
+        opts = job_method
+        job_method = nil
+        methods = @config.registry[class_name].public_instance_methods(false)
+      else
+        methods = [job_method]
+      end
+      remove(class_name, job_method, opts.empty? ? nil : opts)
+      methods.each do |method|
+        @node.add_worker Woodhouse::Layout::Worker.new(class_name, method, opts)
+      end
+    end
+
+    def remove(class_name, job_method = nil, opts = nil)
+      @node.workers.select{|worker|
+        worker.worker_class_name == class_name &&
+          (job_method.nil? || worker.job_method == job_method) &&
+          (opts.nil? || worker.criteria.criteria == opts[:only])
+      }.each do |worker|
+        @node.remove_worker(worker)
+      end
+    end
+
+  end
+
+  def initialize(config, layout = nil)
+    @config = config
+    @layout = layout || Woodhouse::Layout.new
+    @nodes ||= {}
+    yield self if block_given?
+  end
+
+  def node(name)
+    @layout.node(name) || @layout.add_node(name)
+    yield(@nodes[name] ||= NodeBuilder.new(@config, @layout.node(name)))
+  end
+
+end