job_dispatch 0.0.1

data/lib/job_dispatch/identity.rb

@@ -0,0 +1,54 @@
+require 'active_support/core_ext'
+require 'active_support/core_ext/object/json'
+
+module JobDispatch
+  # Identity encapsulates a ZeroMQ socket identity, which is a string of binary characters, typically
+  # containing nulls or non-utf8 compatible characters in ASCII-8BIT encoding.
+  class Identity
+
+    include Comparable
+
+    attr_reader :identity
+
+    def initialize(identity)
+      @identity = identity.to_sym
+    end
+
+    def to_s
+      @identity.to_s
+    end
+
+    def to_str
+      @identity.to_str
+    end
+
+    def to_hex
+      @identity.to_s.bytes.map { |x| '%02x' % x }.join
+    end
+
+    def as_json(options={})
+      to_hex.as_json(options)
+    end
+
+    def to_sym
+      @identity
+    end
+
+    def hash
+      @identity.hash
+    end
+
+    def ==(other)
+      @identity == other.identity
+    end
+
+    def eql?(other)
+      self.class == other.class && @identity == other.identity
+    end
+
+    def <=>(other)
+      @identity <=> other.identity
+    end
+
+  end
+end

data/lib/job_dispatch/job.rb

@@ -0,0 +1,44 @@
+module JobDispatch
+  module Job
+
+    DEFAULT_EXECUTION_TIMEOUT = 30
+
+    PENDING = 0
+    IN_PROGRESS = 1
+    COMPLETED = 2
+    FAILED = 3
+
+    STATUS_STRINGS = {
+        PENDING => 'pending',
+        IN_PROGRESS => 'in progress',
+        COMPLETED => 'completed',
+        FAILED => 'failed'
+    }
+
+    def timed_out?
+      expire_execution_at < Time.now
+    end
+
+    def failed!(results)
+      # update database
+      self.completed_at = Time.now
+      self.result = results
+      if retry_count && retry_count > 0 && retry_delay && retry_delay > 0
+        self.retry_count -= 1
+        self.scheduled_at = Time.now + retry_delay.seconds
+        self.retry_delay *= 2
+        self.status = PENDING
+      else
+        self.status = FAILED
+      end
+      save!
+    end
+
+    def succeeded!(results)
+      self.status = COMPLETED
+      self.result = results
+      self.completed_at = Time.now
+      save!
+    end
+  end
+end

data/lib/job_dispatch/signaller.rb

@@ -0,0 +1,30 @@
+module JobDispatch
+
+  # This class represents a ZeroMQ socket for signalling to the broker that there are jobs immediately available.
+  class Signaller
+    attr :socket
+
+    def initialize(wakeup_connect_address)
+      @wakeup_connect_address = wakeup_connect_address
+    end
+
+    def connect
+      if @socket.nil?
+        @socket = JobDispatch.context.socket(ZMQ::PUB)
+        @socket.connect(@wakeup_connect_address)
+      end
+    end
+
+    def disconnect
+      if @socket
+        @socket.close
+        @socket = nil
+      end
+    end
+
+    # signals are a straight
+    def signal(queue='default')
+      @socket.send(queue)
+    end
+  end
+end

data/lib/job_dispatch/sockets/enqueue.rb

@@ -0,0 +1,18 @@
+module JobDispatch::Sockets
+  class Enqueue
+    def initialize(bind_address)
+      @socket = JobDispatch.context.socket(ZMQ::REQ)
+      @socket.bind(bind_address)
+    end
+
+    def poll_item
+      @poll_item ||= ZMQ::Pollitem(@socket, ZMQ::POLLIN)
+    end
+
+    # Enqueue socket when it receives a message simply stores it in the database.
+    # It will also send a message to wake a connected dispatcher
+    def process
+
+    end
+  end
+end

data/lib/job_dispatch/status.rb

@@ -0,0 +1,79 @@
+require 'text-table'
+
+module JobDispatch
+  class Status
+
+    attr :socket
+
+
+    def initialize(connect_address)
+      @connect_address = connect_address
+    end
+
+    def connect
+      if @socket.nil?
+        @socket = JobDispatch.context.socket(ZMQ::REQ)
+        @socket.connect(@connect_address)
+      end
+    end
+
+    def disconnect
+      @socket.close
+      @socket = nil
+    end
+
+    def fetch
+      @socket.send(JSON.dump({command:'status'}))
+      json = @socket.recv
+      @status = JSON.parse(json).with_indifferent_access
+      @time = Time.now
+    end
+
+    def print
+      puts "Job Dispatcher status: #{@status[:status]} at #{@time}"
+      puts ""
+
+      table = Text::Table.new
+      table.head = ['Queue', 'Worker ID', 'Worker Name', 'Status', 'Job ID', 'Job Details']
+      table.rows = []
+
+      @status[:queues].each do |queue, workers|
+        if workers.empty?
+          table.rows << [
+              queue,
+              '- no workers -',
+              '',
+              '',
+              '',
+              '',
+          ]
+        else
+          workers.each_pair do |worker_id, worker_status|
+
+            job = worker_status[:job]
+            if job
+              params_str = if job[:parameters]
+                             job[:parameters].map(&:inspect).join(',')[0..20]
+                           else
+                             ''
+                           end
+              job_details = "#{job[:target]}.#{job[:method]}(#{params_str})"
+            end
+
+            table.rows << [
+                queue,
+                worker_id,
+                worker_status[:name],
+                worker_status[:status],
+                worker_status[:job_id],
+                job_details,
+            ]
+          end
+        end
+      end
+
+      puts table.to_s
+    end
+
+  end
+end

data/lib/job_dispatch/version.rb

@@ -0,0 +1,3 @@
+module JobDispatch
+  VERSION = "0.0.1"
+end

data/lib/job_dispatch/worker/item.rb

@@ -0,0 +1,43 @@
+# encoding: UTF-8
+
+require 'active_support/dependencies'
+
+module JobDispatch
+  class Worker
+
+    #
+    # This represents a unit of work to be done. It will be serialised to Mongo database
+    #
+    class Item
+      attr_accessor :job_id
+      attr :target
+      attr :method
+      attr :params
+      attr :result
+      attr :status
+
+      def initialize(target, method, *params)
+        @target, @method, @params = target, method, params
+      end
+
+      # execute the method on the target with the given parameters
+      # This will capture standard exceptions for return over network.
+      def execute
+        begin
+          JobDispatch.logger.info "Worker executing job #{job_id}: #{target}.#{method}"
+          Thread.current["JobDispatch::Worker.job_id"] = job_id
+          @klass = target.constantize
+          @result = @klass.__send__(method.to_sym, *params)
+          @status = :success
+        rescue StandardError => ex
+          @result = ex
+          @status = :error
+        ensure
+          Thread.current["JobDispatch::Worker.job_id"] = nil
+          JobDispatch.logger.info "Worker completed job #{job_id}: #{target}.#{method}, status: #{@status}"
+        end
+      end
+    end
+  end
+
+end

data/lib/job_dispatch/worker/socket.rb

@@ -0,0 +1,96 @@
+# encoding: UTF-8
+
+require 'json'
+
+module JobDispatch
+
+  class Worker
+    class Socket
+
+      attr :socket
+      attr :item_class
+
+      def initialize(connect_address, item_klass)
+        @socket = JobDispatch.context.socket(ZMQ::REQ)
+        @socket.connect(connect_address)
+        @item_class = item_klass
+      end
+
+      def poll_item
+        @poll_item ||= ZMQ::Pollitem(@socket, ZMQ::POLLIN)
+      end
+
+      def ask_for_work(queue)
+        @socket.send(JSON.dump({command: 'ready', queue: queue, worker_name: identity}))
+      end
+
+      def send_goodbye(queue)
+        @socket.send(JSON.dump({command: 'goodbye', worker_name: identity}))
+      end
+
+      def close
+        @socket.close
+      end
+
+      def identity
+        @identity ||= begin
+          hostname = ::Socket.gethostname
+          process = Process.pid
+          thread = Thread.current.object_id.to_s(16)
+          ['ruby', hostname, process, thread].join(':')
+        end
+      end
+
+      # read an incoming message. The thread will block if there is no readable message.
+      #
+      # @return [JobDispatch::Item] the item to be processed (or nil if there isn't a valid job)
+      def read_item
+        json = @socket.recv
+        begin
+          params = JSON.parse(json)
+          case params["command"]
+            when "job"
+              item = item_class.new params["target"], params["method"], *params["parameters"]
+            when "idle"
+              item = item_class.new "JobDispatch", "idle"
+            when "quit"
+              puts "It's quittin' time!"
+              Process.exit(0)
+            else
+              item = item_class.new "JobDispatch", "unknown_command", params
+          end
+          item.job_id = params["job_id"]
+        rescue StandardError => e
+          JobDispatch.logger.error "Failed to read message from worker socket: #{e}"
+          nil
+        end
+        item
+      end
+
+      # after execution, send the response.
+      def send_response(job_id, status, result)
+        JobDispatch.logger.info "Worker #{Process.pid} completed job_id: #{job_id}: #{status}, result: #{result}"
+        response = {
+            command: 'completed',
+            ready: true,
+            job_id: job_id,
+            result: result,
+            status: status
+        }
+        @socket.send(JSON.dump(response))
+      end
+
+      def send_touch(job_id, timeout=nil)
+        hash = {
+            command: 'touch',
+            job_id: job_id
+        }
+        hash[:timeout] = timeout if timeout
+        @socket.send(JSON.dump(hash))
+        json = @socket.recv # wait for acknowledgement... this could be done via pub/sub to be asynchronous.
+        JSON.parse(json) rescue {:error => "Failed to decode JSON from dispatcher: #{json}"}
+      end
+
+    end
+  end
+end

data/lib/job_dispatch/worker.rb

@@ -0,0 +1,120 @@
+# encoding: UTF-8
+
+module JobDispatch
+
+  #
+  # This class is the main worker loop. Run it as a whole process or just as a thread in a multi-threaded worker
+  # process.
+  #
+  class Worker
+
+    IDLE_TIME = 3
+    IDLE_COUNT = 10
+
+    attr :socket
+    attr :queue
+    attr :item_class
+
+    def initialize(connect_address, options={})
+      options ||= {}
+      @connect_address = connect_address
+      @queue = options[:queue] || 'default'
+      @running = false
+      @item_class = options[:item_class] || Worker::Item
+    end
+
+    def connect
+      @socket ||= Worker::Socket.new(@connect_address, item_class)
+      Thread.current["JobDispatch::Worker.socket"] = @socket
+    end
+
+    def disconnect
+      if @socket
+        @socket.close
+        @socket = nil
+        Thread.current["JobDispatch::Worker.socket"] = nil
+      end
+    end
+
+    def run
+      @running = true
+      while running?
+        puts "connecting"
+        connect
+        puts "asking for work"
+        ask_for_work
+
+        # if we are idle for too many times, the broker has restarted or gone away, and we will be stuck in receive
+        # state, so we need to close the socket and make a new one to ask for work again.
+
+        idle_count = 0
+        poller = ZMQ::Poller.new
+        poller.register(socket.poll_item)
+        while running? and idle_count < IDLE_COUNT
+          begin
+            poller.poll(IDLE_TIME)
+            if poller.readables.include?(socket.socket)
+              process
+              idle_count = 0
+            else
+              idle
+              idle_count += 1
+            end
+          rescue Interrupt
+            puts "Worker stopping."
+            JobDispatch.logger.info("Worker #{}")
+            stop
+            disconnect
+            connect
+            send_goodbye
+          end
+        end
+        disconnect
+      end
+    end
+
+    def ask_for_work
+      socket.ask_for_work(queue)
+    end
+
+    def send_goodbye
+      socket.send_goodbye(queue)
+    end
+
+    def running?
+      @running
+    end
+
+    def stop
+      @running = false
+    end
+
+    def self.touch(timeout=nil)
+      sock = Thread.current["JobDispatch::Worker.socket"]
+      job_id = Thread.current["JobDispatch::Worker.job_id"]
+      if sock && job_id
+        sock.send_touch(job_id, timeout)
+      end
+    end
+
+    private
+
+    # called when the socket is readable. do some work.
+    def process
+      item = @socket.read_item
+      if item
+        item.execute
+        @socket.send_response(item.job_id, item.status, item.result)
+      else
+        @socket.send_response("unknown", :error, "failed to decode command")
+      end
+    end
+
+    def idle
+      puts "waiting for job to do…"
+    end
+  end
+end
+
+require 'job_dispatch/worker/socket'
+require 'job_dispatch/worker/item'

data/lib/job_dispatch.rb

@@ -0,0 +1,97 @@
+# encoding: UTF-8
+
+require "job_dispatch/version"
+
+require 'active_support/dependencies/autoload'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/module/attribute_accessors'
+require 'nullobject'
+require 'rbczmq'
+
+module JobDispatch
+
+  extend ActiveSupport::Autoload
+
+  autoload :Broker
+  autoload :Client
+  autoload :Configuration
+  autoload :Identity
+  autoload :Job
+  autoload :Signaller
+  autoload :Status
+  autoload :Worker
+
+  def configure(&block)
+    Configuration.configure(&block)
+  end
+
+  def config
+    Configuration.config
+  end
+
+  def load_config_from_yml(filename='config/job_dispatch.yml', environment="default")
+    require 'yaml'
+    _config = YAML.load_file(filename).with_indifferent_access
+    _config = _config[environment] || _config[:default]
+    load_config(_config)
+  end
+
+  def load_config(hash)
+    configure do |c|
+      hash.each_pair do |key, value|
+        c[key] = value
+      end
+    end
+  end
+
+  # @return [ZMQ::Context] return or create a ZeroMQ context.
+  def context
+    ZMQ.context || ZMQ::Context.new
+  end
+
+  def idle
+    "idle, doing nothing"
+  end
+
+  def unknown_command(params)
+    puts "Unknown command: #{params.inspect}"
+  end
+
+  # This signals to the job broker(s) that there are jobs immediately available on the given queue.
+  def signal(queue='default')
+    self.signaller ||= if config.signaller && config.signaller[:connect]
+                         signaller = JobDispatch::Signaller.new(config.signaller[:connect])
+                         signaller.connect
+                         signaller
+                       else
+                         Null::Object.instance
+                       end
+    self.signaller.signal(queue)
+  end
+
+
+  def enqueue(job_attrs)
+    address = JobDispatch.config.broker[:connect]
+    socket = JobDispatch.context.socket(ZMQ::REQ)
+    socket.connect(address)
+    socket.send(JSON.dump({command:'enqueue',job:job_attrs}))
+    result = JSON.parse(socket.recv)
+    socket.close
+    result
+  end
+
+  module_function :context
+  module_function :idle
+  module_function :unknown_command
+  module_function :signal
+  module_function :configure
+  module_function :config
+  module_function :enqueue
+  module_function :load_config
+  module_function :load_config_from_yml
+
+  mattr_accessor :logger
+  mattr_accessor :signaller
+end
+
+JobDispatch.logger = Null::Object.instance

data/spec/factories/jobs.rb

@@ -0,0 +1,19 @@
+FactoryGirl.define do
+  factory :job do
+    id { SecureRandom.uuid }
+    queue :default
+    status JobDispatch::Job::PENDING
+    parameters []
+    target "SecureRandom"
+    add_attribute(:method) { "uuid" }
+
+    enqueued_at { Time.now }
+    scheduled_at { Time.at(0) }
+    expire_execution_at { nil}
+    timeout 10
+    retry_count 0
+    retry_delay 20
+    completed_at nil
+    result nil
+  end
+end

data/spec/job_dispatch/broker/socket_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe JobDispatch::Broker::Socket do
+
+  subject { JobDispatch::Broker::Socket.new('tcp://localhost:1999') }
+
+  context "Reading messages from a worker" do
+    before :each do
+      @socket = double('Socket')
+      subject.stub(:socket => @socket)
+    end
+
+    context "with a valid message" do
+      before :each do
+        message = ZMQ::Message.new
+        message.addstr(JSON.dump({command: 'ready', queue: 'my_queue'}))
+        message.wrap(ZMQ::Frame('my_worker_id'))
+        @socket.stub(:recv_message => message)
+        @command = subject.read_command
+      end
+
+      it "returns a Command" do
+        expect(@command).to be_a(JobDispatch::Broker::Command)
+      end
+
+      it "reads the command" do
+        expect(@command.parameters[:command]).to eq('ready')
+      end
+
+      it "reads the worker id" do
+        expect(@command.worker_id.to_sym).to eq(:my_worker_id)
+      end
+    end
+
+    context "with an invalid message" do
+      before :each do
+        message = ZMQ::Message.new
+        message.addstr("Hello")
+        message.wrap(ZMQ::Frame('my_worker_id'))
+        @socket.stub(:recv_message => message)
+        @command = subject.read_command
+      end
+
+      it "returns a Command" do
+        expect(@command).to be_a(JobDispatch::Broker::Command)
+      end
+
+      it "reads the worker id" do
+        expect(@command.worker_id.to_sym).to eq(:my_worker_id)
+      end
+    end
+  end
+end