job_reactor 0.5.0.beta2

data/README.markdown

@@ -0,0 +1,88 @@
+JobReactor
+==========
+
+JobReactor is a library for creating and processing background jobs.
+It is asynchronous client-server distributed system based on [EventMachine][0].
+
+Quick start
+===========
+Coming soon, see examples
+
+Main features
+=============
+1. Client-server architecture
+-----------------------------
+You can run as many distributors and working nodes as you need. You are free to choose the strategy.
+If you have many background tasks from each part of your application you can use, for example, 3 distributors (one in each process) and 10 working nodes.
+If you don't have many jobs you can leave only one node which will be connected to 3 distributors.
+2. High scalability
+-------------------
+Nodes and distributors are connected via TCP. So, you can run them on any machine you can connect to.
+Nodes may use different storages or the same one. So, you can store vitally important jobs in relational database and
+simple insignificant jobs in memory.
+And more: your nodes may create jobs for others nodes and communicate with each other. See page [advance usage].
+3. Full job control
+-------------------
+You can add callback and errbacks to the job which will be called on the node.
+You also can add 'success feedback' and 'error feedback' which will be called in your main application.
+When job is done on remote node, your application will receive the result inside corespondent 'feedback'.
+If error occur in the job you can see it in errbacks and do what you want.
+Inside the job you can get information about when it starts, which node execute job and etc.
+You also can add some arguments to the job on-the-fly which will be used in the subsequent callbacks and errbacks. See [advance usage].
+4. Reliability
+--------------
+You can run additional nodes and stop any nodes on-the-fly.
+Distributor is smart enough to send jobs to another node if someone is stopped or crashed.
+If no nodes are connected to distributor it will keep jobs in memory and send them when nodes start.
+If node is stopped or crashed it will retry stored jobs after start.
+5. EventMachine available
+-------------------------
+Remember, your jobs will be run inside EventMachine reactor! You can easily use the power of async nature of EventMachine.
+Use asynchronous [http requests], [websockets], [etc.], [etc.], and [etc]. See page [advance usage].
+6. Deferred and periodic jobs
+-----------------------------
+You can use deferred jobs which will run 'after' some time or 'run_at' given time.
+You can create periodic jobs which will run every given time period and cancel them on condition.
+7. No polling
+-------------
+There is no storage polling. Absolutely. When node receives job (no matter instant, periodic or deferred) there will be EventMachine timer created
+which will start job at the right time.
+8. Job retrying
+--------------
+If job fails it will be retried. You can choose global retrying strategy or manage separate jobs.
+9. Predefined nodes
+-------------------
+You can specify node for jobs, so they will be executed in that node environment. And you can specify which node is forbidden for the job.
+If no nodes are specified distributor will try to send the job to the first free node.
+10. Node based priorities
+-----------------------
+There are no priorities like in Delayed::Job or Stalker. Bud there are flexible node-based priorities.
+You can specify the node which should execute the job. You can reserve several nodes for high priority jobs.
+
+
+
+The main parts of JobReactor are:
+---------------------------------
+JobReactor module for creating jobs.
+Distributor module for 'distributing' jobs between working nodes.
+Node object for job processing.
+#TODO
+
+
+
+
+
+
+
+
+How it works
+------------
+#TODO
+
+
+
+
+
+Links:
+------
+[0]: http://rubyeventmachine.com/

data/lib/job_reactor/distributor/client.rb

@@ -0,0 +1,40 @@
+# TODO comment it
+module JobReactor
+  module Distributor
+    class Client < EM::Connection
+      def initialize(name)
+        @name = name
+      end
+
+      def name
+        @name
+      end
+
+      def lock
+        @lock = true
+      end
+
+      def unlock
+        @lock = false
+      end
+
+      def locked?
+        @lock
+      end
+
+      def available?
+        !locked?
+      end
+
+      def receive_data(data)
+        self.unlock if data == 'ok'
+      end
+
+      def unbind
+        JR::Logger.log "#{@name} disconnected"
+        close_connection
+        JobReactor::Distributor.connections.delete(self)
+      end
+    end
+  end
+end

data/lib/job_reactor/distributor/server.rb

@@ -0,0 +1,31 @@
+# TODO comment it
+module JobReactor
+  module Distributor
+    class Server < EM::Connection
+
+      def post_init
+        JR::Logger.log 'Begin node handshake'
+      end
+
+      def receive_data(data)
+        data = Marshal.load(data)
+        if data[:node_info]
+          node_info = data[:node_info]
+          JR::Logger.log "Receive data from node: #{data[:node_info]}"
+          JobReactor::Distributor.nodes << node_info
+          connection = EM.connect(*node_info[:server], Client, node_info[:name])
+          JobReactor::Distributor.connections << connection
+        elsif data[:success]
+          JR.run_succ_feedback(data[:success])
+          send_data('ok')
+        elsif data[:error]
+          JR.run_err_feedback(data[:error])
+          send_data('ok')
+        end
+
+        data
+      end
+
+    end
+  end
+end

data/lib/job_reactor/distributor.rb

@@ -0,0 +1,92 @@
+require 'job_reactor/distributor/client'
+require 'job_reactor/distributor/server'
+
+module JobReactor
+  module Distributor
+    extend self
+
+    def host
+      @@host
+    end
+
+    def port
+      @@port
+    end
+
+    def nodes
+      @@nodes ||= []
+    end
+
+    # Contains connections pool
+    def connections
+      @@connections ||= []
+    end
+
+    #Starts distributor on given hast and port
+
+    def start(host, port)
+      @@host = host
+      @@port = port
+      EM.start_server(host, port, JobReactor::Distributor::Server, [host, port])
+      JR::Logger.log "Distributor listens #{host}:#{port}"
+      #EM.add_periodic_timer(5) do
+      #  JR::Logger.log('Available nodes: ' << JR::Distributor.connections.map(&:name).join(' '))
+      #end
+    end
+
+    # Tries to find available node connection
+    # If it is distributor will send marshalled data
+    # If get_connection returns nil distributor will try again after 1 second
+    #
+    def send_data_to_node(hash)
+      connection = get_connection(hash)
+      if connection
+        data = Marshal.dump(hash)
+        connection.send_data(data)
+        connection.lock
+      else
+        EM.next_tick do
+          send_data_to_node(hash)
+        end
+      end
+    end
+
+    private
+
+    # Looks for available connection.
+    # If job hash specified node, tries check if the node is available.
+    # If not, returns nil or tries to find any other free node if :always_use_specified_node == true
+    # If job hasn't any specified node, methods return any available connection or nil (and will be launched again in one second)
+
+    def get_connection(hash)
+      check_node_pool
+      if hash['node']
+        node_connection = connections.select{ |con| con.name == hash['node'] && con.name != hash['not_node']}.first
+        JR::Logger.log("WARNING: Node #{hash['node']} is not available") unless node_connection
+        if node_connection.try(:available?)
+          node_connection
+        else
+          JR.config[:always_use_specified_node] ?  nil : connections.select{ |con| con.available? && con.name != hash['not_node'] }.first
+        end
+      else
+        connections.select{ |con| con.available? && con.name != hash['not_node'] }.first
+      end
+    end
+
+    # Checks node poll. If it is empty will fail after :when_node_pull_is_empty_will_raise_exception_after seconds
+    # The distributor will fail when number of timers raise to EM.get_max_timers which if default 100000 for the majority system
+    # To exit earlier may be useful for error detection
+    #
+    def check_node_pool
+      if connections.size == 0
+        JR::Logger.log 'Warning: Node pool is empty'
+        EM::Timer.new(JR.config[:when_node_pull_is_empty_will_raise_exception_after]) do
+          if connections.size == 0
+            raise JobReactor::NodePoolIsEmpty
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/job_reactor/job_reactor/config.rb

@@ -0,0 +1,31 @@
+# Names are informative
+# TODO
+module JobReactor
+  def self.config
+    @@config ||= {}
+  end
+end
+
+JR = JobReactor
+
+JR.config[:job_directory] = 'reactor_jobs'
+JR.config[:max_attempt] = 10
+JR.config[:retry_multiplier] = 5
+JR.config[:retry_jobs_at_start] = true
+JR.config[:merge_job_itself_to_args] = true
+JR.config[:log_job_processing] = true
+JR.config[:always_use_specified_node] = false #will send job to another node if specified node is not available
+JR.config[:remove_done_jobs] = true
+JR.config[:remove_cancelled_jobs] = true
+JR.config[:when_node_pull_is_empty_will_raise_exception_after] = 3600
+
+JR.config[:redis_host] = 'localhost'
+JR.config[:redis_port] = 6379
+
+#TODO next releases with rails support
+#JR.config[:active_record_adapter] = 'mysql2'
+#JR.config[:active_record_database] = 'em'
+#JR.config[:active_record_user] = ''
+#JR.config[:active_record_password] = ''
+#JR.config[:active_record_table_name] = 'reactor_jobs'
+#JR.config[:use_custom_active_record_connection] = true

data/lib/job_reactor/job_reactor/exceptions.rb

@@ -0,0 +1,23 @@
+module JobReactor
+
+  # The purpose of exceptions is in their names
+  # TODO
+
+  class NoJobsDefined < RuntimeError
+  end
+  class NoSuchJob < RuntimeError
+  end
+  class CancelJob < RuntimeError
+  end
+  class NodePoolIsEmpty < RuntimeError
+  end
+  class NoSuchNode < RuntimeError
+  end
+  class LostConnection < RuntimeError
+  end
+  class SchedulePeriodicJob < RuntimeError
+  end
+
+end
+
+

data/lib/job_reactor/job_reactor/job_parser.rb

@@ -0,0 +1,39 @@
+# parse jobs defined in the JR.config[:job_directory]
+# build hash of the following structure:
+# {"job_1" => {
+#    job:  Proc,
+#    callbacks: [
+#        ["first_callback", Proc],
+#        ["second_callback", Proc]
+#    ],
+#    errbacks: [
+#        ["first_errback", Proc],
+#        ["second_errback", Proc]
+#    ]
+# },
+# "job_2" => {
+#     job: Proc,
+#     callbacks: [[]],
+#     errbacks: [[]]
+#  }
+# }
+# Names of callbacks and errbacks are optional and may be used just for description
+
+module JobReactor
+  extend self
+
+  def job(name, &block)
+    JR.jobs.merge!(name => { job: block })
+  end
+
+  def job_callback(name, callback_name = 'noname', &block)
+    JR.jobs[name].merge!(callbacks: []) unless JR.jobs[name][:callbacks]
+    JR.jobs[name][:callbacks] << [callback_name, block]
+  end
+
+  def job_errback(name, errback_name = 'noname', &block)
+    JR.jobs[name].merge!(errbacks: []) unless JR.jobs[name][:errbacks]
+    JR.jobs[name][:errbacks] << [errback_name, block]
+  end
+
+end

data/lib/job_reactor/job_reactor/storages.rb

@@ -0,0 +1,25 @@
+# Storages implement simple functionality.
+# There are four methods should be implemented:
+# save(hash).
+# load(hash).
+# destroy(hash).
+# jobs_for(name).
+# The last one is used when node is restarting to retry saved jobs.
+# The storage may not be thread safe, because each node manage it own jobs and don't now anything about others.
+
+# Defines storages for lazy loading
+
+# TODO 'NEXT RELEASE'
+# require 'active_record'
+# class JobReactor::ActiveRecordStorage < ::ActiveRecord::Base; end
+
+module JobReactor::MemoryStorage; end
+module JobReactor::RedisStorage; end
+
+module JobReactor
+  STORAGES = {
+      #'active_record_storage' => JobReactor::ActiveRecordStorage,
+      'memory_storage' => JobReactor::MemoryStorage,
+      'redis_storage' => JobReactor::RedisStorage
+  }
+end

data/lib/job_reactor/job_reactor.rb

@@ -0,0 +1,208 @@
+# The core.
+# Gives API to parse jobs, send them to node using distributor, and make them for node.
+
+require 'job_reactor/job_reactor/config'
+require 'job_reactor/job_reactor/job_parser'
+require 'job_reactor/job_reactor/exceptions'
+require 'job_reactor/job_reactor/storages'
+
+module JobReactor
+
+  # Yes, we monkeypatched Ruby core class.
+  # Now all hashes hash EM::Deferrable callbacks and errbacks.
+  # It is just for simplicity.
+  # It's cool use 'job = {}' instead 'job = JobHash.new.
+  # We are ready to discuss it and change.
+  #
+  Hash.send(:include, EM::Deferrable)
+
+  class << self
+
+    # Accessors to jobs.
+    #
+    def jobs
+      @@jobs ||= { }
+    end
+
+    # Ready flag.
+    # @@ready is true when block is called inside EM reactor.
+    #
+    def ready!
+      @@ready = true
+    end
+
+    def ready?
+      (@@ready ||= false) && EM.reactor_running?
+    end
+
+    # Requires storage
+    # Creates and start node.
+    #
+    def start_node(opts)
+      parse_jobs
+      require_storage!(opts)
+      node = Node.new(opts)
+      node.start
+    end
+
+    def start_distributor(host, port)
+      JR::Distributor.start(host, port)
+    end
+
+    def succ_feedbacks
+      @@callbacks ||= { }
+    end
+
+    def err_feedbacks
+      @@errbacks ||= { }
+    end
+
+    # Here is the only method user can call inside the application (excepts start-up methods, of course).
+    # You have to specify job_name and optionally its args and opts.
+    # The method set initial arguments and send job to distributor which will send it to node.
+    # Options are :after and :period (for deferred and periodic jobs), and :node to specify the preferred node to launch job.
+    # Use :always_use_specified_node option to be sure that job will launched in the specified node.
+    # Job itself is a hash with the following keys:
+    # name, args, make_after, last_error, run_at, failed_at, attempt, period, node, not_node, status, distributor, on_success, on_error.
+    # TODO examples.
+    #
+    def enqueue(name, args = { }, opts = { }, success_proc = nil, error_proc = nil)
+      hash = { 'name' => name, 'args' => args, 'attempt' => 0, 'status' => 'new' }
+
+      hash.merge!('period' => opts[:period]) if opts[:period]
+      opts[:after] = (opts[:run_at] - Time.now) if opts[:run_at]
+      hash.merge!('make_after' => (opts[:after] || 0))
+
+      hash.merge!('node' => opts[:node]) if opts[:node]
+      hash.merge!('not_node' => opts[:not_node]) if opts[:not_node]
+
+      hash.merge!('distributor' => "#{JR::Distributor.host}:#{JR::Distributor.port}")
+
+      add_succ_feedbacks!(hash, success_proc) if success_proc
+      add_err_feedbacks!(hash, error_proc) if error_proc
+
+      JR::Distributor.send_data_to_node(hash)
+    end
+
+
+    # This method is used by node (Node#schedule).
+    # It makes job from hash by calling callback and errback methods.
+    #
+    # The strategy is the following:
+    # First and last callback (add_start_callback) are informational.
+    # Second is the proc specified in JR.job method.
+    # Third and ... are the procs specified in job_callbacks.
+    #
+    # Then errbacks are attached.
+    # They are called when error occurs in callbacks.
+    # The last errback raise exception again to return job back to node workflow.
+    # See Node#do_job method to better understand how this works.
+    #
+    def make(hash) #new job is a Hash
+      raise NoSuchJob unless jr_job = JR.jobs[hash['name']]
+
+      job = hash
+      add_start_callback(job) if JR.config[:log_job_processing]
+      job.callback(&jr_job[:job])
+
+      jr_job[:callbacks].each do |callback|
+        job.callback(&callback[1])
+      end if jr_job[:callbacks]
+
+      add_last_callback(job) if JR.config[:log_job_processing]
+
+      add_start_errback(job) if JR.config[:log_job_processing]
+
+      jr_job[:errbacks].each do |errback|
+        job.errback(&errback[1])
+      end if jr_job[:errbacks]
+
+      add_complete_errback(job) if JR.config[:log_job_processing]
+
+      job
+    end
+
+    # Runs success callbacks with job args
+    #
+    def run_succ_feedback(data)
+      proc = data[:do_not_delete] ? succ_feedbacks[data[:callback_id]] : succ_feedbacks.delete(data[:callback_id])
+      proc.call(data[:args]) if proc
+    end
+
+    # Runs error callbacks with job args
+    # Exception class is in args[:error]
+    #
+    def run_err_feedback(data)
+      proc = err_feedbacks.delete(data[:errback_id])
+      proc.call(data[:args]) if proc
+    end
+
+    private
+
+    # Requires storage and change opts[:storage] to the constant
+    #
+    def require_storage!(opts)
+      require "job_reactor/storages/#{opts[:storage]}"
+      opts[:storage] = STORAGES[opts[:storage]]
+    end
+
+    # Loads all *.rb files in the :job_directory folder
+    # See job_reactor/job_parser to understand how job hash is built
+    #
+    def parse_jobs
+      JR.config[:job_directory] += '/**/*.rb'
+      Dir[JR.config[:job_directory]].each {|file| load file }
+    end
+
+    # Adds success callback which will launch when node reports success
+    #
+    def add_succ_feedbacks!(hash, callback)
+      distributor = "#{JR::Distributor.host}:#{JR::Distributor.port}"
+      feedback_id = "#{distributor}_#{Time.now.utc.to_f}"
+      succ_feedbacks.merge!(feedback_id => callback)
+      hash.merge!('on_success' => feedback_id)
+    end
+
+    # Adds error callback which will launch when node reports error
+    #
+    def add_err_feedbacks!(hash, errback)
+      distributor = "#{JR::Distributor.host}:#{JR::Distributor.port}"
+      feedback_id = "#{distributor}_#{Time.now.utc.to_f}"
+      err_feedbacks.merge!(feedback_id => errback)
+      hash.merge!('on_error' => feedback_id)
+    end
+
+    # Logs the beginning.
+    #
+    def add_start_callback(job)
+      job.callback do
+        JR::Logger.log_event(:start, job)
+      end
+    end
+
+    # Logs the completing
+    #
+    def add_last_callback(job)
+      job.callback do
+        JR::Logger.log_event(:complete, job)
+      end
+    end
+
+    # Logs the beginning or error cycle.
+    #
+    def add_start_errback(job)
+      job.errback do
+        JR::Logger.log_event(:error, job)
+      end
+    end
+
+    # Logs the end of error cycle
+    #
+    def add_complete_errback(job)
+      job.errback do
+        JR::Logger.log_event(:error_complete, job)
+      end
+    end
+
+  end
+end

data/lib/job_reactor/logger.rb

@@ -0,0 +1,57 @@
+module JobReactor
+  module Logger
+################
+#   To set output stream
+    @@logger_method = :puts
+
+    def self.stdout=(value)
+      if value.is_a?(Symbol) && value == :rails_logger
+        @@stdout        = Rails.logger
+        @@logger_method = :info
+      else
+        @@stdout        = value
+        @@logger_method = :puts
+      end
+    end
+
+    def self.stdout
+      @@stdout ||= $stdout
+    end
+
+#################
+#   Is checked in dev_log
+
+    @@development = false
+
+    def self.development=(value)
+      @@development = !!value
+    end
+
+#################
+
+# Log message to output stream
+#
+    def self.log(msg)
+      stdout.public_send(@@logger_method, '-'*100)
+      stdout.public_send(@@logger_method, msg)
+    end
+
+    # Build string for job event and log it
+    #
+    def self.log_event(event, job)
+      log("Log: #{event} #{job['name']}")
+    end
+
+    # Log if JR::Logger.development is set to true
+    #
+    def self.dev_log(msg)
+      log(msg) if development?
+    end
+
+    # Is JR::Logger.development set to true?
+    #
+    def self.development?
+      @@development
+    end
+  end
+end

data/lib/job_reactor/node/client.rb

@@ -0,0 +1,54 @@
+module JobReactor
+  class Node
+    class Client < EM::Connection
+
+      def initialize(node, distributor)
+        @node = node
+        @distributor = distributor
+      end
+
+      def post_init
+        JR::Logger.log("Searching for distributor: #{@distributor.join(' ')} ...")
+      end
+
+      def lock
+        @lock = true
+      end
+
+      def unlock
+        @lock = false
+      end
+
+      def locked?
+        @lock
+      end
+
+      def available?
+        !locked?
+      end
+
+      def receive_data(data)
+        self.unlock if data == 'ok'
+      end
+
+      # Sends node credentials to distributor.
+      #
+      def connection_completed
+        JR::Logger.log('Begin distributor handshake')
+        data = {node_info: {name: @node.config[:name], server: @node.config[:server]} }
+        data = Marshal.dump(data)
+        send_data(data)
+      end
+
+      # Tries to connect.
+      #
+      def unbind
+        EM::Timer.new(1) do
+          @node.connect_to(@distributor)
+        end
+      end
+
+    end
+
+  end
+end

data/lib/job_reactor/node/server.rb

@@ -0,0 +1,38 @@
+module JobReactor
+  class Node
+    class Server < EM::Connection
+
+      #Need to know the storage to call save method on it
+      #Need to now node name to send it to the distributor
+      #
+      def initialize(node, storage)
+        @storage = storage
+        @node = node
+      end
+
+      #Ok, node is connected and ready to work
+      #
+      def post_init
+        JR::Logger.log("#{@node.name} ready to work")
+      end
+
+      # It is the place where job life cycle begins.
+      # This method:
+      # -receives data from distributor;
+      # -saves them in storage;
+      # -returns 'ok' to unlock node connection;
+      # -and schedules job;
+      #
+      def receive_data(data)
+        hash = Marshal.load(data)
+        JR::Logger.log("#{@node.name} received job: #{hash}")
+        hash.merge!('node' => @node.name)
+        @storage.save(hash) do |hash|
+          @node.schedule(hash)
+        end
+        send_data('ok')
+      end
+
+    end
+  end
+end

data/lib/job_reactor/node.rb

@@ -0,0 +1,222 @@
+require 'job_reactor/node/server'
+require 'job_reactor/node/client'
+module JobReactor
+  class Node
+
+    def initialize(opts)
+      @config = { storage: opts[:storage], name: opts[:name], server: opts[:server], distributors: opts[:distributors]}
+    end
+
+    def config
+      @config
+    end
+
+    # Config accessors.
+    #
+    [:storage, :name, :server, :distributors].each do |method|
+      define_method(method) do
+        config[method]
+      end
+    end
+
+    # Store distributor connection instances.
+    #
+    def connections
+      @connections ||= {}
+    end
+
+    # Retrying jobs if any,
+    # starts server and tries to connect to distributors.
+    #
+    def start
+      retry_jobs if JR.config[:retry_jobs_at_start]
+      EM.start_server(*self.config[:server], Server, self, self.storage)
+      self.config[:distributors].each do |distributor|
+        connect_to(distributor)
+      end
+    end
+
+    # Connects to distributor.
+    # This method is public, because it is called by client when connection interrupt.
+    #
+    def connect_to(distributor)
+      if connections[distributor]
+        JR::Logger.log 'Searching for distributors ...'
+        connections[distributor].reconnect(*distributor)
+      else
+        connections.merge!(distributor => EM.connect(*distributor, Client, self, distributor))
+      end
+    end
+
+    # The method is called by node server.
+    # It makes a job and run do_job.
+    #
+    def schedule(hash)
+      EM::Timer.new(hash['make_after']) do  #Of course, we can start job immediately (unless it is 'after' job), but we let EM take care about it. Maybe there is another job is ready to start
+        self.storage.load(hash) do |hash|
+          if job = JR.make(hash)  #If we decide fail silently. See JR.make
+            do_job(job)
+          else
+            #TODO Do nothing or raise exception ????
+          end
+        end
+      end
+    end
+
+    private
+
+    # Calls succeed on deferrable object.
+    # When job (or it's callbacks) fails, errbacks are launched.
+    # If errbacks fails job will be relaunched.
+    #
+    # You can see custom exception 'CancelJob''.
+    # You can use it to change normal execution.
+    #
+    def do_job(job)
+      job['run_at'] = Time.now
+      job['status'] = 'in progress'
+      storage.save(job) do |job|
+        begin
+          args = job['args'].merge(JR.config[:merge_job_itself_to_args] ? {:job_itself => job.dup} : {})
+          job.succeed(args)
+          job['args'] = args
+          job_completed(job)
+        rescue JobReactor::CancelJob
+          cancel_job(job)
+        rescue Exception => e
+          rescue_job(e, job)
+        end
+      end
+    end
+
+    # Reports success to distributor if should do it.
+    # If job is 'periodic' job schedule it again.
+    # Sets status completed or removes job from storage.
+    #
+    def job_completed(job)
+      report_success(job) if job['on_success']
+      if job['period'] && job['period'].to_i > 0
+        job['status'] = 'queued'
+        job['make_after'] = job['period']
+        job['args'].delete(:job_itself)
+        storage.save(job) { |job| schedule(job) }
+      else
+        if JR.config[:remove_done_jobs]
+          storage.destroy(job)
+        else
+          job['status'] = 'complete'
+          storage.save(job)
+        end
+      end
+    end
+
+    #Lanches job errbacks
+    #
+    def rescue_job(e, job)
+      begin
+        job['failed_at']  = Time.now #Save error info
+        job['last_error'] = e
+        job['status']     = 'error'
+        self.storage.save(job) do |job|
+          begin
+            args = job['args'].merge!(:error => e).merge(JR.config[:merge_job_itself_to_args] ? { :job_itself => job.dup } : { })
+            job.fail(args) #Fire errbacks. You can access error in you errbacks (args[:error])
+            job['args'] = args
+            complete_rescue(job)
+          rescue JobReactor::CancelJob
+            cancel_job(job) #If it was cancelled we destroy it or set status 'cancelled'
+          rescue Exception => e  #Recsue Exceptions in errbacks
+            job['args'].merge!(:errback_error => e)
+            complete_rescue(job)
+          end
+        end
+      end
+    end
+
+    #Tryes again or report error
+    #
+    def complete_rescue(job)
+      if job['attempt'].to_i < JobReactor.config[:max_attempt] - 1
+        try_again(job)
+      else
+        report_error(job) if job['on_error']
+      end
+    end
+
+    # Cancels job. Remove or set 'cancelled status'
+    #
+    def cancel_job(job)
+      report_error(job) if job['on_error']
+      if JR.config[:remove_cancelled_jobs]
+        storage.destroy(job)
+      else
+        job['status'] = 'cancelled'
+        storage.save(job)
+      end
+    end
+
+    # try_again has special condition for periodic jobs.
+    # They will be rescheduled after period time.
+    #
+    def try_again(job)
+      job['attempt'] += 1
+      if job['period'] && job['period'] > 0
+        job['make_after'] = job['period']
+      else
+        job['make_after'] = job['attempt'] * JobReactor.config[:retry_multiplier]
+      end
+      job['args'].delete(:job_itself)
+      self.storage.save(job) do |job|
+        self.schedule(job)
+      end
+    end
+
+    # Retries jobs.
+    # Runs only once when node starts.
+    #
+    def retry_jobs
+      storage.jobs_for(name) do |job_to_retry|
+        job_to_retry['args'].merge!(:retrying => true)
+        try_again(job_to_retry) if job_to_retry
+      end
+    end
+
+    # Reports success to node, sends jobs args
+    #
+    def report_success(job)
+      host, port = job['distributor'].split(':')
+      port = port.to_i
+      distributor = self.connections[[host, port]]
+      data = {:success => { callback_id: job['on_success'], args: job['args']}}
+      data[:success].merge!(do_not_delete: true) if job['period'] && job['period'].to_i > 0
+      data = Marshal.dump(data)
+      send_data_to_distributor(distributor, data)
+    end
+
+    # Reports error to node, sends jobs args.
+    # Exception class is merged to args.
+    #
+    def report_error(job)
+      host, port = job['distributor'].split(':')
+      port = port.to_i
+      distributor = self.connections[[host, port]]
+      data = {:error => { errback_id: job['on_error'], args: job['args']}}
+      data = Marshal.dump(data)
+      send_data_to_distributor(distributor, data)
+    end
+
+    # Sends data to distributor
+    #
+    def send_data_to_distributor(distributor, data)
+      if distributor.locked?
+        EM.next_tick do
+          send_data_to_distributor(distributor, data)
+        end
+      else
+        distributor.send_data(data)
+        distributor.lock
+      end
+    end
+
+  end
+end

data/lib/job_reactor/storages/memory_storage.rb

@@ -0,0 +1,39 @@
+# TODO comment it
+module JobReactor
+  module MemoryStorage
+
+    @@storage = { }
+
+    class << self
+      def storage
+        @@storage
+      end
+
+      def load(hash, &block)
+        hash      = storage[hash['id']]
+        hash_copy = { }
+        hash.each { |k, v| hash_copy.merge!(k => v) }
+        block.call(hash_copy) if block_given?
+      end
+
+      def save(hash, &block)
+        unless (hash['id'])
+          id = Time.now.to_f.to_s
+          hash.merge!('id' => id)
+        end
+        storage.merge!(hash['id'] => hash)
+
+        block.call(hash) if block_given?
+      end
+
+      def destroy(hash)
+        storage.delete(hash['id'])
+      end
+
+      def jobs_for(name, &block)  #No persistance
+        nil
+      end
+    end
+
+  end
+end

data/lib/job_reactor/storages/redis_storage.rb

@@ -0,0 +1,76 @@
+# TODO comment it
+require 'em-redis'
+
+module JobReactor
+  module RedisStorage
+    @@storage = EM::Protocols::Redis.connect(host: JobReactor.config[:redis_host], port: JobReactor.config[:redis_port])
+    ATTRS = %w(id name args last_error run_at failed_at attempt period make_after status distributor on_success on_error)
+
+    class << self
+
+      def storage
+        @@storage
+      end
+
+      def load(hash, &block)
+        key = "#{hash['node']}_#{hash['id']}"
+        hash_copy = {'node' => hash['node']} #need new object, because old one has been 'failed'
+
+        storage.hmget(key, *ATTRS) do |record|
+          ATTRS.each_with_index do |attr, i|
+            hash_copy[attr] = record[i]
+          end
+          ['attempt', 'period', 'make_after'].each do |attr|
+            hash_copy[attr] = hash_copy[attr].to_i
+          end
+          hash_copy['args'] = Marshal.load(hash_copy['args'])
+
+          block.call(hash_copy) if block_given?
+        end
+      end
+
+
+      def save(hash, &block)
+        hash.merge!('id' => Time.now.to_f.to_s) unless hash['id']
+        key = "#{hash['node']}_#{hash['id']}"
+        args, hash['args'] = hash['args'], Marshal.dump(hash['args'])
+
+        storage.hmset(key, *ATTRS.map{|attr| [attr, hash[attr]]}.flatten) do
+          hash['args'] = args
+
+          block.call(hash) if block_given?
+        end
+      end
+
+      def destroy(hash)
+        storage.del("#{hash['node']}_#{hash['id']}")
+      end
+
+      def destroy_all_jobs_for(name)
+        pattern = "*#{name}_*"
+        storage.del(*storage.keys(pattern))
+      end
+
+      def jobs_for(name, &block)
+        pattern = "*#{name}_*"
+        storage.keys(pattern) do |keys|
+          keys.each do |key|
+            hash = {}
+            storage.hget(key, 'id') do |id|
+              hash['id'] = id
+              hash['node'] = name
+              self.load(hash) do |hash|
+                if hash['status'] != 'complete' && hash['status'] != 'cancelled' && hash['attempt'].to_i < JobReactor.config[:max_attempt]
+                  block.call(hash)
+                end
+              end
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/job_reactor.rb

@@ -0,0 +1,44 @@
+require 'eventmachine'
+require 'job_reactor/job_reactor'
+require 'job_reactor/logger'
+require 'job_reactor/node'
+require 'job_reactor/distributor'
+
+
+# JobReactor initialization process.
+# Parses jobs, runs EventMachine reactor and call given block inside reactor.
+# The ::run method run EM in Thread to do not prevent execution of application.
+# The ::wait_em_and_run is for using JobReactor with
+# applications already have EventMachine inside and run it at start. Server Thin, for example.
+# The run! method is for using JobReactor as standalone application. Advanced usage. For example you wand use node with distributor in one process
+#
+module JobReactor
+  extend self
+
+  def run(&block)
+    Thread.new do
+      if EM.reactor_running?
+        block.call if block_given?
+        JR.ready!
+      else
+        EM.run do
+          block.call if block_given?
+          JR.ready!
+        end
+      end
+    end
+  end
+
+  def run!(&block)
+    if EM.reactor_running?
+      block.call if block_given?
+      JR.ready!
+    else
+      EM.run do
+        block.call if block_given?
+        JR.ready!
+      end
+    end
+  end
+
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: job_reactor
+version: !ruby/object:Gem::Version
+  version: 0.5.0.beta2
+  prerelease: 6
+platform: ruby
+authors:
+- Anton Mishchuk
+- Andrey Rozhkovskiy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &83843190 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *83843190
+- !ruby/object:Gem::Dependency
+  name: em-redis
+  requirement: &83842960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *83842960
+description: ! "    JobReactor is a library for creating and processing background
+  jobs.\n    It is client-server distributed system based on EventMachine.\n"
+email: anton.mishchuk@gmial.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/job_reactor.rb
+- lib/job_reactor/job_reactor.rb
+- lib/job_reactor/node.rb
+- lib/job_reactor/storages/redis_storage.rb
+- lib/job_reactor/storages/memory_storage.rb
+- lib/job_reactor/distributor/client.rb
+- lib/job_reactor/distributor/server.rb
+- lib/job_reactor/node/client.rb
+- lib/job_reactor/node/server.rb
+- lib/job_reactor/job_reactor/job_parser.rb
+- lib/job_reactor/job_reactor/storages.rb
+- lib/job_reactor/job_reactor/exceptions.rb
+- lib/job_reactor/job_reactor/config.rb
+- lib/job_reactor/distributor.rb
+- lib/job_reactor/logger.rb
+- README.markdown
+homepage: http://github.com/antonmi/job_reactor
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Simple, powerful and high scalable job queueing and background workers system
+  based on EventMachine
+test_files: []