job_reactor 0.5.0.beta2 → 0.5.0.beta3

data/README.markdown

@@ -1,14 +1,59 @@
 JobReactor
 ==========
+Now we are in beta (need to complete documentation and fix some bugs)
+---------------------------------------------------
 
-JobReactor is a library for creating and processing background jobs.
+JobReactor is a library for creating, scheduling and processing background jobs.
 It is asynchronous client-server distributed system based on [EventMachine][0].
+Inspired by Resque, Stalker, DelayedJob, and etc.
+
+JobReactor hasn't 'rails' integration for the time being.
+But it is very close. We need test the system with different servers (clusters) and automatize initialization and restart processes.
+Collaborators, you are welcome!
+
+So, read 'features' part and try JobReactor. You can do a lot with it.
 
 Quick start
 ===========
-Coming soon, see examples
-
-Main features
+Use `gem install job_reactor --pre` to try it.
+
+In you main application:
+`application.rb`
+``` ruby
+require 'job_reactor'
+JR.run do
+  JR.start_distributor('localhost', 5000)
+end
+sleep(1) until(JR.ready?)
+
+# The application
+loop do
+  sleep(3) #Your application is working
+  JR.enqueue 'my_job', {arg1: 'Hello'}
+end
+```
+Define the 'my_job' in separate directory (files with job's definitions must be in separate directory):
+`reactor_jobs/my_jobs.rb`
+``` ruby
+include JobReactor
+job 'my_job' do |args|
+  puts args[:arg1]
+end
+```
+And the last file - 'the worker code':
+`worker.rb`
+``` ruby
+require 'job_reactor'
+JR.config[:job_directory] = 'reactor_jobs' #this default config so you can omit this line
+JR.run! do
+  JR.start_node({:storage => 'memory_storage', :name => 'worker_1', :server => ['localhost', 5001], :distributors => [['localhost', 5000]] })
+end
+```
+Run 'application.rb' in one terminal window and 'worker.rb' in another.
+Node connects to distributor, receives the job and works.
+Cool! But it was the simplest example. See 'examples' directory and read the wiki (coming soon).
+
+Features
 =============
 1. Client-server architecture
 -----------------------------
@@ -18,7 +63,7 @@ If you don't have many jobs you can leave only one node which will be connected
 2. High scalability
 -------------------
 Nodes and distributors are connected via TCP. So, you can run them on any machine you can connect to.
-Nodes may use different storages or the same one. So, you can store vitally important jobs in relational database and
+Nodes may use different storage or the same one. So, you can store vitally important jobs in relational database and
 simple insignificant jobs in memory.
 And more: your nodes may create jobs for others nodes and communicate with each other. See page [advance usage].
 3. Full job control

data/lib/job_reactor/distributor.rb

@@ -13,25 +13,28 @@ module JobReactor
       @@port
     end
 
+    # Gets nodes
+    # You can monitor available nodes connections in you application.
+    # For example
+    # EM::PeriodicTimer.new(10) { JR::Logger.log nodes}
+    #
     def nodes
       @@nodes ||= []
     end
 
-    # Contains connections pool
+    # Contains connections pool - all node connections
+    #
     def connections
       @@connections ||= []
     end
 
     #Starts distributor on given hast and port
-
+    #
     def start(host, port)
       @@host = host
       @@port = port
-      EM.start_server(host, port, JobReactor::Distributor::Server, [host, port])
       JR::Logger.log "Distributor listens #{host}:#{port}"
-      #EM.add_periodic_timer(5) do
-      #  JR::Logger.log('Available nodes: ' << JR::Distributor.connections.map(&:name).join(' '))
-      #end
+      EM.start_server(host, port, JobReactor::Distributor::Server)
     end
 
     # Tries to find available node connection
@@ -57,13 +60,11 @@ module JobReactor
     # If job hash specified node, tries check if the node is available.
     # If not, returns nil or tries to find any other free node if :always_use_specified_node == true
     # If job hasn't any specified node, methods return any available connection or nil (and will be launched again in one second)
-
+    #
     def get_connection(hash)
-      check_node_pool
       if hash['node']
         node_connection = connections.select{ |con| con.name == hash['node'] && con.name != hash['not_node']}.first
-        JR::Logger.log("WARNING: Node #{hash['node']} is not available") unless node_connection
-        if node_connection.try(:available?)
+        if node_connection && node_connection.available?
           node_connection
         else
           JR.config[:always_use_specified_node] ?  nil : connections.select{ |con| con.available? && con.name != hash['not_node'] }.first
@@ -73,20 +74,5 @@ module JobReactor
       end
     end
 
-    # Checks node poll. If it is empty will fail after :when_node_pull_is_empty_will_raise_exception_after seconds
-    # The distributor will fail when number of timers raise to EM.get_max_timers which if default 100000 for the majority system
-    # To exit earlier may be useful for error detection
-    #
-    def check_node_pool
-      if connections.size == 0
-        JR::Logger.log 'Warning: Node pool is empty'
-        EM::Timer.new(JR.config[:when_node_pull_is_empty_will_raise_exception_after]) do
-          if connections.size == 0
-            raise JobReactor::NodePoolIsEmpty
-          end
-        end
-      end
-    end
-
   end
 end

data/lib/job_reactor/job_reactor/config.rb

@@ -17,7 +17,6 @@ JR.config[:log_job_processing] = true
 JR.config[:always_use_specified_node] = false #will send job to another node if specified node is not available
 JR.config[:remove_done_jobs] = true
 JR.config[:remove_cancelled_jobs] = true
-JR.config[:when_node_pull_is_empty_will_raise_exception_after] = 3600
 
 JR.config[:redis_host] = 'localhost'
 JR.config[:redis_port] = 6379

data/lib/job_reactor/job_reactor/exceptions.rb

@@ -1,22 +1,9 @@
 module JobReactor
 
-  # The purpose of exceptions is in their names
-  # TODO
-
-  class NoJobsDefined < RuntimeError
-  end
   class NoSuchJob < RuntimeError
   end
   class CancelJob < RuntimeError
   end
-  class NodePoolIsEmpty < RuntimeError
-  end
-  class NoSuchNode < RuntimeError
-  end
-  class LostConnection < RuntimeError
-  end
-  class SchedulePeriodicJob < RuntimeError
-  end
 
 end
 

data/lib/job_reactor/job_reactor/job_parser.rb

@@ -18,6 +18,31 @@
 #  }
 # }
 # Names of callbacks and errbacks are optional and may be used just for description
+#
+# Job and job_callbacks are absolutely identical on the node side.
+# They become callback of the Deferrable instance. 'job' is the first callback, 'job_callbacks' are the next
+#
+# Use job_callbacks to split your job into small parts.
+# You can send additional arguments from one callback to another by merging them into 'args'.
+# For example:
+#
+# job 'job' do |args|
+#   args.merge!(:another_arg => 'Hello')
+# end
+#
+# job_callback 'job', 'job_callback' do |args|
+#   puts args[:another_arg]
+# end
+#
+# This is true for errbacks too. Note that you can't access additional arguments added in callbacks in your errbacks
+# In errbacks you also have :error key in args which point the error message
+#
+# Note, that callbacks and errbacks are called one after another synchronously in one EM tick.
+#
+# You also have :job_itself in your args. You can turn this option of by setting JR.config[:merge_job_itself_to_args] to false
+# :job_itself point to Hash which has the following keys: "node", "id", name", "last_error", "run_at", "failed_at", "attempt", "period", "status"=>"error", "distributor", "on_success", "on_error"
+# So, you can all information about the 'job' inside job.
+#
 
 module JobReactor
   extend self

data/lib/job_reactor/job_reactor.rb

@@ -9,7 +9,7 @@ require 'job_reactor/job_reactor/storages'
 module JobReactor
 
   # Yes, we monkeypatched Ruby core class.
-  # Now all hashes hash EM::Deferrable callbacks and errbacks.
+  # Now all hashes has EM::Deferrable callbacks and errbacks.
   # It is just for simplicity.
   # It's cool use 'job = {}' instead 'job = JobHash.new.
   # We are ready to discuss it and change.
@@ -35,7 +35,8 @@ module JobReactor
       (@@ready ||= false) && EM.reactor_running?
     end
 
-    # Requires storage
+    # Parses jobs.
+    # Requires storage.
     # Creates and start node.
     #
     def start_node(opts)
@@ -62,9 +63,36 @@ module JobReactor
     # The method set initial arguments and send job to distributor which will send it to node.
     # Options are :after and :period (for deferred and periodic jobs), and :node to specify the preferred node to launch job.
     # Use :always_use_specified_node option to be sure that job will launched in the specified node.
-    # Job itself is a hash with the following keys:
+    # Job itself will be a hash with the following keys:
     # name, args, make_after, last_error, run_at, failed_at, attempt, period, node, not_node, status, distributor, on_success, on_error.
-    # TODO examples.
+    #
+    # Simple job with arguments.
+    # Arguments should be a Hash.
+    # Arguments will be serialized using Marshal.dump before sending to node, so be sure that objects in args can be dumped.
+    # (Do not use procs, objects with singleton methods, etc ... ).
+    #
+    # Example:
+    # JR.enqueue 'job', {:arg1 => 'arg1', :arg2 => 'arg2'}
+    #
+    # You can add the following options:
+    # :run_at - run at given time;
+    # :after - run after some time (in seconds);
+    # :period - will make periodic job which will be launched every opts[:period] seconds;
+    # :node - to send job to the specific node;
+    # :not_node - to do not send job to the node;
+    #
+    # Example:
+    # JR.enqueue 'job', {:arg1 => 'arg1'}, {:period => 100, :node => 'my_favorite_node'}
+    # JR.enqueue 'job', {:arg1 => 'arg1'}, {:after => 10, :not_node => 'some_node'}
+    #
+    # You can add 'success feedback' and 'error feedback'. We use term 'feedback' to distinguish them from callbacks and errbacks which are executed on the node side.
+    # These feedbacks are the procs. The first is 'success feedback', the second - 'error feedback'.
+    # These feedback procs are called with 'job arguments' as arguments.
+    #
+    # Example:
+    # success = proc { |args| result = args }
+    # error = proc { |args| result = args }
+    # JR.enqueue 'job', { :arg1 => 'arg1'}, {}, success, error
     #
     def enqueue(name, args = { }, opts = { }, success_proc = nil, error_proc = nil)
       hash = { 'name' => name, 'args' => args, 'attempt' => 0, 'status' => 'new' }
@@ -95,8 +123,6 @@ module JobReactor
     #
     # Then errbacks are attached.
     # They are called when error occurs in callbacks.
-    # The last errback raise exception again to return job back to node workflow.
-    # See Node#do_job method to better understand how this works.
     #
     def make(hash) #new job is a Hash
       raise NoSuchJob unless jr_job = JR.jobs[hash['name']]

data/lib/job_reactor/node.rb

@@ -41,7 +41,7 @@ module JobReactor
     #
     def connect_to(distributor)
       if connections[distributor]
-        JR::Logger.log 'Searching for distributors ...'
+        JR::Logger.log "Searching for distributor #{distributor.join(' ')}"
         connections[distributor].reconnect(*distributor)
       else
         connections.merge!(distributor => EM.connect(*distributor, Client, self, distributor))
@@ -126,7 +126,7 @@ module JobReactor
           rescue JobReactor::CancelJob
             cancel_job(job) #If it was cancelled we destroy it or set status 'cancelled'
           rescue Exception => e  #Recsue Exceptions in errbacks
-            job['args'].merge!(:errback_error => e)
+            job['args'].merge!(:errback_error => e) #So in args you now have :error and :errback_error
             complete_rescue(job)
           end
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: job_reactor
 version: !ruby/object:Gem::Version
-  version: 0.5.0.beta2
+  version: 0.5.0.beta3
   prerelease: 6
 platform: ruby
 authors:
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-01 00:00:00.000000000 Z
+date: 2012-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &83843190 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +22,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *83843190
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: em-redis
-  requirement: &83842960 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,7 +38,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *83842960
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! "    JobReactor is a library for creating and processing background
   jobs.\n    It is client-server distributed system based on EventMachine.\n"
 email: anton.mishchuk@gmial.com
@@ -41,21 +51,21 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/job_reactor.rb
-- lib/job_reactor/job_reactor.rb
-- lib/job_reactor/node.rb
-- lib/job_reactor/storages/redis_storage.rb
-- lib/job_reactor/storages/memory_storage.rb
-- lib/job_reactor/distributor/client.rb
-- lib/job_reactor/distributor/server.rb
-- lib/job_reactor/node/client.rb
 - lib/job_reactor/node/server.rb
-- lib/job_reactor/job_reactor/job_parser.rb
-- lib/job_reactor/job_reactor/storages.rb
-- lib/job_reactor/job_reactor/exceptions.rb
-- lib/job_reactor/job_reactor/config.rb
+- lib/job_reactor/node/client.rb
+- lib/job_reactor/distributor/server.rb
+- lib/job_reactor/distributor/client.rb
 - lib/job_reactor/distributor.rb
 - lib/job_reactor/logger.rb
+- lib/job_reactor/storages/memory_storage.rb
+- lib/job_reactor/storages/redis_storage.rb
+- lib/job_reactor/job_reactor/storages.rb
+- lib/job_reactor/job_reactor/config.rb
+- lib/job_reactor/job_reactor/job_parser.rb
+- lib/job_reactor/job_reactor/exceptions.rb
+- lib/job_reactor/job_reactor.rb
+- lib/job_reactor/node.rb
+- lib/job_reactor.rb
 - README.markdown
 homepage: http://github.com/antonmi/job_reactor
 licenses: []
@@ -77,7 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.6
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Simple, powerful and high scalable job queueing and background workers system