rocketjob 3.0.0.beta3 → 3.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2893cad7faa49cc61dc42a18e1769b7e4b0684cd
-  data.tar.gz: 0484cd6616b0d1dbef6ecd67d0235ef0d3834623
+  metadata.gz: 5e00bccf78b1addfe4e7714526085787e009c852
+  data.tar.gz: bdeb1d7b808c5b589421086254f7f301837d513c
 SHA512:
-  metadata.gz: 13e4e156847cfebb938b7700eaddda5899b79f816d690b3555d77aa6b993cc046e76c6b796404e83f3767ce482a4db50134a7037a6e7ba3ea9fd85b296834689
-  data.tar.gz: 60856f69ce93c2c0143cbf349f8c9eb393c038e44897d3f1ab16c550cec10557656628892d92ce6fbc833c5353f7b4a936a790932c219b9df7c8384cdd6f360d
+  metadata.gz: 0e1a31bdef5185bf7ba414d847d49037871a6b913e3eeda57b98f4a37101b93bd63de8f55c36cfa8ee4cf9fabf55486b01ac4d1ade391104067f45b06a31bb13
+  data.tar.gz: 13289e381b6f879d5da29f0b633316b1b4888dac9c1f5df1bd1048bf91acd122f64ce9837edf5ef631f2f4e8811dde3a008c43bb953557a3560eba2ee25c10d8

data/README.md

@@ -19,8 +19,22 @@ Checkout http://rocketjob.io/
 
 ## Upgrading to V3
 
-* V3 replaces MongoMapper with Mongoid which supports the latest MongoDB Ruby client driver.
-* Replase usages of `rocket_job do` to set default values:
+V3 replaces MongoMapper with Mongoid which supports the latest MongoDB Ruby client driver.
+
+### Upgrading Mongo Config file
+Replace `mongo.yml` with `mongoid.yml`.
+
+Start with the sample [mongoid.yml](https://github.com/rocketjob/rocketjob/blob/feature/mongoid/test/config/mongoid.yml).
+ 
+For more information on the new [Mongoid config file](https://docs.mongodb.com/ruby-driver/master/tutorials/5.1.0/mongoid-installation/).
+
+Note: The `rocketjob` and `rocketjob_slices` clients in the above `mongoid.yml` file are required.
+
+### Other changes
+
+* Arguments are no longer supported, use fields for defining all named arguments for a job.
+
+* Replace usages of `rocket_job do` to set default values:
 
 ~~~ruby
   rocket_job do |job|
@@ -52,12 +66,11 @@ With:
 field :priority, type: Integer, default: 50, user_editable: true
 ~~~
 
-
 ## Ruby Support
 
 Rocket Job is tested and supported on the following Ruby platforms:
-- Ruby 2.1, 2.2, 2.3, and above
-- JRuby 1.7.23, 9.0.5 and above
+- Ruby 2.1, 2.2, 2.3, 2.4, and above
+- JRuby 9.0.5 and above
 
 ## Versioning
 

data/lib/rocket_job/active_worker.rb

@@ -0,0 +1,74 @@
+module RocketJob
+  # Information about a server currently working on a job
+  class ActiveWorker
+    # When this server started working on this job / slice
+    attr_accessor :started_at
+
+    attr_accessor :name, :job
+
+    # Returns [Hash<String:ActiveWorker>] hash of all servers sorted by name
+    # and what they are currently working on.
+    # Returns {} if no servers are currently busy doing any work
+    #
+    # Parameters
+    #   server_name: [String]
+    #     Only jobs running on the specified server
+    def self.all(server_name = nil)
+      servers = []
+      # Need paused, failed or aborted since servers may still be working on active slices
+      query   = RocketJob::Job.where(:state.in => [:running, :paused, :failed, :aborted])
+      query = query.where(worker_name: /\A#{server_name}/) if server_name
+      query.each do |job|
+        servers += job.rocket_job_active_workers
+      end
+      servers
+    end
+
+    # Requeues all jobs for which the workers have disappeared
+    def self.requeue_zombies
+      all.each do |active_worker|
+        next if !active_worker.zombie? || !active_worker.job.may_requeue?
+        active_worker.job.requeue!(active_worker.server_name)
+      end
+    end
+
+    def initialize(name, started_at, job)
+      @name       = name
+      @started_at = started_at
+      @job        = job
+    end
+
+    # Duration in human readable form
+    def duration
+      RocketJob.seconds_as_duration(duration_s)
+    end
+
+    # Number of seconds this server has been working on this job / slice
+    def duration_s
+      Time.now - started_at
+    end
+
+    # Returns [String] the name of the server running this worker
+    def server_name
+      if match = name.match(/(.*:.*):.*/)
+        match[1]
+      else
+        name
+      end
+    end
+
+    def server
+      @server ||= RocketJob::Server.where(name: server_name).first
+    end
+
+    # The server on which this worker was running is no longer running
+    def zombie?
+      server.nil?
+    end
+
+    def name=(name)
+      @server = nil
+      @name   = name
+    end
+  end
+end

data/lib/rocket_job/performance.rb

@@ -14,17 +14,12 @@ module RocketJob
       @mongo_config = 'config/mongoid.yml'
     end
 
+    # Loads the queue with jobs to be processed once the queue is loaded.
+    # Retain the first and last job for timings, all others are destroyed on completion.
     def run_test_case(count = self.count)
       raise 'Please start servers before starting the performance test' if RocketJob::Server.where(:state.in => ['running', 'paused']).count == 0
 
-      self.servers = 0
-      self.workers = 0
-      RocketJob::Server.running.each do |server|
-        next if server.zombie?
-        self.servers += 1
-        self.workers += server.heartbeat.workers
-      end
-      puts "Running: #{workers} workers, distributed across #{servers} servers"
+      count_running_workers
 
       puts 'Waiting for workers to pause'
       RocketJob::Server.pause_all
@@ -54,6 +49,9 @@ module RocketJob
       end
 
       duration = last.reload.completed_at - first.reload.started_at
+      first.destroy
+      last.destroy
+
       {count: count, duration: duration, jobs_per_second: (count.to_f / duration).to_i}
     end
 
@@ -86,5 +84,16 @@ module RocketJob
       parser.parse! argv
     end
 
+    def count_running_workers
+      self.servers = 0
+      self.workers = 0
+      RocketJob::Server.running.each do |server|
+        next if server.zombie?
+        self.servers += 1
+        self.workers += server.heartbeat.workers
+      end
+      puts "Running: #{workers} workers, distributed across #{servers} servers"
+    end
+
   end
 end

data/lib/rocket_job/plugins/document.rb

@@ -76,7 +76,7 @@ module RocketJob
           apply_defaults
           self
         else
-          raise Mongoid::Error::DocumentNotFound, "Document match #{_id.inspect} does not exist in #{collection.name} collection"
+          raise(Mongoid::Errors::DocumentNotFound.new(self.class, id))
         end
       end
 

data/lib/rocket_job/plugins/job/state_machine.rb

@@ -77,7 +77,7 @@ module RocketJob
 
             event :requeue do
               transitions from: :running, to: :queued,
-                if:             -> _worker_name { worker_name == _worker_name },
+                if:             -> server_name { worker_name.start_with?(server_name) },
                 after:          :rocket_job_clear_started_at
             end
           end

data/lib/rocket_job/plugins/job/throttle.rb

@@ -43,7 +43,7 @@ module RocketJob
 
         # Throttle to add when the throttle is exceeded
         def throttle_filter
-          {:_type.nin => self.class.name}
+          {:_type.nin => [self.class.name]}
         end
 
         # Returns [Boolean] whether the throttle for this job has been exceeded
@@ -62,9 +62,13 @@ module RocketJob
 
         # Merge filter(s)
         def throttle_merge_filter(target, source)
-          source.each_key do |k, v|
-            previous  = target[k]
-            target[k] = previous ? (Array(previous) << v) : v
+          source.each_pair do |k, v|
+            target[k] =
+              if previous = target[k]
+                v.is_a?(Array) ? previous + v : v
+              else
+                v
+              end
           end
         end
 

data/lib/rocket_job/plugins/job/worker.rb

@@ -169,9 +169,9 @@ module RocketJob
         end
 
         # Returns [Hash<String:[Array<ActiveWorker>]>] All servers actively working on this job
-        def rocket_job_active_servers
-          return {} unless running?
-          {worker_name => [ActiveServer.new(worker_name, started_at, self)]}
+        def rocket_job_active_workers(server_name = nil)
+          return [] if !running? || (server_name && !worker_name.start_with(server_name))
+          [ActiveWorker.new(worker_name, started_at, self)]
         end
 
       end

data/lib/rocket_job/server.rb

@@ -266,25 +266,23 @@ module RocketJob
           workers.shift
         else
           # Timeout waiting for worker to stop
-          begin
-            find_and_update(
-              'heartbeat.updated_at' => Time.now,
-              'heartbeat.workers'    => worker_count
-            )
-          rescue Mongoid::Errors::DocumentNotFound
-            logger.warn('Server has been destroyed. Going down hard!')
-            break
-          end
+          find_and_update(
+            'heartbeat.updated_at' => Time.now,
+            'heartbeat.workers'    => worker_count
+          )
         end
       end
 
+      logger.info 'Shutdown'
+    rescue Mongoid::Errors::DocumentNotFound
+      logger.warn('Server has been destroyed. Going down hard!')
+    rescue Exception => exc
+      logger.error('RocketJob::Server is stopping due to an exception', exc)
+    ensure
       # Logs the backtrace for each running worker
       if SemanticLogger::VERSION.to_i >= 4
         workers.each { |thread| logger.backtrace(thread: thread) }
       end
-      logger.info 'Shutdown'
-    rescue Exception => exc
-      logger.error('RocketJob::Server is stopping due to an exception', exc)
     end
 
     # Returns [Fixnum] number of workers (threads) that are alive

data/lib/rocket_job/version.rb

@@ -1,4 +1,4 @@
 # encoding: UTF-8
 module RocketJob #:nodoc
-  VERSION = '3.0.0.beta3'
+  VERSION = '3.0.0.rc1'
 end

data/lib/rocket_job/worker.rb

@@ -99,7 +99,7 @@ module RocketJob
     def process_available_jobs
       # Only clear out the current_filter after every `re_check_seconds`
       time = Time.now
-      if (time - @re_check_start) > re_check_seconds
+      if (time - @re_check_start) > re_check_seconds.to_f
         @recheck_start      = time
         self.current_filter = filter.dup
       end

data/lib/rocketjob.rb

@@ -6,7 +6,7 @@ require 'rocket_job/version'
 
 # @formatter:off
 module RocketJob
-  autoload :ActiveServer,       'rocket_job/active_server'
+  autoload :ActiveWorker,       'rocket_job/active_worker'
   autoload :CLI,                'rocket_job/cli'
   autoload :Config,             'rocket_job/config'
   autoload :DirmonEntry,        'rocket_job/dirmon_entry'

data/test/plugins/job/worker_test.rb

@@ -171,28 +171,26 @@ module Plugins
           end
         end
 
-        describe '#rocket_job_active_servers' do
+        describe '#rocket_job_active_workers' do
           before do
             @job    = QuietJob.create!
             @worker = RocketJob::Server.create!(name: 'worker:123')
           end
 
           it 'should return empty hash for no active jobs' do
-            assert_equal({}, @job.rocket_job_active_servers)
+            assert_equal([], @job.rocket_job_active_workers)
           end
 
           it 'should return active servers' do
             assert job = RocketJob::Job.rocket_job_next_job(@worker.name)
-            assert active = job.rocket_job_active_servers
+            assert active = job.rocket_job_active_workers
             assert_equal 1, active.size
-            assert active_servers = active[@worker.name]
-            assert_equal 1, active_servers.size
-            assert active_server = active_servers.first
-            assert_equal @job.id, active_server.job.id
-            assert_equal @worker.name, active_server.name
-            assert_equal job.started_at, active_server.started_at
-            assert active_server.duration_s
-            assert active_server.duration
+            assert active_worker = active.first
+            assert_equal @job.id, active_worker.job.id
+            assert_equal @worker.name, active_worker.name
+            assert_equal job.started_at, active_worker.started_at
+            assert active_worker.duration_s
+            assert active_worker.duration
           end
         end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 3.0.0.beta3
+  version: 3.0.0.rc1
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-16 00:00:00.000000000 Z
+date: 2016-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -80,7 +80,7 @@ files:
 - Rakefile
 - bin/rocketjob
 - bin/rocketjob_perf
-- lib/rocket_job/active_server.rb
+- lib/rocket_job/active_worker.rb
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
 - lib/rocket_job/dirmon_entry.rb

data/lib/rocket_job/active_server.rb

@@ -1,48 +0,0 @@
-module RocketJob
-  # Information about a server currently working on a job
-  class ActiveServer
-    # When this server started working on this job / slice
-    attr_accessor :started_at
-
-    attr_accessor :name, :job
-
-    # Returns [Hash<String:ActiveWorker>] hash of all servers sorted by name
-    # and what they are currently working on.
-    # Returns {} if no servers are currently busy doing any work
-    def self.all
-      servers = {}
-      # Need paused, failed or aborted since servers may still be working on active slices
-      RocketJob::Job.where(state: [:running, :paused, :failed, :aborted]).each do |job|
-        job.rocket_job_active_servers.each_pair do |name, active_server|
-          (servers[name] ||= []) << active_server
-        end
-      end
-      servers
-    end
-
-    def initialize(name, started_at, job)
-      @name       = name
-      @started_at = started_at
-      @job        = job
-    end
-
-    # Duration in human readable form
-    def duration
-      RocketJob.seconds_as_duration(duration_s)
-    end
-
-    # Number of seconds this server has been working on this job / slice
-    def duration_s
-      Time.now - started_at
-    end
-
-    def server
-      @server ||= RocketJob::Server.find_by(name: name)
-    end
-
-    def name=(name)
-      @server = nil
-      @name   = name
-    end
-  end
-end