rocketjob 5.1.1 → 5.2.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1eaf1de774f144f26c22ce60a7a6051f1300a26dba14d76ba93e4eee705a238d
-  data.tar.gz: 15a2325ebbc517977dfae5ed5695d574ac5941578b4565c9d6e468f84f5523ae
+  metadata.gz: 8920ff4a319e838729863bbf2a7a66a4ee003ac9328cc1c3d3942fc59cecab2c
+  data.tar.gz: 3c5f4927370915b3188295cd181f5b1dc4ae64a17b0cd070e946cfa82b7517c6
 SHA512:
-  metadata.gz: ceb14a804ad87ae2e3b0c94414261efbf68e6db606b0ad1251b07820d715640439cf230f32ab836f2d40049fc74662761d8ca260c2b2435047675d2bd0575198
-  data.tar.gz: 88f29b1f69867c819f83da5e8e3a633662355f9ee4e6e0c749908b8939db08db7bc1fa4b3751a25294c5bbd75576045a739c0d3ac6416d5143d2d9d1ee081be7
+  metadata.gz: 1395412d271a4a1cde3d1f80fa6d45d5a0e948454b34e6a1d1221631b718d1f4fb16e3683a4a57af1e1f64d724219b70ab65cd84f7a2c9658d67e0608d263973
+  data.tar.gz: f5b161cf5dcb32126dff42b645b3356707ea74623f011af9a6e7bf0b8fee6403254b302ec5644a85181c207dbe70cfa3a82349be4a2bf52127e68e9f4ebaf5ab

data/bin/rocketjob

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
-require 'semantic_logger'
-require 'rocket_job/cli'
+require "semantic_logger"
+require "rocket_job/cli"
 
 # Start a rocketjob server instance from the command line
 # Allow Ruby to catch and display any startup exceptions

data/bin/rocketjob_batch_perf

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-require 'rocketjob_batch'
+require "rocketjob_batch"
 
 # Log to console
 SemanticLogger.add_appender(io: STDOUT, formatter: :color)

data/bin/rocketjob_perf

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-require 'rocketjob'
+require "rocketjob"
 
 # Log to console
 SemanticLogger.add_appender(io: STDOUT, formatter: :color)

data/lib/rocket_job/active_worker.rb

@@ -29,6 +29,7 @@ module RocketJob
     def self.requeue_zombies
       all.each do |active_worker|
         next if !active_worker.zombie? || !active_worker.job.may_requeue?(active_worker.server_name)
+
         active_worker.job.requeue!(active_worker.server_name)
       end
     end

data/lib/rocket_job/batch.rb

@@ -1,12 +1,12 @@
-require 'active_support/concern'
-require 'rocket_job/batch/callbacks'
-require 'rocket_job/batch/io'
-require 'rocket_job/batch/logger'
-require 'rocket_job/batch/model'
-require 'rocket_job/batch/state_machine'
-require 'rocket_job/batch/throttle'
-require 'rocket_job/batch/throttle_running_slices'
-require 'rocket_job/batch/worker'
+require "active_support/concern"
+require "rocket_job/batch/callbacks"
+require "rocket_job/batch/io"
+require "rocket_job/batch/logger"
+require "rocket_job/batch/model"
+require "rocket_job/batch/state_machine"
+require "rocket_job/batch/throttle"
+require "rocket_job/batch/throttle_running_workers"
+require "rocket_job/batch/worker"
 
 module RocketJob
   module Batch
@@ -18,15 +18,14 @@ module RocketJob
     include Logger
     include Worker
     include Throttle
-    include ThrottleRunningSlices
+    include ThrottleRunningWorkers
     include IO
 
-    autoload :LowerPriority, 'rocket_job/batch/lower_priority'
-    autoload :Performance, 'rocket_job/batch/performance'
-    autoload :Statistics, 'rocket_job/batch/statistics'
-    autoload :Result, 'rocket_job/batch/result'
-    autoload :Results, 'rocket_job/batch/results'
-    autoload :Tabular, 'rocket_job/batch/tabular'
+    autoload :LowerPriority, "rocket_job/batch/lower_priority"
+    autoload :Performance, "rocket_job/batch/performance"
+    autoload :Statistics, "rocket_job/batch/statistics"
+    autoload :Result, "rocket_job/batch/result"
+    autoload :Results, "rocket_job/batch/results"
+    autoload :Tabular, "rocket_job/batch/tabular"
   end
 end
-

data/lib/rocket_job/batch/callbacks.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -34,7 +34,6 @@ module RocketJob
           set_callback(:after_batch, :after, *filters, &blk)
         end
       end
-
     end
   end
 end

data/lib/rocket_job/batch/io.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -14,7 +14,9 @@ module RocketJob
       #     Default: None ( Uses the single default input collection for this job )
       #     Validates: This value must be one of those listed in #input_categories
       def input(category = :main)
-        raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}" unless input_categories.include?(category) || (category == :main)
+        unless input_categories.include?(category) || (category == :main)
+          raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}"
+        end
 
         (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(rocket_job_io_slice_arguments("inputs", category))
       end
@@ -28,7 +30,9 @@ module RocketJob
       #     Default: None ( Uses the single default output collection for this job )
       #     Validates: This value must be one of those listed in #output_categories
       def output(category = :main)
-        raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}" unless output_categories.include?(category) || (category == :main)
+        unless output_categories.include?(category) || (category == :main)
+          raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}"
+        end
 
         (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(rocket_job_io_slice_arguments("outputs", category))
       end
@@ -110,7 +114,7 @@ module RocketJob
       # * Only use this method for UTF-8 data, for binary data use #input_slice or #input_records.
       # * CSV parsing is slow, so it is usually left for the workers to do.
       def upload(stream = nil, file_name: nil, category: :main, stream_mode: :line, on_first: nil, **args, &block)
-        raise(ArgumentError, 'Either stream, or a block must be supplied') unless stream || block
+        raise(ArgumentError, "Either stream, or a block must be supplied") unless stream || block
 
         stream_mode = stream_mode.to_sym
         # Backward compatibility with existing v4 jobs
@@ -384,7 +388,8 @@ module RocketJob
         if block
           RocketJob::Sliced::Writer::Output.collect(self, input_slice, &block)
         else
-          raise(ArgumentError, 'result parameter is required when no block is supplied') unless result
+          raise(ArgumentError, "result parameter is required when no block is supplied") unless result
+
           RocketJob::Sliced::Writer::Output.collect(self, input_slice) { |writer| writer << result }
         end
       end
@@ -403,7 +408,6 @@ module RocketJob
         end
         args
       end
-
     end
   end
 end

data/lib/rocket_job/batch/logger.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -31,7 +31,7 @@ module RocketJob
       #   - silence noisy jobs by raising log level
       def rocket_job_batch_slice_logger(&block)
         logger.measure_info(
-          'Completed slice',
+          "Completed slice",
           metric:             "#{self.class.name}/slice",
           log_exception:      :full,
           on_exception_level: :error,

data/lib/rocket_job/batch/lower_priority.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 module RocketJob
   module Batch
     # Automatically lower the priority for Jobs with a higher record_count.
@@ -33,7 +33,7 @@ module RocketJob
 
       included do
         unless public_method_defined?(:record_count=)
-          raise(ArgumentError, 'LowerPriority can only be used in conjunction with RocketJob::Plugins::Batch')
+          raise(ArgumentError, "LowerPriority can only be used in conjunction with RocketJob::Plugins::Batch")
         end
 
         # For each of this many records lower the priority by 1.

data/lib/rocket_job/batch/model.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 module RocketJob
   module Batch
     # Model attributes
@@ -77,8 +77,10 @@ module RocketJob
           # Under some circumstances ActiveModel is passing in a nil value even though the
           # attributes have default values
           Array(value).each do |category|
-            record.errors.add(attr, 'must only contain Symbol values') unless category.kind_of?(Symbol)
-            record.errors.add(attr, 'must only consist of lowercase characters, digits, and _') unless category.to_s =~ /\A[a-z_0-9]+\Z/
+            record.errors.add(attr, "must only contain Symbol values") unless category.is_a?(Symbol)
+            unless category.to_s =~ /\A[a-z_0-9]+\Z/
+              record.errors.add(attr, "must only consist of lowercase characters, digits, and _")
+            end
           end
         end
       end
@@ -97,7 +99,7 @@ module RocketJob
       # Returns 0 if the total record count has not yet been set
       def percent_complete
         return 100 if completed?
-        return 0 unless record_count.to_i > 0
+        return 0 unless record_count.to_i.positive?
 
         # Approximate number of input records
         input_records = input.count.to_f * slice_size
@@ -110,32 +112,31 @@ module RocketJob
       end
 
       # Returns [Hash] status of this job
-      def status(time_zone = 'Eastern Time (US & Canada)')
+      def status(time_zone = "Eastern Time (US & Canada)")
         h = {}
-        case
-        when queued?
-          h['queued_slices'] = input.queued.count
-        when running? || paused? || failed?
-          h['active_slices'] = worker_count
-          h['failed_slices'] = input.failed.count
-          h['queued_slices'] = input.queued.count
+        if queued?
+          h["queued_slices"] = input.queued.count
+        elsif running? || paused? || failed?
+          h["active_slices"] = worker_count
+          h["failed_slices"] = input.failed.count
+          h["queued_slices"] = input.queued.count
           # Very high level estimated time left
-          if record_count && running? && (record_count > 0)
+          if record_count && running? && record_count.positive?
             percent = percent_complete
             if percent >= 5
               secs                        = seconds.to_f
-              h['est_remaining_duration'] = RocketJob.seconds_as_duration((((secs / percent) * 100) - secs))
+              h["est_remaining_duration"] = RocketJob.seconds_as_duration((((secs / percent) * 100) - secs))
             end
           end
-        when completed?
-          secs                  = seconds.to_f
-          h['records_per_hour'] = ((record_count.to_f / secs) * 60 * 60).round if record_count && (record_count > 0) && (secs > 0.0)
+        elsif completed?
+          secs = seconds.to_f
+          h["records_per_hour"] = ((record_count.to_f / secs) * 60 * 60).round if record_count&.positive? && (secs > 0.0)
         end
-        h['output_slices'] = output.count if collect_output? && !completed?
+        h["output_slices"] = output.count if collect_output? && !completed?
         h.merge!(super(time_zone))
-        h.delete('result')
+        h.delete("result")
         # Worker name should be retrieved from the slices when processing
-        h.delete('worker_name') if sub_state == :processing
+        h.delete("worker_name") if sub_state == :processing
         h
       end
 
@@ -145,9 +146,9 @@ module RocketJob
 
         case sub_state
         when :before, :after
-          worker_name
+          [worker_name]
         when :processing
-          input.running.collect { |slice| slice.worker_name }
+          input.running.collect(&:worker_name)
         else
           []
         end
@@ -171,7 +172,6 @@ module RocketJob
         @worker_count_last = Time.now.to_i
         @worker_count
       end
-
     end
   end
 end

data/lib/rocket_job/batch/performance.rb

@@ -1,6 +1,6 @@
-require 'optparse'
-require 'csv'
-require 'yaml'
+require "optparse"
+require "csv"
+require "yaml"
 module RocketJob
   module Batch
     class Performance
@@ -8,8 +8,8 @@ module RocketJob
 
       def initialize
         @count        = 10_000_000
-        @environment  = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
-        @mongo_config = 'config/mongoid.yml'
+        @environment  = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+        @mongo_config = "config/mongoid.yml"
         @compress     = false
         @encrypt      = false
         @slice_size   = 1000
@@ -17,7 +17,7 @@ module RocketJob
 
       def run_test_case(count = self.count)
         servers = RocketJob::Server.count
-        raise 'Please start workers before starting the performance test' if servers == 0
+        raise "Please start workers before starting the performance test" if servers.zero?
 
         count_running_workers
 
@@ -33,10 +33,8 @@ module RocketJob
         end
         job.save!
 
-        puts 'Waiting for job to complete'
-        while (!job.reload.completed?)
-          sleep 3
-        end
+        puts "Waiting for job to complete"
+        sleep 3 until job.reload.completed?
 
         duration = job.completed_at - job.started_at
         {count: count, duration: duration, records_per_second: (count.to_f / duration).round(3), workers: workers, servers: servers, compress: compress, encrypt: encrypt}
@@ -47,7 +45,7 @@ module RocketJob
         ruby    = defined?(JRuby) ? "jruby_#{JRUBY_VERSION}" : "ruby_#{RUBY_VERSION}"
         version = RocketJob::VERSION
 
-        CSV.open("job_results_#{ruby}_v#{version}.csv", 'wb') do |csv|
+        CSV.open("job_results_#{ruby}_v#{version}.csv", "wb") do |csv|
           csv << results.first.keys
           results.each { |result| csv << result.values }
         end
@@ -55,28 +53,28 @@ module RocketJob
 
       # Parse command line options
       def parse(argv)
-        parser        = OptionParser.new do |o|
-          o.on('-c', '--count COUNT', 'Count of records to enqueue') do |arg|
+        parser = OptionParser.new do |o|
+          o.on("-c", "--count COUNT", "Count of records to enqueue") do |arg|
             self.count = arg.to_i
           end
-          o.on('-m', '--mongo MONGO_CONFIG_FILE_NAME', 'Location of mongoid.yml config file') do |arg|
+          o.on("-m", "--mongo MONGO_CONFIG_FILE_NAME", "Location of mongoid.yml config file") do |arg|
             self.mongo_config = arg
           end
-          o.on('-e', '--environment ENVIRONMENT', 'The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)') do |arg|
+          o.on("-e", "--environment ENVIRONMENT", "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
             self.environment = arg
           end
-          o.on('-z', '--compress', 'Turn on compression') do |arg|
+          o.on("-z", "--compress", "Turn on compression") do
             self.compress = true
           end
-          o.on('-E', '--encrypt', 'Turn on encryption') do |arg|
+          o.on("-E", "--encrypt", "Turn on encryption") do
             self.encrypt = true
           end
-          o.on('-s', '--slice_size COUNT', 'Slice size') do |arg|
+          o.on("-s", "--slice_size COUNT", "Slice size") do
             self.slice_size = arg.to_i
           end
         end
-        parser.banner = 'rocketjob_batch_perf <options>'
-        parser.on_tail '-h', '--help', 'Show help' do
+        parser.banner = "rocketjob_batch_perf <options>"
+        parser.on_tail "-h", "--help", "Show help" do
           puts parser
           exit 1
         end
@@ -88,12 +86,12 @@ module RocketJob
         self.workers = 0
         RocketJob::Server.running.each do |server|
           next if server.zombie?
+
           self.servers += 1
           self.workers += server.heartbeat.workers
         end
         puts "Running: #{workers} workers, distributed across #{servers} servers"
       end
-
     end
   end
 end

data/lib/rocket_job/batch/result.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch

data/lib/rocket_job/batch/results.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch

data/lib/rocket_job/batch/state_machine.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -23,15 +23,15 @@ module RocketJob
         aasm.state_machine.add_event(:requeue, {}) do
           # Requeue perform
           transitions from:  :running, to: :running,
-                      if:    -> server_name { sub_state == :processing },
+                      if:    ->(_server_name) { sub_state == :processing },
                       after: :rocket_job_requeue_sub_state_processing
           # Requeue after_batch
           transitions from:  :running, to: :running,
-                      if:    -> server_name { worker_on_server?(server_name) && (sub_state == :after) },
+                      if:    ->(server_name) { worker_on_server?(server_name) && (sub_state == :after) },
                       after: :rocket_job_requeue_sub_state_after
           # Requeue before_batch
           transitions from:  :running, to: :queued,
-                      if:    -> server_name { worker_on_server?(server_name) && (sub_state == :before) },
+                      if:    ->(server_name) { worker_on_server?(server_name) && (sub_state == :before) },
                       after: :rocket_job_requeue_sub_state_before
         end
 
@@ -68,7 +68,7 @@ module RocketJob
       end
 
       def rocket_job_sub_state_before
-        self.sub_state = :before unless self.sub_state
+        self.sub_state = :before unless sub_state
       end
 
       def rocket_job_clear_sub_state
@@ -96,7 +96,6 @@ module RocketJob
       def rocket_job_requeue_failed_slices
         input.requeue_failed
       end
-
     end
   end
 end

data/lib/rocket_job/batch/statistics.rb

@@ -1,4 +1,4 @@
-require 'active_support/concern'
+require "active_support/concern"
 
 module RocketJob
   module Batch
@@ -22,11 +22,12 @@ module RocketJob
         end
 
         def inc_key(key, increment = 1)
-          return if increment == 0
+          return if increment.zero?
+
           if in_memory
             # For tests and in-process execution
             inc_in_memory(key, increment)
-          elsif key && key != ''
+          elsif key && key != ""
             stats["statistics.#{key}"] += increment
           end
           self
@@ -40,11 +41,11 @@ module RocketJob
 
         # Navigates path and creates child hashes as needed at the end is reached
         def inc_in_memory(key, increment)
-          paths = key.to_s.split('.')
+          paths = key.to_s.split(".")
           last  = paths.pop
           return unless last
 
-          target       = paths.inject(in_memory) { |target, key| target.key?(key) ? target[key] : target[key] = Hash.new(0) }
+          target = paths.inject(in_memory) { |target, key| target.key?(key) ? target[key] : target[key] = Hash.new(0) }
           target[last] += increment
         end
       end
@@ -57,7 +58,8 @@ module RocketJob
 
       # Increment a statistic
       def statistics_inc(key, increment = 1)
-        return if key.nil? || key == ''
+        return if key.nil? || key == ""
+
         # Being called within tests outside of a perform
         @slice_statistics ||= Stats.new(new_record? ? statistics : nil)
         key.is_a?(Hash) ? @slice_statistics.inc(key) : @slice_statistics.inc_key(key, increment)
@@ -70,12 +72,12 @@ module RocketJob
       def statistics_capture
         @slice_statistics = Stats.new(new_record? ? statistics : nil)
         yield
-        collection.update_one({_id: id}, {'$inc' => @slice_statistics.stats}) unless @slice_statistics.empty?
+        collection.update_one({_id: id}, {"$inc" => @slice_statistics.stats}) unless @slice_statistics.empty?
       end
 
       # Overrides RocketJob::Batch::Logger#rocket_job_batch_log_payload
       def rocket_job_batch_log_payload
-        h              = {
+        h = {
           from:  aasm.from_state,
           to:    aasm.to_state,
           event: aasm.current_event