rocketjob 3.5.2 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd3505de7a318efb53880ef4c0f506a970d1ee30277f531bef7e2a9f257cf500
-  data.tar.gz: '085e0371c60662b51ba53c4e5e6e1b2e016cdc2e37de1b51a437e5477f907a22'
+  metadata.gz: c3d1d8a5bd37991ac8ad2dd5ae28b833dc4cb59d69da1a1cc281c904ab15c3eb
+  data.tar.gz: 39336f01d701f34e4f25e00e03fa5f06299ec19d1f7c0b52af5aff6ae4b4327b
 SHA512:
-  metadata.gz: a9a9441957a5405ed1d3d98bbe5bf0a45febbbdf8283c09de967f19abd1e5a7470cc871d66b804f8569ad90b48732fcd4d649e3d0c282b60be2b000bdd60adc8
-  data.tar.gz: d47b9c8a4eba0fd88798b76926da8c4073b0c26c6743ae3b54883535f278ea3317d6fb7901d57caf8506f710b5fdb7151b9acae65e7e2427bf2607a0c926231a
+  metadata.gz: 42266518d00516c62ccd3d0139e6fa22427f48b14454a2f06f0c04f559620b9bbf71700f38b518db03b8290cd6ab7ad7353c47cfbb579aa2f5642ef2a9b2cfb5
+  data.tar.gz: 444e848667f4a09629b18b467713fe9ae26abb22fd68ceb9048e0adb617c3bbe066ec407c9922577f9ea6bc40271a9d6c84d6064456e5e0532ddfa8f0be85a69

data/README.md

@@ -17,6 +17,59 @@ Checkout http://rocketjob.io/
 * Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
 * [Report bugs](https://github.com/rocketjob/rocketjob/issues)
 
+## Rocket Job 4
+
+Rocket Job Pro is now open sourced and included within Rocket Job. 
+
+The `RocketJob::Batch` plugin now adds batch processing capabilites to break up a single task into many
+concurrent workers processing slices of the entire job at the same time. 
+
+
+Example:
+
+```ruby
+class MyJob < RocketJob::Job
+  include RocketJob::Batch
+  
+  self.description         = "Reverse names"
+  self.destroy_on_complete = false
+  self.collect_output      = true
+
+  # Method to call by all available workers at the same time.
+  # Reverse the characters for each line: 
+  def perform(line)
+    line.reverse
+  end
+end
+```
+
+Upload a file for processing, for example `names.csv` which could contain:
+
+```
+jack
+jane
+bill
+john
+blake
+chris
+dave
+marc
+```
+
+To queue the above job for processing:
+
+```ruby
+job = MyJob.new
+job.upload('names.csv')
+job.save!
+```
+
+Once the job has completed, download the results into a file:
+
+```ruby
+job.download('names_reversed.csv')
+```
+
 ## Contributing to the documentation
 
 To contribute to the documentation it is as easy as forking the repository
@@ -99,6 +152,14 @@ Rocket Job is tested and supported on the following Ruby platforms:
 - Ruby 2.1, 2.2, 2.3, 2.4, and above
 - JRuby 9.0.5 and above
 
+## Dependencies
+
+* [MongoDB](https://www.mongodb.org)
+    * Persists job information.
+    * Version 2.7 or greater.
+* [Semantic Logger](https://rocketjob.github.io/semantic_logger)
+    * Highly concurrent scalable logging.
+
 ## Versioning
 
 This project uses [Semantic Versioning](http://semver.org/).
@@ -109,4 +170,5 @@ This project uses [Semantic Versioning](http://semver.org/).
 
 ## Contributors
 
-* [Chris Lamb](https://github.com/lambcr)
+[Contributors](https://github.com/rocketjob/rocketjob/graphs/contributors)
+

data/bin/rocketjob

@@ -7,6 +7,7 @@ require 'rocket_job/cli'
 begin
   RocketJob::CLI.new(ARGV).run
 rescue Exception => exc
+  return if exc.class == SystemExit
   # Failsafe logger that writes to STDERR
   SemanticLogger.add_appender(io: STDERR, level: :error, formatter: :color)
   SemanticLogger['RocketJob'].error('Rocket Job shutting down due to exception', exc)

data/bin/rocketjob_batch_perf

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+require 'rocketjob_batch'
+
+# Log to console
+SemanticLogger.add_appender(io: STDOUT, formatter: :color)
+
+perf = RocketJob::Batch::Performance.new
+perf.parse(ARGV)
+RocketJob::Config.load!(perf.environment, perf.mongo_config)
+results = perf.run_test_case
+p results

data/lib/rocket_job/batch.rb

@@ -0,0 +1,32 @@
+require 'active_support/concern'
+require 'rocket_job/batch/callbacks'
+require 'rocket_job/batch/io'
+require 'rocket_job/batch/logger'
+require 'rocket_job/batch/model'
+require 'rocket_job/batch/state_machine'
+require 'rocket_job/batch/throttle'
+require 'rocket_job/batch/throttle_running_slices'
+require 'rocket_job/batch/worker'
+
+module RocketJob
+  module Batch
+    extend ActiveSupport::Concern
+
+    include Model
+    include StateMachine
+    include Callbacks
+    include Logger
+    include Worker
+    include Throttle
+    include ThrottleRunningSlices
+    include IO
+
+    autoload :LowerPriority, 'rocket_job/batch/lower_priority'
+    autoload :Performance, 'rocket_job/batch/performance'
+    autoload :Statistics, 'rocket_job/batch/statistics'
+    autoload :Result, 'rocket_job/batch/result'
+    autoload :Results, 'rocket_job/batch/results'
+    autoload :Tabular, 'rocket_job/batch/tabular'
+  end
+end
+

data/lib/rocket_job/batch/callbacks.rb

@@ -0,0 +1,40 @@
+require 'active_support/concern'
+
+module RocketJob
+  module Batch
+    module Callbacks
+      extend ActiveSupport::Concern
+      include ActiveSupport::Callbacks
+
+      included do
+        define_callbacks :slice
+
+        def self.before_slice(*filters, &blk)
+          set_callback(:slice, :before, *filters, &blk)
+        end
+
+        def self.after_slice(*filters, &blk)
+          set_callback(:slice, :after, *filters, &blk)
+        end
+
+        def self.around_slice(*filters, &blk)
+          set_callback(:slice, :around, *filters, &blk)
+        end
+
+        # before_batch and after_batch are called asynchronously.
+        # around_batch is not supported.
+        define_callbacks :before_batch
+        define_callbacks :after_batch
+
+        def self.before_batch(*filters, &blk)
+          set_callback(:before_batch, :before, *filters, &blk)
+        end
+
+        def self.after_batch(*filters, &blk)
+          set_callback(:after_batch, :after, *filters, &blk)
+        end
+      end
+
+    end
+  end
+end

data/lib/rocket_job/batch/io.rb

@@ -0,0 +1,154 @@
+require 'active_support/concern'
+
+module RocketJob
+  module Batch
+    # IO methods for sliced jobs
+    module IO
+      extend ActiveSupport::Concern
+
+      # Returns [RocketJob::Sliced::Input] input collection for holding input slices
+      #
+      # Parameters:
+      #   category [Symbol]
+      #     The name of the category to access or upload data into
+      #     Default: None ( Uses the single default input collection for this job )
+      #     Validates: This value must be one of those listed in #input_categories
+      def input(category = :main)
+        raise "Category #{category.inspect}, must be registered in input_categories: #{input_categories.inspect}" unless input_categories.include?(category) || (category == :main)
+
+        collection_name = "rocket_job.inputs.#{id}"
+        collection_name << ".#{category}" unless category == :main
+
+        (@inputs ||= {})[category] ||= RocketJob::Sliced::Input.new(slice_arguments(collection_name))
+      end
+
+      # Returns [RocketJob::Sliced::Output] output collection for holding output slices
+      # Returns nil if no output is being collected
+      #
+      # Parameters:
+      #   category [Symbol]
+      #     The name of the category to access or download data from
+      #     Default: None ( Uses the single default output collection for this job )
+      #     Validates: This value must be one of those listed in #output_categories
+      def output(category = :main)
+        raise "Category #{category.inspect}, must be registered in output_categories: #{output_categories.inspect}" unless output_categories.include?(category) || (category == :main)
+
+        collection_name = "rocket_job.outputs.#{id}"
+        collection_name << ".#{category}" unless category == :main
+
+        (@outputs ||= {})[category] ||= RocketJob::Sliced::Output.new(slice_arguments(collection_name))
+      end
+
+      # Upload the supplied file_name or stream
+      #
+      # Updates the record_count after adding the records
+      #
+      # Options
+      #     :file_name [String]
+      #       When file_name_or_io is an IO, the original base file name if any.
+      #       Default: nil
+      #
+      # See RocketJob::Sliced::Input#upload for remaining options
+      #
+      # Returns [Integer] the number of records uploaded
+      #
+      # Note:
+      # * Not thread-safe. Only call from one thread at a time
+      def upload(file_name_or_io = nil, file_name: nil, category: :main, **args, &block)
+        if file_name
+          self.upload_file_name = file_name
+        elsif file_name_or_io.is_a?(String)
+          self.upload_file_name = file_name_or_io
+        end
+        count             = input(category).upload(file_name_or_io, file_name: file_name, **args, &block)
+        self.record_count = (record_count || 0) + count
+        count
+      end
+
+      # Upload the supplied slices for processing by workers
+      #
+      # Updates the record_count after adding the records
+      #
+      # Returns [Integer] the number of records uploaded
+      #
+      # Parameters
+      #   `slice` [ Array<Hash | Array | String | Integer | Float | Symbol | Regexp | Time> ]
+      #     All elements in `array` must be serializable to BSON
+      #     For example the following types are not supported: Date
+      #
+      # Note:
+      #   The caller should honor `:slice_size`, the entire slice is loaded as-is.
+      #
+      # Note:
+      #   Not thread-safe. Only call from one thread at a time
+      def upload_slice(slice)
+        input.insert(slice)
+        count             = slice.size
+        self.record_count = (record_count || 0) + count
+        count
+      end
+
+      # Download the output data into the supplied file_name or stream
+      #
+      # Parameters
+      #   file_name_or_io [String|IO]
+      #     The file_name of the file to write to, or an IO Stream that implements #write.
+      #
+      #   options:
+      #     category [Symbol]
+      #       The category of output to download
+      #       Default: :main
+      #
+      # See RocketJob::Sliced::Output#download for remaining options
+      #
+      # Returns [Integer] the number of records downloaded
+      def download(file_name_or_io = nil, category: :main, **args, &block)
+        raise "Cannot download incomplete job: #{id}. Currently in state: #{state}-#{sub_state}" if rocket_job_processing?
+
+        output(category).download(file_name_or_io, **args, &block)
+      end
+
+      # Writes the supplied result, Batch::Result or Batch::Results to the relevant collections.
+      #
+      # If a block is supplied, the block is supplied with a writer that should be used to
+      # accumulate the results.
+      #
+      # Examples
+      #
+      # job.write_output('hello world')
+      #
+      # job.write_output do |writer|
+      #   writer << 'hello world'
+      # end
+      #
+      # job.write_output do |writer|
+      #   result = RocketJob::Batch::Results
+      #   result << RocketJob::Batch::Result.new(:main, 'hello world')
+      #   result << RocketJob::Batch::Result.new(:errors, 'errors')
+      #   writer << result
+      # end
+      #
+      # result = RocketJob::Batch::Results
+      # result << RocketJob::Batch::Result.new(:main, 'hello world')
+      # result << RocketJob::Batch::Result.new(:errors, 'errors')
+      # job.write_output(result)
+      def write_output(result = nil, input_slice = nil, &block)
+        if block
+          RocketJob::Sliced::Writer::Output.collect(self, input_slice, &block)
+        else
+          raise(ArgumentError, 'result parameter is required when no block is supplied') unless result
+          RocketJob::Sliced::Writer::Output.collect(self, input_slice) { |writer| writer << result }
+        end
+      end
+
+      private
+
+      def slice_arguments(collection_name)
+        {
+          collection_name: collection_name,
+          slice_size:      slice_size
+        }
+      end
+    end
+  end
+end

data/lib/rocket_job/batch/logger.rb

@@ -0,0 +1,57 @@
+require 'active_support/concern'
+
+module RocketJob
+  module Batch
+    module Logger
+      extend ActiveSupport::Concern
+
+      included do
+        # Log all state transitions
+        after_start :rocket_job_batch_log_state_change
+        after_complete :rocket_job_batch_log_state_change
+        after_fail :rocket_job_batch_log_state_change
+        after_retry :rocket_job_batch_log_state_change
+        after_pause :rocket_job_batch_log_state_change
+        after_resume :rocket_job_batch_log_state_change
+        after_abort :rocket_job_batch_log_state_change
+        after_requeue :rocket_job_batch_log_state_change
+
+        around_slice :rocket_job_batch_slice_logger
+
+        # Remove perform level logger and replace with slice level logger
+        skip_callback(:perform, :around, :rocket_job_around_logger)
+      end
+
+      private
+
+      # Add logging around processing of each slice
+      #   - metric allows duration to be forwarded to statsd, etc.
+      #   - log_exception logs entire exception if raised
+      #   - on_exception_level changes log level from info to error on exception
+      #   - silence noisy jobs by raising log level
+      def rocket_job_batch_slice_logger(&block)
+        logger.measure_info(
+          'Completed slice',
+          metric:             "#{self.class.name}/slice",
+          log_exception:      :full,
+          on_exception_level: :error,
+          silence:            log_level,
+          payload:            {records: rocket_job_slice&.size},
+          &block
+        )
+      end
+
+      def rocket_job_batch_log_state_change
+        logger.info(aasm.current_event.to_s.camelcase, rocket_job_batch_log_payload)
+      end
+
+      def rocket_job_batch_log_payload
+        {
+          from:  aasm.from_state,
+          to:    aasm.to_state,
+          event: aasm.current_event
+        }
+      end
+    end
+  end
+end

data/lib/rocket_job/batch/lower_priority.rb

@@ -0,0 +1,54 @@
+require 'active_support/concern'
+module RocketJob
+  module Batch
+    # Automatically lower the priority for Jobs with a higher record_count.
+    #
+    # Note:
+    #   - Add `:lower_priority` as a before_batch, but only once the `record_count` has been set.
+    #   - If the `record_count` is not set by the time this plugins `before_batch`
+    #     is called, then the priority will not be modified.
+    #
+    # class SampleJob < RocketJob::Job
+    #   include RocketJob::Plugins::Batch
+    #   include RocketJob::Plugins::Batch::LowerPriority
+    #
+    #   before_batch :upload_data, :lower_priority
+    #
+    #   def perform(record)
+    #     record.reverse
+    #   end
+    #
+    #   private
+    #
+    #   def upload_data
+    #     upload do |stream|
+    #       stream << 'abc'
+    #       stream << 'def'
+    #       stream << 'ghi'
+    #     end
+    #   end
+    # end
+    module LowerPriority
+      extend ActiveSupport::Concern
+
+      included do
+        unless public_method_defined?(:record_count=)
+          raise(ArgumentError, 'LowerPriority can only be used in conjunction with RocketJob::Plugins::Batch')
+        end
+
+        # For each of this many records lower the priority by 1.
+        class_attribute :lower_priority_count
+        self.lower_priority_count = 100_000
+      end
+
+      private
+
+      def lower_priority
+        return unless record_count
+
+        new_priority  = priority + (record_count.to_f / lower_priority_count).to_i
+        self.priority = [new_priority, 100].min
+      end
+    end
+  end
+end