rocketjob 3.5.2 → 4.0.0

data/lib/rocket_job/cli.rb

@@ -1,15 +1,16 @@
 require 'optparse'
+require 'json'
 require 'semantic_logger'
 require 'mongoid'
 require 'rocketjob'
-require 'rocket_job/extensions/mongoid/factory'
+require 'pathname'
 module RocketJob
   # Command Line Interface parser for Rocket Job
   class CLI
     include SemanticLogger::Loggable
     attr_accessor :name, :workers, :environment, :pidfile, :directory, :quiet,
                   :log_level, :log_file, :mongo_config, :symmetric_encryption_config,
-                  :filter
+                  :include_filter, :exclude_filter, :where_filter
 
     def initialize(argv)
       @name                        = nil
@@ -22,7 +23,8 @@ module RocketJob
       @log_file                    = nil
       @mongo_config                = nil
       @symmetric_encryption_config = nil
-      @filter                      = nil
+      @include_filter              = nil
+      @exclude_filter              = nil
       parse(argv)
     end
 
@@ -38,10 +40,13 @@ module RocketJob
       # In case Rails did not load the Mongoid Config
       RocketJob::Config.load!(environment, mongo_config, symmetric_encryption_config) if ::Mongoid::Config.clients.empty?
 
+      filter = build_filter
+
       opts               = {}
       opts[:name]        = name if name
       opts[:max_workers] = workers if workers
-      opts[:filter]      = {_type: filter} if filter
+      opts[:filter]      = filter if filter
+
       Server.run(opts)
     end
 
@@ -91,7 +96,7 @@ module RocketJob
 
       require 'rocketjob'
       begin
-        require 'rocketjob_pro'
+        require 'rocketjob_batch'
       rescue LoadError
         nil
       end
@@ -148,6 +153,17 @@ module RocketJob
       end
     end
 
+    # Returns [Hash] a where clause filter to apply to this server.
+    # Returns nil if no filter should be applied
+    def build_filter
+      raise(ArgumentError, 'Cannot supply both a filter and an exclusion filter') if include_filter && exclude_filter
+
+      filter                  = where_filter
+      (filter ||= {})['_type'] = include_filter if include_filter
+      (filter ||= {})['_type'] = {'$not' => exclude_filter} if exclude_filter
+      filter
+    end
+
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
       parser        = OptionParser.new do |o|
@@ -161,8 +177,14 @@ module RocketJob
           warn '-t and --threads are deprecated, use -w or --workers'
           @workers = arg.to_i
         end
-        o.on('-F', '--filter REGEXP', 'Limit this worker to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
-          @filter = Regexp.new(arg, true)
+        o.on('-F', '--filter REGEXP', 'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+          @include_filter = Regexp.new(arg, true)
+        end
+        o.on('-E', '--exclude REGEXP', 'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+          @exclude_filter = Regexp.new(arg, true)
+        end
+        o.on('-W', '--where JSON', "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
+          @where_filter = JSON.parse(arg)
         end
         o.on('-q', '--quiet', 'Do not write to stdout, only to logfile. Necessary when running as a daemon') do
           @quiet = true

data/lib/rocket_job/config.rb

@@ -55,7 +55,7 @@ module RocketJob
       raise(ArgumentError, "Mongo Configuration file: #{config_file} not found") unless config_file.file?
 
       logger.debug "Reading Mongo configuration from: #{config_file}"
-      Mongoid.load!(config_file, environment)
+      ::Mongoid.load!(config_file, environment)
 
       # Load Encryption configuration file if present
       return unless defined?(SymmetricEncryption)

data/lib/rocket_job/extensions/mongoid/clients/options.rb

@@ -0,0 +1,37 @@
+require 'mongoid/criteria'
+require 'mongoid/document'
+module RocketJob
+  module MongoidClients
+    module Options
+      extend ActiveSupport::Concern
+
+      def with_collection(collection_name)
+        self.collection_name = collection_name
+        self
+      end
+
+      def collection(parent = nil)
+        @collection_name ? mongo_client[@collection_name] : super(parent)
+      end
+
+      def collection_name
+        @collection_name || super
+      end
+
+      def collection_name=(collection_name)
+        @collection_name = collection_name&.to_sym
+      end
+
+      private
+
+      module ClassMethods
+        def with_collection(collection_name)
+          all.with_collection(collection_name)
+        end
+      end
+    end
+  end
+end
+
+::Mongoid::Criteria.include(RocketJob::MongoidClients::Options)
+::Mongoid::Document.include(RocketJob::MongoidClients::Options)

data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb

@@ -0,0 +1,17 @@
+::Mongoid::Contextual::Mongo
+module Mongoid
+  module Contextual
+    class Mongo
+      def initialize(criteria)
+        @criteria, @klass, @cache = criteria, criteria.klass, criteria.options[:cache]
+        # Only line changed is here, get collection name from criteria, not @klass
+        #@collection = @klass.collection
+        @collection = criteria.collection
+
+        criteria.send(:merge_type_selection)
+        @view = collection.find(criteria.selector, session: _session)
+        apply_options
+      end
+    end
+  end
+end

data/lib/rocket_job/extensions/mongoid/factory.rb

@@ -3,10 +3,10 @@ require 'mongoid/factory'
 module RocketJob
   # Don't convert to Mongoid::Factory since it conflicts with Mongoid use.
   module MongoidFactory
-    def from_db(klass, attributes = nil, selected_fields = nil)
-      super
-    rescue NameError
-      RocketJob::Job.instantiate(attributes, selected_fields)
+    def from_db(klass, attributes = nil, criteria = nil)
+      obj                 = super(klass, attributes, criteria)
+      obj.collection_name = criteria.collection_name if criteria
+      obj
     end
   end
 end

data/lib/rocket_job/extensions/mongoid_5/clients/options.rb

@@ -0,0 +1,38 @@
+require 'mongoid/criteria'
+require 'mongoid/document'
+module RocketJob
+  module Mongoid5Clients
+    module Options
+      extend ActiveSupport::Concern
+
+      def with_collection(collection_name)
+        self.collection_name = collection_name
+        self
+      end
+
+      def collection
+        return (@klass || self.class).with(persistence_options || {}).collection unless @collection_name
+        (@klass || self.class).mongo_client[@collection_name]
+      end
+
+      def collection_name
+        @collection_name || super
+      end
+
+      def collection_name=(collection_name)
+        @collection_name = collection_name&.to_sym
+      end
+
+      private
+
+      module ClassMethods
+        def with_collection(collection_name)
+          all.with_collection(collection_name)
+        end
+      end
+    end
+  end
+end
+
+::Mongoid::Criteria.include(RocketJob::Mongoid5Clients::Options)
+::Mongoid::Document.include(RocketJob::Mongoid5Clients::Options)

data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb

@@ -0,0 +1,64 @@
+::Mongoid::Contextual::Mongo
+module Mongoid
+  module Contextual
+    class Mongo
+      def initialize(criteria)
+        @criteria, @klass, @cache = criteria, criteria.klass, criteria.options[:cache]
+
+        # Only line changed is here, get collection name from criteria, not @klass
+        #@collection = @klass.with(criteria.persistence_options || {}).collection
+        @collection = criteria.collection
+
+        criteria.send(:merge_type_selection)
+        @view = collection.find(criteria.selector)
+        apply_options
+      end
+
+      #
+      # Patches below add `criteria` as the last argument to `Factory.from_db`
+      #
+      def first
+        return documents.first if cached? && cache_loaded?
+        try_cache(:first) do
+          if raw_doc = view.limit(-1).first
+            doc = Factory.from_db(klass, raw_doc, criteria.options[:fields], criteria)
+            eager_load([doc]).first
+          end
+        end
+      end
+
+      def find_first
+        return documents.first if cached? && cache_loaded?
+        if raw_doc = view.first
+          doc = Factory.from_db(klass, raw_doc, criteria.options[:fields], criteria)
+          eager_load([doc]).first
+        end
+      end
+
+      def last
+        try_cache(:last) do
+          with_inverse_sorting do
+            if raw_doc = view.limit(-1).first
+              doc = Factory.from_db(klass, raw_doc, criteria.options[:fields], criteria)
+              eager_load([doc]).first
+            end
+          end
+        end
+      end
+
+      def documents_for_iteration
+        return documents if cached? && !documents.empty?
+        return view unless eager_loadable?
+        docs = view.map{ |doc| Factory.from_db(klass, doc, criteria.options[:fields], criteria) }
+        eager_load(docs)
+      end
+
+      def yield_document(document, &block)
+        doc = document.respond_to?(:_id) ?
+                document : Factory.from_db(klass, document, criteria.options[:fields], criteria)
+        yield(doc)
+        documents.push(doc) if cacheable?
+      end
+    end
+  end
+end

data/lib/rocket_job/extensions/mongoid_5/factory.rb

@@ -0,0 +1,13 @@
+require 'mongoid/factory'
+
+module RocketJob
+  module Mongoid5Factory
+    def from_db(klass, attributes = nil, selected_fields = nil, criteria = nil)
+      obj                 = super(klass, attributes, selected_fields)
+      obj.collection_name = criteria.collection_name if criteria
+      obj
+    end
+  end
+end
+
+::Mongoid::Factory.extend(RocketJob::Mongoid5Factory)

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -0,0 +1,127 @@
+# Generalized Batch Job.
+#
+# Often used for data correction or cleansing.
+#
+# Example: Iterate over all rows in a table:
+#   code = <<-CODE
+#     if user = User.find(row)
+#       user.cleanse_attributes!
+#       user.save(validate: false)
+#     end
+#   CODE
+#   job  = RocketJob::Jobs::OnDemandBatchJob.new(code: code, description: 'cleanse users')
+#   arel = User.unscoped.all.order('updated_at DESC')
+#   job.record_count = input.upload_arel(arel)
+#   job.save!
+#
+# Console Testing:
+#   code = <<-CODE
+#     if user = User.find(row)
+#       user.cleanse_attributes!
+#       user.save(validate: false)
+#     end
+#   CODE
+#   job  = RocketJob::Jobs::OnDemandBatchJob.new(code: code, description: 'cleanse users')
+#
+#   # Run against a sub-set using a limit
+#   arel = User.unscoped.all.order('updated_at DESC').limit(100)
+#   job.record_count = job.input.upload_arel(arel)
+#
+#   # Run the subset directly within the console
+#   job.perform_now
+#   job.cleanup!
+#
+# By default output is not collected, add the option `collect_output: true` to collect output.
+# Example:
+#   job = RocketJob::Jobs::OnDemandBatchJob(description: 'Fix data', code: code, throttle_running_slices: 5, priority: 30, collect_output: true)
+#
+# Example: Move the upload operation into a before_batch.
+#   upload_code = <<-CODE
+#     arel = User.unscoped.all.order('updated_at DESC')
+#     self.record_count = input.upload_arel(arel)
+#   CODE
+#
+#   code = <<-CODE
+#     if user = User.find(row)
+#       user.cleanse_attributes!
+#       user.save(validate: false)
+#     end
+#   CODE
+#
+#   RocketJob::Jobs::OnDemandBatchJob.create!(
+#     upload_code: upload_code,
+#     code:        code,
+#     description: 'cleanse users'
+#   )
+module RocketJob
+  module Jobs
+    class OnDemandBatchJob < RocketJob::Job
+      include RocketJob::Plugins::Cron
+      include RocketJob::Batch
+      include RocketJob::Batch::Statistics
+
+      self.priority            = 90
+      self.description         = 'Batch Job'
+      self.destroy_on_complete = false
+
+      # Code that is performed against every row / record.
+      field :code, type: String
+
+      # Optional code to execute before the batch is run.
+      # Usually to upload data into the job.
+      field :before_code, type: String
+
+      # Optional code to execute after the batch is run.
+      # Usually to upload data into the job.
+      field :after_code, type: String
+
+      # Data that is made available to the job during the perform.
+      # Be sure to store key names only as Strings, not Symbols.
+      field :data, type: Hash, default: {}
+
+      validates :code, presence: true
+      validate :validate_code
+      validate :validate_before_code
+      validate :validate_after_code
+
+      before_slice :load_perform_code
+      before_batch :run_before_code
+      after_batch :run_after_code
+
+      private
+
+      def load_perform_code
+        instance_eval("def perform(row)\n#{code}\nend")
+      end
+
+      def run_before_code
+        instance_eval(before_code) if before_code
+      end
+
+      def run_after_code
+        instance_eval(after_code) if after_code
+      end
+
+      def validate_code
+        return if code.nil?
+        validate_field(:code) { load_perform_code }
+      end
+
+      def validate_before_code
+        return if before_code.nil?
+        validate_field(:before_code) { instance_eval("def __before_code\n#{before_code}\nend") }
+      end
+
+      def validate_after_code
+        return if after_code.nil?
+        validate_field(:after_code) { instance_eval("def __after_code\n#{after_code}\nend") }
+      end
+
+      def validate_field(field)
+        yield
+      rescue Exception => exc
+        errors.add(field, "Failed to load :#{field}, #{exc.inspect}")
+      end
+    end
+  end
+end

data/lib/rocket_job/jobs/performance_job.rb

@@ -0,0 +1,18 @@
+module RocketJob
+  module Jobs
+    class PerformanceJob < RocketJob::Job
+      include RocketJob::Batch
+
+      # Define the job's default attributes
+      self.description         = 'Performance Test'
+      self.priority            = 5
+      self.slice_size          = 100
+      self.destroy_on_complete = false
+
+      # No operation, just return the supplied line (record)
+      def perform(line)
+        line
+      end
+    end
+  end
+end

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -60,11 +60,8 @@ module RocketJob
 
       def upload_file(job)
         if job.respond_to?(:upload)
-          if original_file_name && defined?(IOStreams)
-            streams = IOStreams.streams_for_file_name(original_file_name)
-            job.upload(upload_file_name, streams: streams)
-            # job.upload sets the archived filename, we want it to be the original file name.
-            job.upload_file_name = original_file_name
+          if original_file_name
+            job.upload(upload_file_name, file_name: original_file_name)
           else
             job.upload(upload_file_name)
           end