rocketjob 5.4.0.beta2 → 6.0.0.rc3

data/lib/rocket_job/category/output.rb

@@ -0,0 +1,25 @@
+module RocketJob
+  module Category
+    # Define the layout for each category of input or output data
+    class Output
+      include SemanticLogger::Loggable
+      include Plugins::Document
+      include Category::Base
+
+      embedded_in :job, class_name: "RocketJob::Job", inverse_of: :output_categories
+
+      # Whether to skip nil values returned from the `perform` method.
+      #   true: save nil values to the output categories.
+      #   false: do not save nil values to the output categories.
+      field :nils, type: ::Mongoid::Boolean, default: false
+
+      # Renders [String] the header line.
+      # Returns [nil] if no header is needed.
+      def render_header
+        return if !tabular? || !tabular.requires_header?
+
+        tabular.render_header
+      end
+    end
+  end
+end

data/lib/rocket_job/cli.rb

@@ -233,24 +233,23 @@ module RocketJob
 
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
-      parser = OptionParser.new do |o|
+      parser        = OptionParser.new do |o|
         o.on("-n", "--name NAME", "Unique Name of this server (Default: host_name:PID)") do |arg|
           Config.name = arg
         end
         o.on("-w", "--workers COUNT", "Number of workers (threads) to start") do |arg|
           @max_workers = arg.to_i
         end
-        o.on("--include REGEXP", 'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+        o.on("--include REGEXP",
+             'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
           @include_filter = Regexp.new(arg, true)
         end
-        o.on("-F", "--filter REGEXP", "DEPRECATED. Use --include") do |arg|
-          warn "-F and --filter are deprecated, use --include"
-          @include_filter = Regexp.new(arg, true)
-        end
-        o.on("-E", "--exclude REGEXP", 'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+        o.on("-E", "--exclude REGEXP",
+             'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
           @exclude_filter = Regexp.new(arg, true)
         end
-        o.on("-W", "--where JSON", "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
+        o.on("-W", "--where JSON",
+             "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
           @where_filter = JSON.parse(arg)
         end
         o.on("-q", "--quiet", "Do not write to stdout, only to logfile. Necessary when running as a daemon") do
@@ -259,7 +258,8 @@ module RocketJob
         o.on("-d", "--dir DIR", "Directory containing Rails app, if not current directory") do |arg|
           @directory = arg
         end
-        o.on("-e", "--environment ENVIRONMENT", "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
+        o.on("-e", "--environment ENVIRONMENT",
+             "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
           @environment = arg
         end
         o.on("-l", "--log_level trace|debug|info|warn|error|fatal", "The log level to use") do |arg|
@@ -274,38 +274,46 @@ module RocketJob
         o.on("-m", "--mongo MONGO_CONFIG_FILE_NAME", "Path and filename of config file. Default: config/mongoid.yml") do |arg|
           @mongo_config = arg
         end
-        o.on("-s", "--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME", "Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml") do |arg|
+        o.on("-s", "--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME",
+             "Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml") do |arg|
           @symmetric_encryption_config = arg
         end
-        o.on("--list [FILTER]", "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
+        o.on("--list [FILTER]",
+             "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
           @quiet        = true
           @server       = false
           @list_servers = filter || :all
         end
-        o.on("--refresh [SECONDS]", "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
+        o.on("--refresh [SECONDS]",
+             "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
           @refresh = (seconds || 1).to_s.to_f
         end
-        o.on("--stop [SERVER_NAME]", "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
+        o.on("--stop [SERVER_NAME]",
+             "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @stop_server = server_name || :all
         end
-        o.on("--kill [SERVER_NAME]", "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
+        o.on("--kill [SERVER_NAME]",
+             "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @kill_server = server_name || :all
         end
-        o.on("--pause [SERVER_NAME]", "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
+        o.on("--pause [SERVER_NAME]",
+             "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
           @quiet        = true
           @server       = false
           @pause_server = server_name || :all
         end
-        o.on("--resume [SERVER_NAME]", "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
+        o.on("--resume [SERVER_NAME]",
+             "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
           @quiet         = true
           @server        = false
           @resume_server = server_name || :all
         end
-        o.on("--dump [SERVER_NAME]", "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
+        o.on("--dump [SERVER_NAME]",
+             "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @thread_dump = server_name || :all

data/lib/rocket_job/dirmon_entry.rb

@@ -57,20 +57,12 @@ module RocketJob
     # If this DirmonEntry is in the failed state, exception contains the cause
     embeds_one :exception, class_name: "RocketJob::JobException"
 
-    # The maximum number of files that should ever match during a single poll of the pattern.
-    #
-    # Too many files could be as a result of an invalid pattern specification.
-    # Exceeding this number will result in an exception being logged in a failed Dirmon instance.
-    # Dirmon processing will continue with new instances.
-    # TODO: Implement max_hits
-    # field :max_hits, type: Integer, default: 100
-
     #
     # Read-only attributes
     #
 
     # Current state, as set by the state machine. Do not modify directly.
-    field :state, type: Symbol, default: :pending
+    field :state, type: Mongoid::StringifiedSymbol, default: :pending
 
     # Unique index on pattern to help prevent two entries from scanning the same files
     index({pattern: 1}, background: true, unique: true)
@@ -240,7 +232,7 @@ module RocketJob
         job_class_name:     job_class_name,
         properties:         properties,
         description:        "#{name}: #{iopath.basename}",
-        upload_file_name:   archive_path.to_s,
+        upload_file_name:   archive_path,
         original_file_name: iopath.to_s,
         job_id:             job_id
       )
@@ -249,7 +241,7 @@ module RocketJob
         message: "Created RocketJob::Jobs::UploadFileJob",
         payload: {
           dirmon_entry_name:  name,
-          upload_file_name:   archive_path.to_s,
+          upload_file_name:   archive_path,
           original_file_name: iopath.to_s,
           job_class_name:     job_class_name,
           job_id:             job_id.to_s,
@@ -295,7 +287,25 @@ module RocketJob
       properties.each_pair do |k, _v|
         next if klass.public_method_defined?("#{k}=".to_sym)
 
-        errors.add(:properties, "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}")
+        if %i[output_categories input_categories].include?(k)
+          category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
+          properties[k].each do |category|
+            category.each_pair do |key, _value|
+              next if category_class.public_method_defined?("#{key}=".to_sym)
+
+              errors.add(
+                :properties,
+                "Unknown Property in #{k}: Attempted to set a value for #{key}.#{k} which is not allowed on the job #{job_class_name}"
+              )
+            end
+          end
+          next
+        end
+
+        errors.add(
+          :properties,
+          "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}"
+        )
       end
     end
   end

data/lib/rocket_job/event.rb

@@ -35,7 +35,7 @@ module RocketJob
     #     :shutdown
     #     :pause
     #     :updated
-    field :action, type: Symbol
+    field :action, type: Mongoid::StringifiedSymbol
 
     # Hash Parameters to be sent with the event (event specific).
     field :parameters, type: Hash

data/lib/rocket_job/extensions/iostreams/path.rb

@@ -0,0 +1,32 @@
+module IOStreams
+  class Path
+    # Converts an object of this instance into a database friendly value.
+    def mongoize
+      to_s
+    end
+
+    # Get the object as it was stored in the database, and instantiate
+    # this custom class from it.
+    def self.demongoize(object)
+      return if object.nil?
+
+      IOStreams.new(object)
+    end
+
+    # Takes any possible object and converts it to how it would be
+    # stored in the database.
+    def self.mongoize(object)
+      return if object.nil?
+
+      object.to_s
+    end
+
+    # Converts the object that was supplied to a criteria and converts it
+    # into a database friendly form.
+    def self.evolve(object)
+      return if object.nil?
+
+      object.to_s
+    end
+  end
+end

data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb

@@ -4,8 +4,8 @@ module Mongoid
     class Mongo
       def initialize(criteria)
         @criteria = criteria
-        @klass = criteria.klass
-        @cache = criteria.options[:cache]
+        @klass    = criteria.klass
+        @cache    = criteria.options[:cache]
         # Only line changed is here, get collection name from criteria, not @klass
         # @collection = @klass.collection
         @collection = criteria.collection

data/lib/rocket_job/extensions/mongoid/factory.rb

@@ -3,18 +3,10 @@ require "mongoid/factory"
 module RocketJob
   # Don't convert to Mongoid::Factory since it conflicts with Mongoid use.
   module MongoidFactory
-    if Mongoid::VERSION.to_f >= 7.1
-      def from_db(klass, attributes = nil, criteria = nil, selected_fields = nil)
-        obj                 = super(klass, attributes, criteria, selected_fields)
-        obj.collection_name = criteria.collection_name if criteria
-        obj
-      end
-    else
-      def from_db(klass, attributes = nil, criteria = nil)
-        obj                 = super(klass, attributes, criteria)
-        obj.collection_name = criteria.collection_name if criteria
-        obj
-      end
+    def from_db(klass, attributes = nil, criteria = nil, selected_fields = nil)
+      obj                 = super(klass, attributes, criteria, selected_fields)
+      obj.collection_name = criteria.collection_name if criteria
+      obj
     end
   end
 end

data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# A class which sends values to the database as Strings but returns them to the user as Symbols.
+module Mongoid
+  class StringifiedSymbol
+    class << self
+      # Convert the object from its mongo friendly ruby type to this type.
+      #
+      # @example Demongoize the object.
+      #   Symbol.demongoize(object)
+      #
+      # @param [ Object ] object The object to demongoize.
+      #
+      # @return [ Symbol ] The object.
+      #
+      # @api private
+      def demongoize(object)
+        if object.nil?
+          object
+        else
+          object.to_s.to_sym
+        end
+      end
+
+      # Turn the object from the ruby type we deal with to a Mongo friendly
+      # type.
+      #
+      # @example Mongoize the object.
+      #   Symbol.mongoize("123.11")
+      #
+      # @param [ Object ] object The object to mongoize.
+      #
+      # @return [ Symbol ] The object mongoized.
+      #
+      # @api private
+      def mongoize(object)
+        if object.nil?
+          object
+        else
+          object.to_s
+        end
+      end
+
+      # @api private
+      def evolve(object)
+        mongoize(object)
+      end
+    end
+  end
+end

data/lib/rocket_job/extensions/psych/yaml_tree.rb

@@ -0,0 +1,8 @@
+require "psych/visitors/yaml_tree"
+
+class Psych::Visitors::YAMLTree
+  # Serialize IOStream path as a string
+  def visit_IOStreams_Path(o)
+    visit_String(o.to_s)
+  end
+end

data/lib/rocket_job/extensions/rocket_job_adapter.rb

@@ -55,13 +55,13 @@ module ActiveJob
     #   - Completed jobs will not appear in completed since the Active Job adapter
     #     uses the default Rocket Job `destroy_on_completion` of `false`.
     class RocketJobAdapter
-      def self.enqueue(active_job) #:nodoc:
+      def self.enqueue(active_job)
         job                        = RocketJob::Jobs::ActiveJob.create!(active_job_params(active_job))
         active_job.provider_job_id = job.id.to_s if active_job.respond_to?(:provider_job_id=)
         job
       end
 
-      def self.enqueue_at(active_job, timestamp) #:nodoc:
+      def self.enqueue_at(active_job, timestamp)
         params          = active_job_params(active_job)
         params[:run_at] = Time.at(timestamp).utc
 

data/lib/rocket_job/jobs/conversion_job.rb

@@ -0,0 +1,39 @@
+# Convert to and from CSV, JSON, xlsx, and PSV files.
+#
+# Example, Convert CSV file to JSON.
+#   job = RocketJob::ConversionJob.new
+#   job.upload("data.csv")
+#   job.output_category.file_name = "data.json"
+#   job.save!
+#
+# Example, Convert JSON file to PSV and compress it with GZip.
+#   job = RocketJob::ConversionJob.new
+#   job.upload("data.json")
+#   job.output_category.file_name = "data.psv.gz"
+#   job.save!
+#
+# Example, Read a CSV file that has been zipped from a remote website and the convert it to a GZipped json file.
+#   job = RocketJob::ConversionJob.new
+#   job.upload("https://example.org/file.zip")
+#   job.output_category.file_name = "data.json.gz"
+#   job.save!
+#
+module RocketJob
+  class ConversionJob < RocketJob::Job
+    include RocketJob::Batch
+
+    self.destroy_on_complete = false
+
+    # Detects file extension for its type
+    input_category format: :auto
+    output_category format: :auto
+
+    # When the job completes it will write the result to the output_category.file_name
+    after_batch :download
+
+    def perform(hash)
+      # For this job return the input hash record as-is. Could be transformed here as needed.
+      hash
+    end
+  end
+end

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -18,7 +18,7 @@ module RocketJob
     #   file name of the archived file is passed into the job as either
     #   `upload_file_name` or `full_file_name`.
 
-    # Note:
+    # Notes:
     # - Jobs that do not implement #upload _must_ have either `upload_file_name` or `full_file_name` as an attribute.
     #
     # With RocketJob Pro, the file is automatically uploaded into the job itself
@@ -82,7 +82,7 @@ module RocketJob
             key           = iopath.to_s.tr(".", "_")
             previous_size = previous_file_names[key]
             # Check every few minutes for a file size change before trying to process the file.
-            size = check_file(entry, iopath, previous_size)
+            size                = check_file(entry, iopath, previous_size)
             new_file_names[key] = size if size
           end
         end

data/lib/rocket_job/jobs/housekeeping_job.rb

@@ -35,7 +35,7 @@ module RocketJob
       self.cron_schedule = "*/15 * * * * UTC"
 
       # Whether to destroy zombie servers automatically
-      field :destroy_zombies, type: Boolean, default: true, user_editable: true, copy_on_restart: true
+      field :destroy_zombies, type: Mongoid::Boolean, default: true, user_editable: true, copy_on_restart: true
 
       # Retention intervals in seconds.
       # Set to nil to retain everything.
@@ -54,12 +54,12 @@ module RocketJob
         RocketJob::Job.paused.where(completed_at: {"$lte" => paused_retention.seconds.ago}).destroy_all if paused_retention
         RocketJob::Job.queued.where(created_at: {"$lte" => queued_retention.seconds.ago}).destroy_all if queued_retention
 
-        if destroy_zombies
-          # Cleanup zombie servers
-          RocketJob::Server.destroy_zombies
-          # Requeue jobs where the worker is in the zombie state and its server has gone away
-          RocketJob::ActiveWorker.requeue_zombies
-        end
+        return unless destroy_zombies
+
+        # Cleanup zombie servers
+        RocketJob::Server.destroy_zombies
+        # Requeue jobs where the worker is in the zombie state and its server has gone away
+        RocketJob::ActiveWorker.requeue_zombies
       end
     end
   end

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -31,16 +31,17 @@
 #   job.perform_now
 #   job.cleanup!
 #
-# By default output is not collected, add the option `collect_output: true` to collect output.
+# By default output is not collected, call the method `#collect_output` to collect output.
 #
 # Example:
 #   job = RocketJob::Jobs::OnDemandBatchJob(
 #     description:              'Fix data',
 #     code:                     code,
 #     throttle_running_workers: 5,
-#     priority:                 30,
-#     collect_output:           true
+#     priority:                 30
 #   )
+#   job.collect_output
+#   job.save!
 #
 # Example: Move the upload operation into a before_batch.
 #   upload_code = <<-CODE
@@ -95,10 +96,20 @@ module RocketJob
       before_batch :run_before_code
       after_batch :run_after_code
 
+      # Shortcut for setting the slice_size
+      def slice_size=(slice_size)
+        input_category.slice_size = slice_size
+      end
+
+      # Add a new output category and collect output for it.
+      def add_output_category(**args)
+        self.output_categories << RocketJob::Category::Output.new(**args)
+      end
+
       private
 
       def load_perform_code
-        instance_eval("def perform(row)\n#{code}\nend")
+        instance_eval("def perform(row)\n#{code}\nend", __FILE__, __LINE__)
       end
 
       def run_before_code
@@ -118,13 +129,13 @@ module RocketJob
       def validate_before_code
         return if before_code.nil?
 
-        validate_field(:before_code) { instance_eval("def __before_code\n#{before_code}\nend") }
+        validate_field(:before_code) { instance_eval("def __before_code\n#{before_code}\nend", __FILE__, __LINE__) }
       end
 
       def validate_after_code
         return if after_code.nil?
 
-        validate_field(:after_code) { instance_eval("def __after_code\n#{after_code}\nend") }
+        validate_field(:after_code) { instance_eval("def __after_code\n#{after_code}\nend", __FILE__, __LINE__) }
       end
 
       def validate_field(field)