rocketjob 5.4.1 → 6.0.0

data/lib/rocket_job/category/output.rb

@@ -0,0 +1,34 @@
+module RocketJob
+  module Category
+    # Define the layout for each category of input or output data
+    class Output
+      include SemanticLogger::Loggable
+      include Plugins::Document
+      include Category::Base
+
+      embedded_in :job, class_name: "RocketJob::Job", inverse_of: :output_categories
+
+      # Whether to skip nil values returned from the `perform` method.
+      #   true: save nil values to the output categories.
+      #   false: do not save nil values to the output categories.
+      field :nils, type: ::Mongoid::Boolean, default: false
+
+      validates_inclusion_of :serializer, in: %i[none compress encrypt bz2 encrypted_bz2 bzip2]
+
+      # Renders [String] the header line.
+      # Returns [nil] if no header is needed.
+      def render_header
+        return if !tabular? || !tabular.requires_header?
+
+        tabular.render_header
+      end
+
+      def data_store(job)
+        RocketJob::Sliced::Output.new(
+          collection_name: build_collection_name(:output, job),
+          slice_class:     serializer_class
+        )
+      end
+    end
+  end
+end

data/lib/rocket_job/cli.rb

@@ -233,24 +233,23 @@ module RocketJob
 
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
-      parser = OptionParser.new do |o|
+      parser        = OptionParser.new do |o|
         o.on("-n", "--name NAME", "Unique Name of this server (Default: host_name:PID)") do |arg|
           Config.name = arg
         end
         o.on("-w", "--workers COUNT", "Number of workers (threads) to start") do |arg|
           @max_workers = arg.to_i
         end
-        o.on("--include REGEXP", 'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+        o.on("--include REGEXP",
+             'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
           @include_filter = Regexp.new(arg, true)
         end
-        o.on("-F", "--filter REGEXP", "DEPRECATED. Use --include") do |arg|
-          warn "-F and --filter are deprecated, use --include"
-          @include_filter = Regexp.new(arg, true)
-        end
-        o.on("-E", "--exclude REGEXP", 'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+        o.on("-E", "--exclude REGEXP",
+             'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
           @exclude_filter = Regexp.new(arg, true)
         end
-        o.on("-W", "--where JSON", "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
+        o.on("-W", "--where JSON",
+             "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
           @where_filter = JSON.parse(arg)
         end
         o.on("-q", "--quiet", "Do not write to stdout, only to logfile. Necessary when running as a daemon") do
@@ -259,7 +258,8 @@ module RocketJob
         o.on("-d", "--dir DIR", "Directory containing Rails app, if not current directory") do |arg|
           @directory = arg
         end
-        o.on("-e", "--environment ENVIRONMENT", "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
+        o.on("-e", "--environment ENVIRONMENT",
+             "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
           @environment = arg
         end
         o.on("-l", "--log_level trace|debug|info|warn|error|fatal", "The log level to use") do |arg|
@@ -274,38 +274,46 @@ module RocketJob
         o.on("-m", "--mongo MONGO_CONFIG_FILE_NAME", "Path and filename of config file. Default: config/mongoid.yml") do |arg|
           @mongo_config = arg
         end
-        o.on("-s", "--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME", "Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml") do |arg|
+        o.on("-s", "--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME",
+             "Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml") do |arg|
           @symmetric_encryption_config = arg
         end
-        o.on("--list [FILTER]", "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
+        o.on("--list [FILTER]",
+             "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
           @quiet        = true
           @server       = false
           @list_servers = filter || :all
         end
-        o.on("--refresh [SECONDS]", "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
+        o.on("--refresh [SECONDS]",
+             "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
           @refresh = (seconds || 1).to_s.to_f
         end
-        o.on("--stop [SERVER_NAME]", "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
+        o.on("--stop [SERVER_NAME]",
+             "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @stop_server = server_name || :all
         end
-        o.on("--kill [SERVER_NAME]", "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
+        o.on("--kill [SERVER_NAME]",
+             "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @kill_server = server_name || :all
         end
-        o.on("--pause [SERVER_NAME]", "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
+        o.on("--pause [SERVER_NAME]",
+             "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
           @quiet        = true
           @server       = false
           @pause_server = server_name || :all
         end
-        o.on("--resume [SERVER_NAME]", "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
+        o.on("--resume [SERVER_NAME]",
+             "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
           @quiet         = true
           @server        = false
           @resume_server = server_name || :all
         end
-        o.on("--dump [SERVER_NAME]", "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
+        o.on("--dump [SERVER_NAME]",
+             "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @thread_dump = server_name || :all

data/lib/rocket_job/dirmon_entry.rb

@@ -57,20 +57,12 @@ module RocketJob
     # If this DirmonEntry is in the failed state, exception contains the cause
     embeds_one :exception, class_name: "RocketJob::JobException"
 
-    # The maximum number of files that should ever match during a single poll of the pattern.
-    #
-    # Too many files could be as a result of an invalid pattern specification.
-    # Exceeding this number will result in an exception being logged in a failed Dirmon instance.
-    # Dirmon processing will continue with new instances.
-    # TODO: Implement max_hits
-    # field :max_hits, type: Integer, default: 100
-
     #
     # Read-only attributes
     #
 
     # Current state, as set by the state machine. Do not modify directly.
-    field :state, type: Symbol, default: :pending
+    field :state, type: Mongoid::StringifiedSymbol, default: :pending
 
     # Unique index on pattern to help prevent two entries from scanning the same files
     index({pattern: 1}, background: true, unique: true)
@@ -181,7 +173,7 @@ module RocketJob
       counts
     end
 
-    # Passes each filename [Pathname] found that matches the pattern into the supplied block
+    # Yields [IOStreams::Path] for each file found that matches the current pattern.
     def each
       SemanticLogger.named_tagged(dirmon_entry: id.to_s) do
         # Case insensitive filename matching
@@ -240,7 +232,7 @@ module RocketJob
         job_class_name:     job_class_name,
         properties:         properties,
         description:        "#{name}: #{iopath.basename}",
-        upload_file_name:   archive_path.to_s,
+        upload_file_name:   archive_path,
         original_file_name: iopath.to_s,
         job_id:             job_id
       )
@@ -249,7 +241,7 @@ module RocketJob
         message: "Created RocketJob::Jobs::UploadFileJob",
         payload: {
           dirmon_entry_name:  name,
-          upload_file_name:   archive_path.to_s,
+          upload_file_name:   archive_path,
           original_file_name: iopath.to_s,
           job_class_name:     job_class_name,
           job_id:             job_id.to_s,
@@ -295,7 +287,25 @@ module RocketJob
       properties.each_pair do |k, _v|
         next if klass.public_method_defined?("#{k}=".to_sym)
 
-        errors.add(:properties, "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}")
+        if %i[output_categories input_categories].include?(k)
+          category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
+          properties[k].each do |category|
+            category.each_pair do |key, _value|
+              next if category_class.public_method_defined?("#{key}=".to_sym)
+
+              errors.add(
+                :properties,
+                "Unknown Property in #{k}: Attempted to set a value for #{key}.#{k} which is not allowed on the job #{job_class_name}"
+              )
+            end
+          end
+          next
+        end
+
+        errors.add(
+          :properties,
+          "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}"
+        )
       end
     end
   end

data/lib/rocket_job/event.rb

@@ -35,7 +35,7 @@ module RocketJob
     #     :shutdown
     #     :pause
     #     :updated
-    field :action, type: Symbol
+    field :action, type: Mongoid::StringifiedSymbol
 
     # Hash Parameters to be sent with the event (event specific).
     field :parameters, type: Hash

data/lib/rocket_job/extensions/iostreams/path.rb

@@ -0,0 +1,32 @@
+module IOStreams
+  class Path
+    # Converts an object of this instance into a database friendly value.
+    def mongoize
+      to_s
+    end
+
+    # Get the object as it was stored in the database, and instantiate
+    # this custom class from it.
+    def self.demongoize(object)
+      return if object.nil?
+
+      IOStreams.new(object)
+    end
+
+    # Takes any possible object and converts it to how it would be
+    # stored in the database.
+    def self.mongoize(object)
+      return if object.nil?
+
+      object.to_s
+    end
+
+    # Converts the object that was supplied to a criteria and converts it
+    # into a database friendly form.
+    def self.evolve(object)
+      return if object.nil?
+
+      object.to_s
+    end
+  end
+end

data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb

@@ -4,8 +4,8 @@ module Mongoid
     class Mongo
       def initialize(criteria)
         @criteria = criteria
-        @klass = criteria.klass
-        @cache = criteria.options[:cache]
+        @klass    = criteria.klass
+        @cache    = criteria.options[:cache]
         # Only line changed is here, get collection name from criteria, not @klass
         # @collection = @klass.collection
         @collection = criteria.collection

data/lib/rocket_job/extensions/mongoid/factory.rb

@@ -3,18 +3,10 @@ require "mongoid/factory"
 module RocketJob
   # Don't convert to Mongoid::Factory since it conflicts with Mongoid use.
   module MongoidFactory
-    if Mongoid::VERSION.to_f >= 7.1
-      def from_db(klass, attributes = nil, criteria = nil, selected_fields = nil)
-        obj                 = super(klass, attributes, criteria, selected_fields)
-        obj.collection_name = criteria.collection_name if criteria
-        obj
-      end
-    else
-      def from_db(klass, attributes = nil, criteria = nil)
-        obj                 = super(klass, attributes, criteria)
-        obj.collection_name = criteria.collection_name if criteria
-        obj
-      end
+    def from_db(klass, attributes = nil, criteria = nil, selected_fields = nil)
+      obj                 = super(klass, attributes, criteria, selected_fields)
+      obj.collection_name = criteria.collection_name if criteria
+      obj
     end
   end
 end

data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# A class which sends values to the database as Strings but returns them to the user as Symbols.
+module Mongoid
+  class StringifiedSymbol
+    class << self
+      # Convert the object from its mongo friendly ruby type to this type.
+      #
+      # @example Demongoize the object.
+      #   Symbol.demongoize(object)
+      #
+      # @param [ Object ] object The object to demongoize.
+      #
+      # @return [ Symbol ] The object.
+      #
+      # @api private
+      def demongoize(object)
+        if object.nil?
+          object
+        else
+          object.to_s.to_sym
+        end
+      end
+
+      # Turn the object from the ruby type we deal with to a Mongo friendly
+      # type.
+      #
+      # @example Mongoize the object.
+      #   Symbol.mongoize("123.11")
+      #
+      # @param [ Object ] object The object to mongoize.
+      #
+      # @return [ Symbol ] The object mongoized.
+      #
+      # @api private
+      def mongoize(object)
+        if object.nil?
+          object
+        else
+          object.to_s
+        end
+      end
+
+      # @api private
+      def evolve(object)
+        mongoize(object)
+      end
+    end
+  end
+end

data/lib/rocket_job/extensions/psych/yaml_tree.rb

@@ -0,0 +1,8 @@
+require "psych/visitors/yaml_tree"
+
+class Psych::Visitors::YAMLTree
+  # Serialize IOStream path as a string
+  def visit_IOStreams_Path(o)
+    visit_String(o.to_s)
+  end
+end

data/lib/rocket_job/extensions/rocket_job_adapter.rb

@@ -55,13 +55,13 @@ module ActiveJob
     #   - Completed jobs will not appear in completed since the Active Job adapter
     #     uses the default Rocket Job `destroy_on_completion` of `false`.
     class RocketJobAdapter
-      def self.enqueue(active_job) #:nodoc:
+      def self.enqueue(active_job)
         job                        = RocketJob::Jobs::ActiveJob.create!(active_job_params(active_job))
         active_job.provider_job_id = job.id.to_s if active_job.respond_to?(:provider_job_id=)
         job
       end
 
-      def self.enqueue_at(active_job, timestamp) #:nodoc:
+      def self.enqueue_at(active_job, timestamp)
         params          = active_job_params(active_job)
         params[:run_at] = Time.at(timestamp).utc
 

data/lib/rocket_job/jobs/conversion_job.rb

@@ -0,0 +1,43 @@
+# Convert to and from CSV, JSON, xlsx, and PSV files.
+#
+# Example, Convert CSV file to JSON.
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "data.csv"
+#   job.output_category.file_name = "data.json"
+#   job.save!
+#
+# Example, Convert JSON file to PSV and compress it with GZip.
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "data.json"
+#   job.output_category.file_name = "data.psv.gz"
+#   job.save!
+#
+# Example, Read a CSV file that has been zipped from a remote website and the convert it to a GZipped json file.
+#   job = RocketJob::Jobs::ConversionJob.new
+#   job.input_category.file_name  = "https://example.org/file.zip"
+#   job.output_category.file_name = "data.json.gz"
+#   job.save!
+#
+module RocketJob
+  module Jobs
+    class ConversionJob < RocketJob::Job
+      include RocketJob::Batch
+
+      self.destroy_on_complete = false
+
+      # Detects file extension for its type
+      input_category format: :auto
+      output_category format: :auto
+
+      # Upload the file specified in `input_category.file_name` unless already uploaded.
+      before_batch :upload, unless: :record_count
+
+      # When the job completes it will write the result to `output_category.file_name`.
+      after_batch :cleanup!, :download
+
+      def perform(hash)
+        hash
+      end
+    end
+  end
+end

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -18,7 +18,7 @@ module RocketJob
     #   file name of the archived file is passed into the job as either
     #   `upload_file_name` or `full_file_name`.
 
-    # Note:
+    # Notes:
     # - Jobs that do not implement #upload _must_ have either `upload_file_name` or `full_file_name` as an attribute.
     #
     # With RocketJob Pro, the file is automatically uploaded into the job itself
@@ -30,59 +30,48 @@ module RocketJob
     #
     # If another DirmonJob instance is already queued or running, then the create
     # above will fail with:
-    #   MongoMapper::DocumentNotValid: Validation failed: State Another instance of this job is already queued or running
+    #   Validation failed: State Another instance of this job is already queued or running
     #
     # Or to start DirmonJob and ignore errors if already running
     #   RocketJob::Jobs::DirmonJob.create
     class DirmonJob < RocketJob::Job
-      # Only allow one DirmonJob instance to be running at a time
-      include RocketJob::Plugins::Singleton
-      # Start a new job when this one completes, fails, or aborts
-      include RocketJob::Plugins::Restart
+      include RocketJob::Plugins::Cron
 
-      self.priority = 30
-
-      # Number of seconds between directory scans. Default 5 mins
-      field :check_seconds, type: Float, default: 300.0, copy_on_restart: true
+      # Runs every 5 minutes by default
+      self.cron_schedule = "*/5 * * * * UTC"
+      self.description   = "Directory Monitor"
+      self.priority      = 30
 
       # Hash[file_name, size]
       field :previous_file_names, type: Hash, default: {}, copy_on_restart: true
 
-      before_create :set_run_at
-
-      # Iterate over each Dirmon entry looking for new files
-      # If a new file is found, it is not processed immediately, instead
-      # it is passed to the next run of this job along with the file size.
-      # If the file size has not changed, the Job is kicked off.
+      # Checks the directories for new files, starting jobs if files have not changed since the last run.
       def perform
         check_directories
       end
 
       private
 
-      # Set a run_at when a new instance of this job is created
-      def set_run_at
-        self.run_at = Time.now + check_seconds
-      end
-
-      # Checks the directories for new files, starting jobs if files have not changed
-      # since the last run
+      # Iterate over each Dirmon Entry looking for new files
+      # If a new file is found, it is not processed immediately, instead
+      # it is passed to the next run of this job along with the file size.
+      # If the file size has not changed, the Job is kicked off.
       def check_directories
         new_file_names = {}
-        DirmonEntry.enabled.each do |entry|
-          entry.each do |iopath|
-            # S3 files are only visible once completely uploaded.
-            unless iopath.partial_files_visible?
-              logger.info("File: #{iopath}. Starting: #{entry.job_class_name}")
-              entry.later(iopath)
+        DirmonEntry.enabled.each do |dirmon_entry|
+          dirmon_entry.each do |path|
+            # Skip file size checking since S3 files are only visible once completely uploaded.
+            unless path.partial_files_visible?
+              logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
+              dirmon_entry.later(path)
               next
             end
 
             # BSON Keys cannot contain periods
-            key           = iopath.to_s.tr(".", "_")
+            key           = path.to_s.tr(".", "_")
             previous_size = previous_file_names[key]
             # Check every few minutes for a file size change before trying to process the file.
-            size = check_file(entry, iopath, previous_size)
+            size                = check_file(dirmon_entry, path, previous_size)
             new_file_names[key] = size if size
           end
         end
@@ -91,14 +80,14 @@ module RocketJob
 
       # Checks if a file should result in starting a job
       # Returns [Integer] file size, or nil if the file started a job
-      def check_file(entry, iopath, previous_size)
-        size = iopath.size
+      def check_file(dirmon_entry, path, previous_size)
+        size = path.size
         if previous_size && (previous_size == size)
-          logger.info("File stabilized: #{iopath}. Starting: #{entry.job_class_name}")
-          entry.later(iopath)
+          logger.info("File stabilized: #{path}. Starting: #{dirmon_entry.job_class_name}")
+          dirmon_entry.later(path)
           nil
         else
-          logger.info("Found file: #{iopath}. File size: #{size}")
+          logger.info("Found file: #{path}. File size: #{size}")
           # Keep for the next run
           size
         end