rocketjob 5.3.3 → 6.0.0.rc1

data/lib/rocket_job/category/base.rb

@@ -0,0 +1,78 @@
+require "active_support/concern"
+
+module RocketJob
+  module Category
+    # Define the layout for each category of input or output data
+    module Base
+      extend ActiveSupport::Concern
+
+      included do
+        field :name, type: ::Mongoid::StringifiedSymbol, default: :main
+
+        # Whether to compress, encrypt, or use the bzip2 serialization for data in this category.
+        field :serializer, type: ::Mongoid::StringifiedSymbol, default: :compress
+        validates_inclusion_of :serializer, in: [:none, :compress, :encrypt, :bzip2]
+
+        # The header columns when the file does not include a header row.
+        # Note:
+        # - All column names must be strings so that it can be serialized into MongoDB.
+        field :columns, type: Array
+
+        # On an input collection `format` specifies the format of the input data so that it can be
+        # transformed into a Hash when passed into the `#perform` method.
+        #
+        # On an output collection `format` specifies the format to transform the output hash into.
+        #
+        # `:auto` it uses the `file_name` on this category to determine the format.
+        # `nil` no transformation is performed on the data returned by the `#perform` method.
+        # Any other format supported by IOStreams, for example: csv, :hash, :array, :json, :psv, :fixed
+        #
+        # Default: `nil`
+        field :format, type: ::Mongoid::StringifiedSymbol
+        validates_inclusion_of :format, in: [nil, :auto] + IOStreams::Tabular.registered_formats
+
+        # Any specialized format specific options. For example, `:fixed` format requires a `:layout`.
+        field :format_options, type: Hash
+
+        # When `:format` is not supplied the file name can be used to infer the required format.
+        # Optional.
+        # Default: nil
+        field :file_name, type: IOStreams::Path
+      end
+
+      # Return which slice serializer class to use that matches the current options.
+      def serializer_class
+        case serializer
+        when :none
+          Sliced::Slice
+        when :compress
+          Sliced::CompressedSlice
+        when :encrypt
+          Sliced::EncryptedSlice
+        when :bzip2
+          Sliced::BZip2OutputSlice
+        else
+          raise(ArgumentError, "serialize: #{serializer.inspect} must be :none, :compress, :encrypt, or :bzip2")
+        end
+      end
+
+      def tabular
+        @tabular ||= IOStreams::Tabular.new(
+          columns:        columns,
+          format:         format == :auto ? nil : format,
+          format_options: format_options&.deep_symbolize_keys,
+          file_name:      file_name
+        )
+      end
+
+      def reset_tabular
+        @tabular = nil
+      end
+
+      # Returns [true|false] whether this category has the attributes defined for tabular to work.
+      def tabular?
+        format.present?
+      end
+    end
+  end
+end

data/lib/rocket_job/category/input.rb

@@ -0,0 +1,110 @@
+module RocketJob
+  module Category
+    # Define the layout for each category of input or output data
+    class Input
+      include SemanticLogger::Loggable
+      include Plugins::Document
+      include Category::Base
+
+      embedded_in :job, class_name: "RocketJob::Job", inverse_of: :input_categories
+
+      # Slice size for this input collection
+      field :slice_size, type: Integer, default: 100
+
+      #
+      # The fields below only apply if the field `format` has been set:
+      #
+
+      # List of columns to allow.
+      # Default: nil ( Allow all columns )
+      # Note:
+      #   When supplied any columns that are rejected will be returned in the cleansed columns
+      #   as nil so that they can be ignored during processing.
+      field :allowed_columns, type: Array
+
+      # List of columns that must be present, otherwise an Exception is raised.
+      field :required_columns, type: Array
+
+      # Whether to skip unknown columns in the uploaded file.
+      # Ignores any column that was not found in the `allowed_columns` list.
+      #
+      # false:
+      #   Raises IOStreams::Tabular::InvalidHeader when a column is supplied that is not in `allowed_columns`.
+      # true:
+      #   Ignore additional columns in a file that are not listed in `allowed_columns`
+      #   Job processing will skip the additional columns entirely as if they were not supplied at all.
+      #   A warning is logged with the names of the columns that were ignored.
+      #   The `columns` field will list all skipped columns with a nil value so that downstream workers
+      #   know to ignore those columns.
+      #
+      # Notes:
+      # - Only applicable when `allowed_columns` has been set.
+      # - Recommended to leave as `false` otherwise a misspelled column can result in missed columns.
+      field :skip_unknown, type: ::Mongoid::Boolean, default: false
+      validates_inclusion_of :skip_unknown, in: [true, false]
+
+      # When `#upload` is called with a file_name, it uploads the file using any of the following approaches:
+      # :line
+      #   Uploads the file a line (String) at a time for processing by workers.
+      #   This is the default behavior and is the most performant since it leaves the parsing of each line
+      #   up to the workers themselves.
+      # :array
+      #   Parses each line from the file as an Array and uploads each array for processing by workers.
+      #   Every line in the input file is parsed and converted into an array before uploading.
+      #   This approach ensures that the entire files is valid before starting to process it.
+      #   Ideal for when files may contain invalid lines.
+      #   Not recommended for large files since the CSV or other parsing is performed sequentially during the
+      #   upload process.
+      # :hash
+      #   Parses each line from the file into a Hash and uploads each hash for processing by workers.
+      #   Similar to :array above in that the entire file is parsed before processing is started.
+      #   Slightly less efficient than :array since it stores every record as a hash with both the key and value.
+      #
+      # Recommend using :array when the entire file must be parsed/validated before processing is started, and
+      # upload time is not important.
+      # See IOStreams#each for more details.
+      field :mode, type: ::Mongoid::StringifiedSymbol, default: :line
+      validates_inclusion_of :mode, in: %i[line array hash]
+
+      # When reading tabular input data (e.g. CSV, PSV) the header is automatically cleansed.
+      # This removes issues when the input header varies in case and other small ways. See IOStreams::Tabular
+      # Currently Supported:
+      #   :default
+      #     Each column is cleansed as follows:
+      #     - Leading and trailing whitespace is stripped.
+      #     - All characters converted to lower case.
+      #     - Spaces and '-' are converted to '_'.
+      #     - All characters except for letters, digits, and '_' are stripped.
+      #   :none
+      #     Do not cleanse the columns names supplied in the header row.
+      #
+      # Note: Submit a ticket if you have other cleansers that you want added.
+      field :header_cleanser, type: ::Mongoid::StringifiedSymbol, default: :default
+      validates :header_cleanser, inclusion: %i[default none]
+
+      validates_presence_of :slice_size
+
+      # Cleanses the header column names when `cleanse_header` is true
+      def cleanse_header!
+        return unless header_cleanser == :default
+
+        ignored_columns = tabular.header.cleanse!
+        logger.warn("Stripped out invalid columns from custom header", ignored_columns) unless ignored_columns.empty?
+
+        self.columns = tabular.header.columns
+      end
+
+      def tabular
+        @tabular ||= IOStreams::Tabular.new(
+          columns:          columns,
+          format:           format == :auto ? nil : format,
+          format_options:   format_options&.deep_symbolize_keys,
+          file_name:        file_name,
+          allowed_columns:  allowed_columns,
+          required_columns: required_columns,
+          skip_unknown:     skip_unknown
+        )
+      end
+    end
+  end
+end

data/lib/rocket_job/category/output.rb

@@ -0,0 +1,25 @@
+module RocketJob
+  module Category
+    # Define the layout for each category of input or output data
+    class Output
+      include SemanticLogger::Loggable
+      include Plugins::Document
+      include Category::Base
+
+      embedded_in :job, class_name: "RocketJob::Job", inverse_of: :output_categories
+
+      # Whether to skip nil values returned from the `perform` method.
+      #   true: save nil values to the output categories.
+      #   false: do not save nil values to the output categories.
+      field :nils, type: ::Mongoid::Boolean, default: false
+
+      # Renders [String] the header line.
+      # Returns [nil] if no header is needed.
+      def render_header
+        return if !tabular? || !tabular.requires_header?
+
+        tabular.render_header
+      end
+    end
+  end
+end

data/lib/rocket_job/cli.rb

@@ -240,17 +240,16 @@ module RocketJob
         o.on("-w", "--workers COUNT", "Number of workers (threads) to start") do |arg|
           @max_workers = arg.to_i
         end
-        o.on("--include REGEXP", 'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+        o.on("--include REGEXP",
+             'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
           @include_filter = Regexp.new(arg, true)
         end
-        o.on("-F", "--filter REGEXP", "DEPRECATED. Use --include") do |arg|
-          warn "-F and --filter are deprecated, use --include"
-          @include_filter = Regexp.new(arg, true)
-        end
-        o.on("-E", "--exclude REGEXP", 'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+        o.on("-E", "--exclude REGEXP",
+             'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
           @exclude_filter = Regexp.new(arg, true)
         end
-        o.on("-W", "--where JSON", "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
+        o.on("-W", "--where JSON",
+             "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
           @where_filter = JSON.parse(arg)
         end
         o.on("-q", "--quiet", "Do not write to stdout, only to logfile. Necessary when running as a daemon") do
@@ -259,7 +258,8 @@ module RocketJob
         o.on("-d", "--dir DIR", "Directory containing Rails app, if not current directory") do |arg|
           @directory = arg
         end
-        o.on("-e", "--environment ENVIRONMENT", "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
+        o.on("-e", "--environment ENVIRONMENT",
+             "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
           @environment = arg
         end
         o.on("-l", "--log_level trace|debug|info|warn|error|fatal", "The log level to use") do |arg|
@@ -274,38 +274,46 @@ module RocketJob
         o.on("-m", "--mongo MONGO_CONFIG_FILE_NAME", "Path and filename of config file. Default: config/mongoid.yml") do |arg|
           @mongo_config = arg
         end
-        o.on("-s", "--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME", "Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml") do |arg|
+        o.on("-s", "--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME",
+             "Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml") do |arg|
           @symmetric_encryption_config = arg
         end
-        o.on("--list [FILTER]", "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
+        o.on("--list [FILTER]",
+             "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
           @quiet        = true
           @server       = false
           @list_servers = filter || :all
         end
-        o.on("--refresh [SECONDS]", "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
+        o.on("--refresh [SECONDS]",
+             "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
           @refresh = (seconds || 1).to_s.to_f
         end
-        o.on("--stop [SERVER_NAME]", "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
+        o.on("--stop [SERVER_NAME]",
+             "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @stop_server = server_name || :all
         end
-        o.on("--kill [SERVER_NAME]", "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
+        o.on("--kill [SERVER_NAME]",
+             "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @kill_server = server_name || :all
         end
-        o.on("--pause [SERVER_NAME]", "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
+        o.on("--pause [SERVER_NAME]",
+             "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
           @quiet        = true
           @server       = false
           @pause_server = server_name || :all
         end
-        o.on("--resume [SERVER_NAME]", "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
+        o.on("--resume [SERVER_NAME]",
+             "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
           @quiet         = true
           @server        = false
           @resume_server = server_name || :all
         end
-        o.on("--dump [SERVER_NAME]", "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
+        o.on("--dump [SERVER_NAME]",
+             "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @thread_dump = server_name || :all

data/lib/rocket_job/dirmon_entry.rb

@@ -57,20 +57,12 @@ module RocketJob
     # If this DirmonEntry is in the failed state, exception contains the cause
     embeds_one :exception, class_name: "RocketJob::JobException"
 
-    # The maximum number of files that should ever match during a single poll of the pattern.
-    #
-    # Too many files could be as a result of an invalid pattern specification.
-    # Exceeding this number will result in an exception being logged in a failed Dirmon instance.
-    # Dirmon processing will continue with new instances.
-    # TODO: Implement max_hits
-    # field :max_hits, type: Integer, default: 100
-
     #
     # Read-only attributes
     #
 
     # Current state, as set by the state machine. Do not modify directly.
-    field :state, type: Symbol, default: :pending
+    field :state, type: Mongoid::StringifiedSymbol, default: :pending
 
     # Unique index on pattern to help prevent two entries from scanning the same files
     index({pattern: 1}, background: true, unique: true)
@@ -240,7 +232,7 @@ module RocketJob
         job_class_name:     job_class_name,
         properties:         properties,
         description:        "#{name}: #{iopath.basename}",
-        upload_file_name:   archive_path.to_s,
+        upload_file_name:   archive_path,
         original_file_name: iopath.to_s,
         job_id:             job_id
       )
@@ -249,7 +241,7 @@ module RocketJob
         message: "Created RocketJob::Jobs::UploadFileJob",
         payload: {
           dirmon_entry_name:  name,
-          upload_file_name:   archive_path.to_s,
+          upload_file_name:   archive_path,
           original_file_name: iopath.to_s,
           job_class_name:     job_class_name,
           job_id:             job_id.to_s,
@@ -295,7 +287,25 @@ module RocketJob
       properties.each_pair do |k, _v|
         next if klass.public_method_defined?("#{k}=".to_sym)
 
-        errors.add(:properties, "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}")
+        if %i[output_categories input_categories].include?(k)
+          category_class = k == :input_categories ? RocketJob::Category::Input : RocketJob::Category::Output
+          properties[k].each do |category|
+            category.each_pair do |key, _value|
+              next if category_class.public_method_defined?("#{key}=".to_sym)
+
+              errors.add(
+                :properties,
+                "Unknown Property in #{k}: Attempted to set a value for #{key}.#{k} which is not allowed on the job #{job_class_name}"
+              )
+            end
+          end
+          next
+        end
+
+        errors.add(
+          :properties,
+          "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}"
+        )
       end
     end
   end

data/lib/rocket_job/event.rb

@@ -35,7 +35,7 @@ module RocketJob
     #     :shutdown
     #     :pause
     #     :updated
-    field :action, type: Symbol
+    field :action, type: Mongoid::StringifiedSymbol
 
     # Hash Parameters to be sent with the event (event specific).
     field :parameters, type: Hash

data/lib/rocket_job/extensions/iostreams/path.rb

@@ -0,0 +1,32 @@
+module IOStreams
+  class Path
+    # Converts an object of this instance into a database friendly value.
+    def mongoize
+      to_s
+    end
+
+    # Get the object as it was stored in the database, and instantiate
+    # this custom class from it.
+    def self.demongoize(object)
+      return if object.nil?
+
+      IOStreams.new(object)
+    end
+
+    # Takes any possible object and converts it to how it would be
+    # stored in the database.
+    def self.mongoize(object)
+      return if object.nil?
+
+      object.to_s
+    end
+
+    # Converts the object that was supplied to a criteria and converts it
+    # into a database friendly form.
+    def self.evolve(object)
+      return if object.nil?
+
+      object.to_s
+    end
+  end
+end

data/lib/rocket_job/extensions/mongoid/factory.rb

@@ -3,18 +3,10 @@ require "mongoid/factory"
 module RocketJob
   # Don't convert to Mongoid::Factory since it conflicts with Mongoid use.
   module MongoidFactory
-    if Mongoid::VERSION.to_f >= 7.1
-      def from_db(klass, attributes = nil, criteria = nil, selected_fields = nil)
-        obj                 = super(klass, attributes, criteria, selected_fields)
-        obj.collection_name = criteria.collection_name if criteria
-        obj
-      end
-    else
-      def from_db(klass, attributes = nil, criteria = nil)
-        obj                 = super(klass, attributes, criteria)
-        obj.collection_name = criteria.collection_name if criteria
-        obj
-      end
+    def from_db(klass, attributes = nil, criteria = nil, selected_fields = nil)
+      obj                 = super(klass, attributes, criteria, selected_fields)
+      obj.collection_name = criteria.collection_name if criteria
+      obj
     end
   end
 end

data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# A class which sends values to the database as Strings but returns them to the user as Symbols.
+module Mongoid
+  class StringifiedSymbol
+    class << self
+      # Convert the object from its mongo friendly ruby type to this type.
+      #
+      # @example Demongoize the object.
+      #   Symbol.demongoize(object)
+      #
+      # @param [ Object ] object The object to demongoize.
+      #
+      # @return [ Symbol ] The object.
+      #
+      # @api private
+      def demongoize(object)
+        if object.nil?
+          object
+        else
+          object.to_s.to_sym
+        end
+      end
+
+      # Turn the object from the ruby type we deal with to a Mongo friendly
+      # type.
+      #
+      # @example Mongoize the object.
+      #   Symbol.mongoize("123.11")
+      #
+      # @param [ Object ] object The object to mongoize.
+      #
+      # @return [ Symbol ] The object mongoized.
+      #
+      # @api private
+      def mongoize(object)
+        if object.nil?
+          object
+        else
+          object.to_s
+        end
+      end
+
+      # @api private
+      def evolve(object)
+        mongoize(object)
+      end
+    end
+  end
+end