rocketjob 5.1.1 → 5.2.0.beta1

data/lib/rocket_job/cli.rb

@@ -1,9 +1,9 @@
-require 'optparse'
-require 'json'
-require 'semantic_logger'
-require 'mongoid'
-require 'rocketjob'
-require 'pathname'
+require "optparse"
+require "json"
+require "semantic_logger"
+require "mongoid"
+require "rocketjob"
+require "pathname"
 module RocketJob
   # Command Line Interface parser for Rocket Job
   class CLI
@@ -18,7 +18,7 @@ module RocketJob
       @quiet                       = false
       @environment                 = nil
       @pidfile                     = nil
-      @directory                   = '.'
+      @directory                   = "."
       @log_level                   = nil
       @log_file                    = nil
       @mongo_config                = nil
@@ -36,7 +36,7 @@ module RocketJob
 
     # Run a RocketJob::Server from the command line
     def run
-      Thread.current.name = 'rocketjob main'
+      Thread.current.name = "rocketjob main"
       RocketJob.server! if server
       setup_environment
       setup_logger
@@ -60,7 +60,7 @@ module RocketJob
     def rails?
       @rails ||=
         begin
-          boot_file = Pathname.new(directory).join('config/environment.rb').expand_path
+          boot_file = Pathname.new(directory).join("config/environment.rb").expand_path
           boot_file.file?
         end
     end
@@ -71,13 +71,13 @@ module RocketJob
       logger.info "Loading Rails environment: #{environment}"
       RocketJob.rails!
 
-      require 'rails'
-      require 'rocket_job/railtie'
-      boot_file = Pathname.new(directory).join('config/environment.rb').expand_path
+      require "rails"
+      require "rocket_job/railtie"
+      boot_file = Pathname.new(directory).join("config/environment.rb").expand_path
       require(boot_file.to_s)
 
       begin
-        require 'rails_semantic_logger'
+        require "rails_semantic_logger"
       rescue LoadError
         raise "Add the following line to your Gemfile when running rails:\n gem 'rails_semantic_logger'"
       end
@@ -87,10 +87,10 @@ module RocketJob
 
       return unless Rails.configuration.eager_load
 
-      logger.measure_info('Eager loaded Rails and all Engines') do
+      logger.measure_info("Eager loaded Rails and all Engines") do
         Rails.application.eager_load!
         Rails::Engine.subclasses.each(&:eager_load!)
-        self.class.eager_load_jobs(File.expand_path('jobs', File.dirname(__FILE__)))
+        self.class.eager_load_jobs(File.expand_path("jobs", File.dirname(__FILE__)))
       end
     end
 
@@ -98,13 +98,13 @@ module RocketJob
     def boot_standalone
       # Try to load bundler if present
       begin
-        require 'bundler/setup'
+        require "bundler/setup"
         Bundler.require(environment)
       rescue LoadError
         nil
       end
 
-      require 'rocketjob'
+      require "rocketjob"
 
       # Log to file except when booting rails, when it will add the log file path
       path = log_file ? Pathname.new(log_file) : Pathname.pwd.join("log/#{environment}.log")
@@ -113,7 +113,7 @@ module RocketJob
 
       logger.info "Rails not detected. Running standalone: #{environment}"
       RocketJob::Config.load!(environment, mongo_config, symmetric_encryption_config)
-      self.class.eager_load_jobs(File.expand_path('jobs', File.dirname(__FILE__)))
+      self.class.eager_load_jobs(File.expand_path("jobs", File.dirname(__FILE__)))
       self.class.eager_load_jobs
     end
 
@@ -128,8 +128,9 @@ module RocketJob
     # Create a PID file if requested
     def write_pidfile
       return unless pidfile
+
       pid = $PID
-      File.open(pidfile, 'w') { |f| f.puts(pid) }
+      File.open(pidfile, "w") { |f| f.puts(pid) }
 
       # Remove pidfile on exit
       at_exit do
@@ -140,9 +141,9 @@ module RocketJob
     def setup_environment
       # Override Env vars when environment is supplied
       if environment
-        ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
+        ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
       else
-        self.environment = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+        self.environment = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
       end
     end
 
@@ -158,9 +159,10 @@ module RocketJob
     end
 
     # Eager load files in jobs folder
-    def self.eager_load_jobs(job_path = 'jobs')
+    def self.eager_load_jobs(job_path = "jobs")
       Pathname.glob("#{job_path}/**/*.rb").each do |path|
         next if path.directory?
+
         logger.debug "Loading #{path}"
         require path.expand_path.to_s
       end
@@ -169,7 +171,7 @@ module RocketJob
     def perform_list_servers(filter)
       return list_the_servers(filter) unless refresh
 
-      while true
+      loop do
         list_the_servers(filter)
         sleep(refresh)
         puts
@@ -177,10 +179,10 @@ module RocketJob
     end
 
     def list_the_servers(filter)
-      format = "%50.50s %20.20s %20.20s %20.20s %10.10s"
-      puts format % %w[Server\ Name Workers(Current/Max) Started Heartbeat State]
+      layout = "%50.50s %20.20s %20.20s %20.20s %10.10s"
+      puts format(layout, "Server Name", "Workers(Current/Max)", "Started", "Heartbeat", "State")
       header = "=" * 50
-      puts format % [header, header, header, header, header]
+      puts format(layout, header, header, header, header, header)
       query = filter == :all ? RocketJob::Server.all : RocketJob::Server.where(name: /#{filter}/)
       query.each do |server|
         workers   = "#{server&.heartbeat&.workers}/#{server.max_workers}"
@@ -188,14 +190,14 @@ module RocketJob
         started   = "#{RocketJob.seconds_as_duration(duration)} ago"
         duration  = Time.now - (server&.heartbeat&.updated_at || Time.now)
         heartbeat = "#{RocketJob.seconds_as_duration(duration)} ago"
-        puts format % [server.name, workers, started, heartbeat, server.state]
+        puts format(layout, server.name, workers, started, heartbeat, server.state)
       end
       0
     end
 
     def perform_server_action(server_name, action)
       server_ids(server_name).each { |server_id| RocketJob::Subscribers::Server.publish(action, server_id: server_id) }
-      #RocketJob::Subscribers::Worker.publish(:stop, worker_id: 1, server_id: RocketJob::Server.running.last.id)
+      # RocketJob::Subscribers::Worker.publish(:stop, worker_id: 1, server_id: RocketJob::Server.running.last.id)
       0
     end
 
@@ -219,6 +221,7 @@ module RocketJob
       if pid
         server = RocketJob::Server.where(name: server_name).first
         raise(ArgumentError, "No server with exact name: #{server_name} was found.") unless server
+
         return [server.id]
       end
 
@@ -230,90 +233,90 @@ module RocketJob
 
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
-      parser        = OptionParser.new do |o|
-        o.on('-n', '--name NAME', 'Unique Name of this server (Default: host_name:PID)') do |arg|
+      parser = OptionParser.new do |o|
+        o.on("-n", "--name NAME", "Unique Name of this server (Default: host_name:PID)") do |arg|
           Config.name = arg
         end
-        o.on('-w', '--workers COUNT', 'Number of workers (threads) to start') do |arg|
+        o.on("-w", "--workers COUNT", "Number of workers (threads) to start") do |arg|
           @max_workers = arg.to_i
         end
-        o.on('--include REGEXP', 'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+        o.on("--include REGEXP", 'Limit this server to only those job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
           @include_filter = Regexp.new(arg, true)
         end
-        o.on('-F', '--filter REGEXP', 'DEPRECATED. Use --include') do |arg|
-          warn '-F and --filter are deprecated, use --include'
+        o.on("-F", "--filter REGEXP", "DEPRECATED. Use --include") do |arg|
+          warn "-F and --filter are deprecated, use --include"
           @include_filter = Regexp.new(arg, true)
         end
-        o.on('-E', '--exclude REGEXP', 'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
+        o.on("-E", "--exclude REGEXP", 'Prevent this server from working on any job classes that match this regular expression (case-insensitive). Example: "DirmonJob|WeeklyReportJob"') do |arg|
           @exclude_filter = Regexp.new(arg, true)
         end
-        o.on('-W', '--where JSON', "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
+        o.on("-W", "--where JSON", "Limit this server instance to the supplied mongo query filter. Supply as a string in JSON format. Example: '{\"priority\":{\"$lte\":25}}'") do |arg|
           @where_filter = JSON.parse(arg)
         end
-        o.on('-q', '--quiet', 'Do not write to stdout, only to logfile. Necessary when running as a daemon') do
+        o.on("-q", "--quiet", "Do not write to stdout, only to logfile. Necessary when running as a daemon") do
           @quiet = true
         end
-        o.on('-d', '--dir DIR', 'Directory containing Rails app, if not current directory') do |arg|
+        o.on("-d", "--dir DIR", "Directory containing Rails app, if not current directory") do |arg|
           @directory = arg
         end
-        o.on('-e', '--environment ENVIRONMENT', 'The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)') do |arg|
+        o.on("-e", "--environment ENVIRONMENT", "The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)") do |arg|
           @environment = arg
         end
-        o.on('-l', '--log_level trace|debug|info|warn|error|fatal', 'The log level to use') do |arg|
+        o.on("-l", "--log_level trace|debug|info|warn|error|fatal", "The log level to use") do |arg|
           @log_level = arg
         end
-        o.on('-f', '--log_file FILE_NAME', 'The log file to write to. Default: log/<environment>.log') do |arg|
+        o.on("-f", "--log_file FILE_NAME", "The log file to write to. Default: log/<environment>.log") do |arg|
           @log_file = arg
         end
-        o.on('--pidfile PATH', 'Use PATH as a pidfile') do |arg|
+        o.on("--pidfile PATH", "Use PATH as a pidfile") do |arg|
           @pidfile = arg
         end
-        o.on('-m', '--mongo MONGO_CONFIG_FILE_NAME', 'Path and filename of config file. Default: config/mongoid.yml') do |arg|
+        o.on("-m", "--mongo MONGO_CONFIG_FILE_NAME", "Path and filename of config file. Default: config/mongoid.yml") do |arg|
           @mongo_config = arg
         end
-        o.on('-s', '--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME', 'Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml') do |arg|
+        o.on("-s", "--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME", "Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml") do |arg|
           @symmetric_encryption_config = arg
         end
-        o.on('--list [FILTER]', "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
+        o.on("--list [FILTER]", "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
           @quiet        = true
           @server       = false
           @list_servers = filter || :all
         end
-        o.on('--refresh [SECONDS]', "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
+        o.on("--refresh [SECONDS]", "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
           @refresh = (seconds || 1).to_s.to_f
         end
-        o.on('--stop [SERVER_NAME]', "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
+        o.on("--stop [SERVER_NAME]", "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @stop_server = server_name || :all
         end
-        o.on('--kill [SERVER_NAME]', "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
+        o.on("--kill [SERVER_NAME]", "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @kill_server = server_name || :all
         end
-        o.on('--pause [SERVER_NAME]', "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
+        o.on("--pause [SERVER_NAME]", "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
           @quiet        = true
           @server       = false
           @pause_server = server_name || :all
         end
-        o.on('--resume [SERVER_NAME]', "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
+        o.on("--resume [SERVER_NAME]", "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
           @quiet         = true
           @server        = false
           @resume_server = server_name || :all
         end
-        o.on('--dump [SERVER_NAME]', "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
+        o.on("--dump [SERVER_NAME]", "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
           @quiet       = true
           @server      = false
           @thread_dump = server_name || :all
         end
-        o.on('-v', '--version', 'Print the version information') do
+        o.on("-v", "--version", "Print the version information") do
           puts "Rocket Job v#{RocketJob::VERSION}"
           exit 1
         end
       end
-      parser.banner = 'rocketjob <options>'
-      parser.on_tail '-h', '--help', 'Show help' do
+      parser.banner = "rocketjob <options>"
+      parser.on_tail "-h", "--help", "Show help" do
         puts parser
         exit 1
       end

data/lib/rocket_job/config.rb

@@ -1,4 +1,4 @@
-require 'yaml'
+require "yaml"
 module RocketJob
   # Rocket Job Configuration
   class Config
@@ -72,8 +72,8 @@ module RocketJob
     self.where_filter = nil
 
     # Configure Mongoid
-    def self.load!(environment = 'development', file_name = nil, encryption_file_name = nil)
-      config_file = file_name ? Pathname.new(file_name) : Pathname.pwd.join('config/mongoid.yml')
+    def self.load!(environment = "development", file_name = nil, encryption_file_name = nil)
+      config_file = file_name ? Pathname.new(file_name) : Pathname.pwd.join("config/mongoid.yml")
 
       raise(ArgumentError, "Mongo Configuration file: #{config_file} not found") unless config_file.file?
 
@@ -84,7 +84,7 @@ module RocketJob
         if encryption_file_name
           Pathname.new(encryption_file_name)
         else
-          Pathname.pwd.join('config/symmetric-encryption.yml')
+          Pathname.pwd.join("config/symmetric-encryption.yml")
         end
 
       return unless config_file.file?
@@ -97,14 +97,12 @@ module RocketJob
     #    include_filter, exclude_filter, and where_filter.
     # Returns nil if no filter should be applied.
     def self.filter
-      if include_filter && exclude_filter
-        raise(ArgumentError, 'Cannot supply both an include_filter and an exclude_filter')
-      end
+      raise(ArgumentError, "Cannot supply both an include_filter and an exclude_filter") if include_filter && exclude_filter
 
       filter                   = where_filter
-      (filter ||= {})['_type'] = include_filter if include_filter
-      (filter ||= {})['_type'] = {'$not' => exclude_filter} if exclude_filter
-      filter
+      (filter ||= {})["_type"] = include_filter if include_filter
+      (filter ||= {})["_type"] = {"$not" => exclude_filter} if exclude_filter
+      filter&.dup
     end
   end
 end

data/lib/rocket_job/dirmon_entry.rb

@@ -1,5 +1,5 @@
-require 'concurrent'
-require 'fileutils'
+require "concurrent"
+require "fileutils"
 module RocketJob
   class DirmonEntry
     include Plugins::Document
@@ -8,9 +8,9 @@ module RocketJob
     # The default archive directory that is used when the job being queued does not respond
     # to #upload, and does not have an `archive_directory` specified in this entry
     class_attribute :default_archive_directory
-    self.default_archive_directory = 'archive'.freeze
+    self.default_archive_directory = "archive".freeze
 
-    store_in collection: 'rocket_job.dirmon_entries'
+    store_in collection: "rocket_job.dirmon_entries"
 
     # User defined name used to identify this DirmonEntry in the Web Interface.
     field :name, type: String
@@ -55,7 +55,7 @@ module RocketJob
     field :archive_directory, type: String, default: default_archive_directory
 
     # If this DirmonEntry is in the failed state, exception contains the cause
-    embeds_one :exception, class_name: 'RocketJob::JobException'
+    embeds_one :exception, class_name: "RocketJob::JobException"
 
     # The maximum number of files that should ever match during a single poll of the pattern.
     #
@@ -73,7 +73,7 @@ module RocketJob
     field :state, type: Symbol, default: :pending
 
     # Unique index on pattern to help prevent two entries from scanning the same files
-    index({pattern: 1}, background: true, unique: true, drop_dups: true)
+    index({pattern: 1}, background: true, unique: true)
 
     before_validation :strip_whitespace
     validates_presence_of :pattern, :job_class_name, :archive_directory
@@ -175,8 +175,8 @@ module RocketJob
     #   # => {}
     def self.counts_by_state
       counts = {}
-      collection.aggregate([{'$group' => {_id: '$state', count: {'$sum' => 1}}}]).each do |result|
-        counts[result['_id'].to_sym] = result['count']
+      collection.aggregate([{"$group" => {_id: "$state", count: {"$sum" => 1}}}]).each do |result|
+        counts[result["_id"].to_sym] = result["count"]
       end
       counts
     end
@@ -213,7 +213,7 @@ module RocketJob
         exception.worker_name = worker_name
       else
         build_exception(
-          class_name:  'RocketJob::DirmonEntryException',
+          class_name:  "RocketJob::DirmonEntryException",
           message:     exc_or_message,
           backtrace:   [],
           worker_name: worker_name
@@ -224,6 +224,7 @@ module RocketJob
     # Returns the Job to be created.
     def job_class
       return if job_class_name.nil?
+
       job_class_name.constantize
     rescue NameError
       nil
@@ -245,7 +246,7 @@ module RocketJob
       )
 
       logger.info(
-        message: 'Created RocketJob::Jobs::UploadFileJob',
+        message: "Created RocketJob::Jobs::UploadFileJob",
         payload: {
           dirmon_entry_name:  name,
           upload_file_name:   archive_path.to_s,
@@ -282,6 +283,7 @@ module RocketJob
     def job_is_a_rocket_job
       klass = job_class
       return if job_class_name.nil? || klass&.ancestors&.include?(RocketJob::Job)
+
       errors.add(:job_class_name, "Job #{job_class_name} must be defined and inherit from RocketJob::Job")
     end
 
@@ -292,6 +294,7 @@ module RocketJob
 
       properties.each_pair do |k, _v|
         next if klass.public_method_defined?("#{k}=".to_sym)
+
         errors.add(:properties, "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}")
       end
     end

data/lib/rocket_job/event.rb

@@ -1,4 +1,4 @@
-require 'concurrent-ruby'
+require "concurrent-ruby"
 
 module RocketJob
   # RocketJob::Event
@@ -10,7 +10,7 @@ module RocketJob
     include Plugins::Document
     include Mongoid::Timestamps
 
-    ALL_EVENTS = '*'.freeze
+    ALL_EVENTS = "*".freeze
 
     # Capped collection long polling interval.
     class_attribute :long_poll_seconds, instance_accessor: false
@@ -42,7 +42,7 @@ module RocketJob
 
     validates_presence_of :name
 
-    store_in collection: 'rocket_job.events'
+    store_in collection: "rocket_job.events"
     index({created_at: 1}, background: true)
 
     # Add a subscriber for its events.
@@ -84,14 +84,14 @@ module RocketJob
     # Indefinitely tail the capped collection looking for new events.
     #   time: the start time from which to start looking for new events.
     def self.listener(time: @load_time)
-      Thread.current.name = 'rocketjob event'
+      Thread.current.name = "rocketjob event"
       create_capped_collection
 
-      logger.info('Event listener started')
+      logger.info("Event listener started")
       tail_capped_collection(time) { |event| process_event(event) }
-    rescue Exception => exc
-      logger.error('#listener Event listener is terminating due to unhandled exception', exc)
-      raise(exc)
+    rescue Exception => e
+      logger.error("#listener Event listener is terminating due to unhandled exception", e)
+      raise(e)
     end
 
     # Create the capped collection only if it does not exist.
@@ -117,13 +117,13 @@ module RocketJob
 
     def self.tail_capped_collection(time)
       with(socket_timeout: long_poll_seconds + 10) do
-        filter = {created_at: {'$gt' => time}}
+        filter = {created_at: {"$gt" => time}}
         collection.
           find(filter).
           await_data.
           cursor_type(:tailable_await).
           max_await_time_ms(long_poll_seconds * 1000).
-          sort('$natural' => 1).
+          sort("$natural" => 1).
           each do |doc|
           event = Mongoid::Factory.from_db(Event, doc)
           # Recovery will occur from after the last message read
@@ -131,14 +131,14 @@ module RocketJob
           yield(event)
         end
       end
-    rescue Mongo::Error::SocketError, Mongo::Error::SocketTimeoutError, Mongo::Error::OperationFailure, Timeout::Error => exc
-      logger.info("Creating a new cursor and trying again: #{exc.class.name} #{exc.message}")
+    rescue Mongo::Error::SocketError, Mongo::Error::SocketTimeoutError, Mongo::Error::OperationFailure, Timeout::Error => e
+      logger.info("Creating a new cursor and trying again: #{e.class.name} #{e.message}")
       retry
     end
 
     # Process a new event, calling registered subscribers.
     def self.process_event(event)
-      logger.info('Event Received', event.attributes)
+      logger.info("Event Received", event.attributes)
 
       if @subscribers.key?(event.name)
         @subscribers[event.name].each { |subscriber| subscriber.process_action(event.action, event.parameters) }
@@ -147,8 +147,8 @@ module RocketJob
       if @subscribers.key?(ALL_EVENTS)
         @subscribers[ALL_EVENTS].each { |subscriber| subscriber.process_event(event.name, event.action, event.parameters) }
       end
-    rescue StandardError => exc
-      logger.error('Unknown subscriber. Continuing..', exc)
+    rescue StandardError => e
+      logger.error("Unknown subscriber. Continuing..", e)
     end
 
     def self.collection_exists?
@@ -157,7 +157,7 @@ module RocketJob
 
     # Convert a non-capped collection to capped
     def self.convert_to_capped_collection(size)
-      collection.database.command('convertToCapped' => collection_name.to_s, 'size' => size)
+      collection.database.command("convertToCapped" => collection_name.to_s, "size" => size)
     end
   end
 end