rocketjob 2.1.3 → 3.0.0.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 69c06ffc4a4e337c787f06d36873d763ccced1be
-  data.tar.gz: 401c2bda273c7413b0ceda2853aef6c325b15159
+  metadata.gz: 328cd371a61e647e687589d126b222c603dfe3a0
+  data.tar.gz: c13d9c10d2156acbacbf92b05af835639c15b112
 SHA512:
-  metadata.gz: 87d8f78b0e766a6d1ab06efb00ff83a3b7b27ade12d11109630f01bed3a753ba8f1fe6241d198209870657ee2e6256d5f6c6ba49293c21ad6dc305c85cc4835e
-  data.tar.gz: d87017700e5fb67815b6ce934b73a683d4c08d96ed04e5f802e58ced4a4aa28b2c67be371258c6f643edb6444f90d66e57f0dc1d698b7398dd199bb80eb1d4c6
+  metadata.gz: bb17eef819ef602e0d58fd89546495ac6f080e803c8421511be078e6821f9500860e40443120f417710cd1313e8e0fd329328c4cdcc0ace6ce4056aa7ab9703f
+  data.tar.gz: 74691dda4d998f54208a5ee3edbb004f49b078dd9f6d9d003b4795b654999aa292986fc36041aca282bca22f5b3b70d45e8a15e55323ebad602195b0d47a4e32

data/README.md

@@ -17,6 +17,42 @@ Checkout http://rocketjob.io/
 * Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
 * [Report bugs](https://github.com/rocketjob/rocketjob/issues)
 
+## Upgrading to V3
+
+* V3 replaces MongoMapper with Mongoid which supports the latest MongoDB Ruby client driver.
+* Replase usages of `rocket_job do` to set default values:
+
+~~~ruby
+  rocket_job do |job|
+    job.priority = 25
+  end
+~~~
+
+With:
+
+~~~ruby
+  self.priority = 25
+~~~
+
+* Replace `key` with `field` when adding attributes to a job:
+
+~~~ruby
+  key :inquiry_defaults, Hash
+~~~
+
+With:
+
+~~~ruby
+  field :inquiry_defaults, type: Hash, default: {}
+~~~
+
+* Replace usage of `public_rocket_job_properties` with the `user_editable` option:
+
+~~~ruby
+field :priority, type: Integer, default: 50, user_editable: true
+~~~
+
+
 ## Ruby Support
 
 Rocket Job is tested and supported on the following Ruby platforms:

data/lib/rocket_job/active_server.rb

@@ -0,0 +1,48 @@
+module RocketJob
+  # Information about a server currently working on a job
+  class ActiveServer
+    # When this server started working on this job / slice
+    attr_accessor :started_at
+
+    attr_accessor :name, :job
+
+    # Returns [Hash<String:ActiveWorker>] hash of all servers sorted by name
+    # and what they are currently working on.
+    # Returns {} if no servers are currently busy doing any work
+    def self.all
+      servers = {}
+      # Need paused, failed or aborted since servers may still be working on active slices
+      RocketJob::Job.where(state: [:running, :paused, :failed, :aborted]).each do |job|
+        job.rocket_job_active_servers.each_pair do |name, active_server|
+          (servers[name] ||= []) << active_server
+        end
+      end
+      servers
+    end
+
+    def initialize(name, started_at, job)
+      @name       = name
+      @started_at = started_at
+      @job        = job
+    end
+
+    # Duration in human readable form
+    def duration
+      RocketJob.seconds_as_duration(duration_s)
+    end
+
+    # Number of seconds this server has been working on this job / slice
+    def duration_s
+      Time.now - started_at
+    end
+
+    def server
+      @server ||= RocketJob::Server.find_by(name: name)
+    end
+
+    def name=(name)
+      @server = nil
+      @name   = name
+    end
+  end
+end

data/lib/rocket_job/cli.rb

@@ -4,21 +4,23 @@ module RocketJob
   # Command Line Interface parser for RocketJob
   class CLI
     include SemanticLogger::Loggable
-    attr_accessor :name, :threads, :environment, :pidfile, :directory, :quiet, :log_level, :log_file
+    attr_accessor :name, :workers, :environment, :pidfile, :directory, :quiet, :log_level, :log_file, :mongo_config, :symmetric_encryption_config
 
     def initialize(argv)
-      @name        = nil
-      @threads     = nil
-      @quiet       = false
-      @environment = nil
-      @pidfile     = nil
-      @directory   = '.'
-      @log_level   = nil
-      @log_file    = nil
+      @name                        = nil
+      @workers                     = nil
+      @quiet                       = false
+      @environment                 = nil
+      @pidfile                     = nil
+      @directory                   = '.'
+      @log_level                   = nil
+      @log_file                    = nil
+      @mongo_config                = nil
+      @symmetric_encryption_config = nil
       parse(argv)
     end
 
-    # Run a RocketJob::Worker from the command line
+    # Run a RocketJob::Server from the command line
     def run
       Thread.current.name = 'rocketjob main'
       setup_environment
@@ -28,8 +30,8 @@ module RocketJob
 
       opts               = {}
       opts[:name]        = name if name
-      opts[:max_threads] = threads if threads
-      Worker.run(opts)
+      opts[:max_workers] = workers if workers
+      Server.run(opts)
     end
 
     def rails?
@@ -57,7 +59,7 @@ module RocketJob
       SemanticLogger.default_level = log_level.to_sym if log_level
 
       if Rails.configuration.eager_load
-        RocketJob::Worker.logger.measure_info('Eager loaded Rails and all Engines') do
+        logger.measure_info('Eager loaded Rails and all Engines') do
           Rails.application.eager_load!
           Rails::Engine.subclasses.each(&:eager_load!)
         end
@@ -85,7 +87,7 @@ module RocketJob
       SemanticLogger.add_appender(file_name: path.to_s, formatter: :color)
 
       logger.info "Rails not detected. Running standalone: #{environment}"
-      RocketJob::Config.load!(environment)
+      RocketJob::Config.load!(environment, mongo_config, symmetric_encryption_config)
       self.class.eager_load_jobs
     end
 
@@ -130,11 +132,15 @@ module RocketJob
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
       parser        = OptionParser.new do |o|
-        o.on('-n', '--name NAME', 'Unique Name of this worker instance (Default: host_name:PID)') do |arg|
+        o.on('-n', '--name NAME', 'Unique Name of this server (Default: host_name:PID)') do |arg|
           @name = arg
         end
-        o.on('-t', '--threads COUNT', 'Number of worker threads to start') do |arg|
-          @threads = arg.to_i
+        o.on('-w', '--workers COUNT', 'Number of workers (threads) to start') do |arg|
+          @workers = arg.to_i
+        end
+        o.on('-t', '--threads COUNT', 'Deprecated') do |arg|
+          warn '-t and --threads are deprecated, use -w or --workers'
+          @workers = arg.to_i
         end
         o.on('-q', '--quiet', 'Do not write to stdout, only to logfile. Necessary when running as a daemon') do
           @quiet = true
@@ -154,6 +160,12 @@ module RocketJob
         o.on('--pidfile PATH', 'Use PATH as a pidfile') do |arg|
           @pidfile = arg
         end
+        o.on('-m', '--mongo MONGO_CONFIG_FILE_NAME', 'Path and filename of config file. Default: config/mongoid.yml') do |arg|
+          @mongo_config = arg
+        end
+        o.on('-s', '--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME', 'Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml') do |arg|
+          @symmetric_encryption_config = arg
+        end
         o.on('-v', '--version', 'Print the version information') do
           puts "Rocket Job v#{RocketJob::VERSION}"
           exit 1

data/lib/rocket_job/config.rb

@@ -21,15 +21,22 @@ module RocketJob
     # Also, exceptions will be raised instead of failing the job.
     cattr_accessor(:inline_mode) { false }
 
-    # @formatter:off
-    # The maximum number of worker threads to create on any one worker
-    key :max_worker_threads,         Integer, default: 10
+    #
+    # Servers
+    #
+
+    # The maximum number of workers to create on any one server
+    field :max_worker_threads, type: Integer, default: 10
 
-    # Number of seconds between heartbeats from Rocket Job Worker processes
-    key :heartbeat_seconds,          Integer, default: 15
+    # Number of seconds between heartbeats from a Rocket Job Server process
+    field :heartbeat_seconds, type: Integer, default: 15
+
+    #
+    # Workers
+    #
 
     # Maximum number of seconds a Worker will wait before checking for new jobs
-    key :max_poll_seconds,           Integer, default: 5
+    field :max_poll_seconds, type: Integer, default: 5
 
     # Number of seconds between checking for:
     # - Jobs with a higher priority
@@ -40,33 +47,14 @@ module RocketJob
     #
     # Note:
     #   Not all job types support pausing in the middle
-    key :re_check_seconds,           Integer, default: 60
-
-    # @formatter:on
-
-    # Replace the MongoMapper default mongo connection for holding jobs
-    def self.mongo_connection=(connection)
-      connection(connection)
-      Worker.connection(connection)
-      Job.connection(connection)
-      Config.connection(connection)
-      DirmonEntry.connection(connection)
-
-      db_name = connection.db.name
-      set_database_name(db_name)
-      Worker.set_database_name(db_name)
-      Job.set_database_name(db_name)
-      Config.set_database_name(db_name)
-      DirmonEntry.set_database_name(db_name)
-    end
+    field :re_check_seconds, type: Integer, default: 60
 
-    # Configure MongoMapper
-    def self.load!(environment='development', file_name=nil, encryption_file_name=nil)
-      config_file = file_name ? Pathname.new(file_name) : Pathname.pwd.join('config/mongo.yml')
+    # Configure Mongoid
+    def self.load!(environment = 'development', file_name = nil, encryption_file_name = nil)
+      config_file = file_name ? Pathname.new(file_name) : Pathname.pwd.join('config/mongoid.yml')
       if config_file.file?
-        logger.debug "Reading MongoDB configuration from: #{config_file}"
-        config = YAML.load(ERB.new(config_file.read).result)
-        MongoMapper.setup(config, environment)
+        logger.debug "Reading Mongo configuration from: #{config_file}"
+        Mongoid.load!(config_file, environment)
       else
         raise(ArgumentError, "Mongo Configuration file: #{config_file.to_s} not found")
       end

data/lib/rocket_job/dirmon_entry.rb

@@ -6,9 +6,8 @@ module RocketJob
     include Plugins::Document
     include Plugins::StateMachine
 
-    # @formatter:off
     # User defined name used to identify this DirmonEntry in Mission Control
-    key :name,               String
+    field :name, type: String
 
     # Pattern for finding files
     #
@@ -27,36 +26,19 @@ module RocketJob
     # - If there is no '*' in the pattern then an exact filename match is expected
     # - The pattern is not validated to ensure the path exists, it will be validated against the
     #   `whitelist_paths` when processed by DirmonJob
-    key :pattern,            String
+    field :pattern, type: String
 
     # Job to enqueue for processing for every file that matches the pattern
     #
     # Example:
     #   "ProcessItJob"
-    key :job_class_name,     String
-
-    # Any user supplied arguments for the method invocation
-    # All keys must be UTF-8 strings. The values can be any valid BSON type:
-    #   Integer
-    #   Float
-    #   Time    (UTC)
-    #   String  (UTF-8)
-    #   Array
-    #   Hash
-    #   True
-    #   False
-    #   Symbol
-    #   nil
-    #   Regular Expression
-    #
-    # Note: Date is not supported, convert it to a UTC time
-    key :arguments,          Array
+    field :job_class_name, type: String
 
     # Any job properties to set
     #
     # Example, override the default job priority:
     #   { priority: 45 }
-    key :properties,         Hash
+    field :properties, type: Hash, default: {}
 
     # Archive directory to move files to when processed to prevent processing the
     # file again.
@@ -64,10 +46,10 @@ module RocketJob
     # If supplied, the file will be moved to this directory before the job is started
     # If the file was in a sub-directory, the corresponding sub-directory will
     # be created in the archive directory.
-    key :archive_directory,  String
+    field :archive_directory, type: String
 
     # If this DirmonEntry is in the failed state, exception contains the cause
-    one :exception,          class_name: 'RocketJob::JobException'
+    embeds_one :exception, class_name: 'RocketJob::JobException'
 
     # The maximum number of files that should ever match during a single poll of the pattern.
     #
@@ -75,14 +57,17 @@ module RocketJob
     # Exceeding this number will result in an exception being logged in a failed Dirmon instance.
     # Dirmon processing will continue with new instances.
     # TODO: Implement max_hits
-    #key :max_hits,           Integer, default: 100
+    #field :max_hits, type: Integer, default: 100
 
     #
     # Read-only attributes
     #
 
     # Current state, as set by the state machine. Do not modify directly.
-    key :state,              Symbol, default: :pending
+    field :state, type: Symbol, default: :pending
+
+    # Unique index on pattern to help prevent two entries from scanning the same files
+    index({pattern: 1}, background: true, unique: true, drop_dups: true)
 
     # State Machine events and transitions
     #
@@ -107,13 +92,13 @@ module RocketJob
       state :disabled
 
       event :enable do
-        transitions from: :pending,  to: :enabled
+        transitions from: :pending, to: :enabled
         transitions from: :disabled, to: :enabled
       end
 
       event :disable do
         transitions from: :enabled, to: :disabled
-        transitions from: :failed,  to: :disabled
+        transitions from: :failed, to: :disabled
       end
 
       event :fail, before: :set_exception do
@@ -134,13 +119,6 @@ module RocketJob
       record.errors.add(attr, 'job_class_name must be defined and must be derived from RocketJob::Job') unless exists
     end
 
-    validates_each :arguments do |record, attr, value|
-      if klass = record.job_class
-        count = klass.rocket_job_argument_count
-        record.errors.add(attr, "There must be #{count} argument(s)") if value.size != count
-      end
-    end
-
     validates_each :properties do |record, attr, value|
       if record.job_class && (methods = record.job_class.instance_methods)
         value.each_pair do |k, v|
@@ -149,12 +127,6 @@ module RocketJob
       end
     end
 
-    # Create indexes
-    def self.create_indexes
-      # Unique index on pattern to help prevent two entries from scanning the same files
-      ensure_index({pattern: 1}, background: true, unique: true)
-    end
-
     # Security Settings
     #
     # A whitelist of paths from which to process files.
@@ -306,7 +278,7 @@ module RocketJob
     def later(pathname)
       if klass = job_class
         logger.measure_info "Enqueued: #{name}, Job class: #{job_class_name}" do
-          job = klass.new(properties.merge(arguments: arguments))
+          job = klass.new(properties)
           upload_file(job, pathname)
           job.save!
           job
@@ -341,10 +313,8 @@ module RocketJob
         job.upload_file_name = full_file_name
       elsif job.respond_to?(:full_file_name=)
         job.full_file_name = full_file_name
-      elsif job.arguments.first.is_a?(Hash)
-        job.arguments.first[:full_file_name] = full_file_name
       else
-        raise(ArgumentError, "#{job_class_name} must either have attribute 'upload_file_name' or the first argument must be a Hash")
+        raise(ArgumentError, "#{job_class_name} must either have attribute 'upload_file_name' or 'full_file_name'")
       end
     end
 

data/lib/rocket_job/extensions/mongo/logging.rb

@@ -0,0 +1,26 @@
+require 'mongo/monitoring/command_log_subscriber'
+
+module Mongo
+  class Monitoring
+    class CommandLogSubscriber
+      include SemanticLogger::Loggable
+      self.logger.name = 'Mongo'
+
+      def started(event)
+        @event_command = event.command
+      end
+
+      def succeeded(event)
+        logger.debug(message: prefix(event), duration: (event.duration * 1000), payload: @event_command)
+      end
+
+      def failed(event)
+        logger.debug(message: "#{prefix(event)} Failed: #{event.message}", duration: (event.duration * 1000), payload: @event_command)
+      end
+
+      def prefix(event)
+        "#{event.address.to_s} | #{event.database_name}.#{event.command_name}"
+      end
+    end
+  end
+end

data/lib/rocket_job/extensions/rocket_job_adapter.rb

@@ -1,5 +1,3 @@
-#require 'rocketjob'
-
 module ActiveJob
   module QueueAdapters
     # == Rocket Job adapter for Active Job
@@ -38,9 +36,9 @@ module ActiveJob
       end
 
       class JobWrapper < RocketJob::Job #:nodoc:
-        key :active_job_id, String
-        key :active_job_class, String
-        key :active_job_queue, String
+        field :active_job_id, type: String
+        field :active_job_class, type: String
+        field :active_job_queue, type: String
 
         def perform(job_data)
           Base.execute job_data