rocketjob 3.4.3 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 72a9239aa38ba9581931c3bd6f1402278b3c9d41
-  data.tar.gz: aa360338a810a8853a463149084554978c57774f
+SHA256:
+  metadata.gz: 9dd8c3c0ab00ae83d73d63aef3887f62d1d2a695399f263814ad7a5881aa980a
+  data.tar.gz: 715304e4587a9eda64c3cb5f73a69cd72ecdbf4ee232884c61f36666ef7c1a3a
 SHA512:
-  metadata.gz: 1ba33a298630312129bc76f34084e49a348e27bba34e6f452a7d990a0c96d800c664db58b04724503d2cdcdd944a3120ccf09e583661453fb93832d2d38458b3
-  data.tar.gz: f141cfbd428307638df1ac8ed20d00e345483370e6c4c40523d9409d2afc6f2d23b8c49d8b27c6eedac4aa72b48d6168dbfd1880b43cd20aaf452fdf223e4ce9
+  metadata.gz: 3bc50f4e10068d2b193747c24880dee618be8df6ab58e3093f442332f93c591796901132490a856ee955aa19cb78abaad6034eb7280e080c494676c7c36d3899
+  data.tar.gz: 86d8dbe971e8e44a8e4854d1691078ac93a25f84fd8e482b9e84a3669457ad8886f84fc47007f199625e70fcecc1f4b1403804e38dc50acebd5c06b13b11d20b

data/README.md

@@ -17,6 +17,33 @@ Checkout http://rocketjob.io/
 * Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
 * [Report bugs](https://github.com/rocketjob/rocketjob/issues)
 
+## Contributing to the documentation
+
+To contribute to the documentation it is as easy as forking the repository
+and then editing the markdown pages directly via the github web interface.
+
+For more complex documentation changes checkout the source code locally.
+
+#### Local checkout
+
+* Fork the repository in github.
+* Checkout your fork of the source code locally.
+* Install Jekyll
+~~~
+    cd docs
+    bundle update
+~~~
+* Run Jekyll web server:
+~~~
+    jekyll serve
+~~~
+* Open a web browser to view the local documentation:
+    [http://127.0.0.1:4000](http://127.0.0.1:4000)
+* Edit the files in the `/docs` folder.
+* Refresh the page to see the changes.
+
+Once the changes are complete, submit a github pull request.
+
 ## Upgrading to V3
 
 V3 replaces MongoMapper with Mongoid which supports the latest MongoDB Ruby client driver.

data/bin/rocketjob

@@ -6,7 +6,7 @@ require 'rocket_job/cli'
 # Start a rocketjob server instance from the command line
 begin
   RocketJob::CLI.new(ARGV).run
-rescue => exc
+rescue Exception => exc
   # Failsafe logger that writes to STDERR
   SemanticLogger.add_appender(io: STDERR, level: :error, formatter: :color)
   SemanticLogger['RocketJob'].error('Rocket Job shutting down due to exception', exc)

data/lib/rocket_job/active_worker.rb

@@ -4,7 +4,8 @@ module RocketJob
     # When this server started working on this job / slice
     attr_accessor :started_at
 
-    attr_accessor :name, :job
+    attr_accessor :job
+    attr_reader :name
 
     # Returns [Hash<String:ActiveWorker>] hash of all servers sorted by name
     # and what they are currently working on.
@@ -16,7 +17,7 @@ module RocketJob
     def self.all(server_name = nil)
       servers = []
       # Need paused, failed or aborted since servers may still be working on active slices
-      query = RocketJob::Job.where(:state.in => [:running, :paused, :failed, :aborted])
+      query = RocketJob::Job.where(:state.in => %i[running paused failed aborted])
       query = query.where(worker_name: /\A#{server_name}/) if server_name
       query.each do |job|
         servers += job.rocket_job_active_workers
@@ -50,7 +51,7 @@ module RocketJob
 
     # Returns [String] the name of the server running this worker
     def server_name
-      if match = name.to_s.match(/(.*:.*):.*/)
+      if (match = name.to_s.match(/(.*:.*):.*/))
         match[1]
       else
         name

data/lib/rocket_job/cli.rb

@@ -4,7 +4,7 @@ require 'mongoid'
 require 'rocketjob'
 require 'rocket_job/extensions/mongoid/factory'
 module RocketJob
-  # Command Line Interface parser for RocketJob
+  # Command Line Interface parser for Rocket Job
   class CLI
     include SemanticLogger::Loggable
     attr_accessor :name, :workers, :environment, :pidfile, :directory, :quiet,
@@ -41,7 +41,7 @@ module RocketJob
       opts               = {}
       opts[:name]        = name if name
       opts[:max_workers] = workers if workers
-      opts[:filter]      = {:_type => filter} if filter
+      opts[:filter]      = {_type: filter} if filter
       Server.run(opts)
     end
 
@@ -70,12 +70,12 @@ module RocketJob
       # Override Rails log level if command line option was supplied
       SemanticLogger.default_level = log_level.to_sym if log_level
 
-      if Rails.configuration.eager_load
-        logger.measure_info('Eager loaded Rails and all Engines') do
-          Rails.application.eager_load!
-          Rails::Engine.subclasses.each(&:eager_load!)
-          self.class.eager_load_jobs(File.expand_path('jobs', File.dirname(__FILE__)))
-        end
+      return unless Rails.configuration.eager_load
+
+      logger.measure_info('Eager loaded Rails and all Engines') do
+        Rails.application.eager_load!
+        Rails::Engine.subclasses.each(&:eager_load!)
+        self.class.eager_load_jobs(File.expand_path('jobs', File.dirname(__FILE__)))
       end
     end
 
@@ -86,12 +86,14 @@ module RocketJob
         require 'bundler/setup'
         Bundler.require(environment)
       rescue LoadError
+        nil
       end
 
       require 'rocketjob'
       begin
         require 'rocketjob_pro'
       rescue LoadError
+        nil
       end
 
       # Log to file except when booting rails, when it will add the log file path
@@ -138,10 +140,10 @@ module RocketJob
     end
 
     # Eager load files in jobs folder
-    def self.eager_load_jobs(path = 'jobs')
-      Pathname.glob("#{path}/**/*.rb").each do |path|
+    def self.eager_load_jobs(job_path = 'jobs')
+      Pathname.glob("#{job_path}/**/*.rb").each do |path|
         next if path.directory?
-        logger.debug "Loading #{path.to_s}"
+        logger.debug "Loading #{path}"
         require path.expand_path.to_s
       end
     end
@@ -198,6 +200,5 @@ module RocketJob
       end
       parser.parse! argv
     end
-
   end
 end

data/lib/rocket_job/config.rb

@@ -51,22 +51,26 @@ module RocketJob
     # Configure Mongoid
     def self.load!(environment = 'development', file_name = nil, encryption_file_name = nil)
       config_file = file_name ? Pathname.new(file_name) : Pathname.pwd.join('config/mongoid.yml')
-      if config_file.file?
-        logger.debug "Reading Mongo configuration from: #{config_file}"
-        Mongoid.load!(config_file, environment)
-      else
-        raise(ArgumentError, "Mongo Configuration file: #{config_file.to_s} not found")
-      end
+
+      raise(ArgumentError, "Mongo Configuration file: #{config_file} not found") unless config_file.file?
+
+      logger.debug "Reading Mongo configuration from: #{config_file}"
+      Mongoid.load!(config_file, environment)
 
       # Load Encryption configuration file if present
-      if defined?(SymmetricEncryption)
-        config_file = encryption_file_name ? Pathname.new(encryption_file_name) : Pathname.pwd.join('config/symmetric-encryption.yml')
-        if config_file.file?
-          logger.debug "Reading SymmetricEncryption configuration from: #{config_file}"
-          SymmetricEncryption.load!(config_file.to_s, environment)
+      return unless defined?(SymmetricEncryption)
+
+      config_file =
+        if encryption_file_name
+          Pathname.new(encryption_file_name)
+        else
+          Pathname.pwd.join('config/symmetric-encryption.yml')
         end
-      end
-    end
 
+      return unless config_file.file?
+
+      logger.debug "Reading SymmetricEncryption configuration from: #{config_file}"
+      SymmetricEncryption.load!(config_file.to_s, environment)
+    end
   end
 end

data/lib/rocket_job/dirmon_entry.rb

@@ -6,9 +6,14 @@ module RocketJob
     include Plugins::Document
     include Plugins::StateMachine
 
+    # The default archive directory that is used when the job being queued does not respond
+    # to #upload, and does not have an `archive_directory` specified in this entry
+    class_attribute :default_archive_directory
+    self.default_archive_directory = 'archive'.freeze
+
     store_in collection: 'rocket_job.dirmon_entries'
 
-    # User defined name used to identify this DirmonEntry in Mission Control
+    # User defined name used to identify this DirmonEntry in the Web Interface.
     field :name, type: String
 
     # Pattern for finding files
@@ -48,7 +53,7 @@ module RocketJob
     # If supplied, the file will be moved to this directory before the job is started
     # If the file was in a sub-directory, the corresponding sub-directory will
     # be created in the archive directory.
-    field :archive_directory, type: String
+    field :archive_directory, type: String, default: default_archive_directory
 
     # If this DirmonEntry is in the failed state, exception contains the cause
     embeds_one :exception, class_name: 'RocketJob::JobException'
@@ -59,7 +64,7 @@ module RocketJob
     # Exceeding this number will result in an exception being logged in a failed Dirmon instance.
     # Dirmon processing will continue with new instances.
     # TODO: Implement max_hits
-    #field :max_hits, type: Integer, default: 100
+    # field :max_hits, type: Integer, default: 100
 
     #
     # Read-only attributes
@@ -71,6 +76,11 @@ module RocketJob
     # Unique index on pattern to help prevent two entries from scanning the same files
     index({pattern: 1}, background: true, unique: true, drop_dups: true)
 
+    before_validation :strip_whitespace
+    validates_presence_of :pattern, :job_class_name, :archive_directory
+    validate :job_is_a_rocket_job
+    validate :job_has_properties
+
     # State Machine events and transitions
     #
     # :pending -> :enabled  -> :disabled
@@ -108,27 +118,6 @@ module RocketJob
       end
     end
 
-    # @formatter:on
-    validates_presence_of :pattern, :job_class_name
-
-    validates_each :job_class_name do |record, attr, value|
-      exists =
-        begin
-          value.nil? ? false : record.job_class.ancestors.include?(RocketJob::Job)
-        rescue NameError
-          false
-        end
-      record.errors.add(attr, 'job_class_name must be defined and must be derived from RocketJob::Job') unless exists
-    end
-
-    validates_each :properties do |record, attr, value|
-      if record.job_class && (methods = record.job_class.instance_methods)
-        value.each_pair do |k, v|
-          record.errors.add(attr, "Unknown property: #{k.inspect} with value: #{v}") unless methods.include?("#{k}=".to_sym)
-        end
-      end
-    end
-
     # Security Settings
     #
     # A whitelist of paths from which to process files.
@@ -147,7 +136,7 @@ module RocketJob
     #
     # Returns [Array<String>] a copy of the whitelisted paths
     def self.get_whitelist_paths
-      self.whitelist_paths.dup
+      whitelist_paths.dup
     end
 
     # Add a path to the whitelist
@@ -155,8 +144,8 @@ module RocketJob
     def self.add_whitelist_path(path)
       # Confirms that path exists
       path = Pathname.new(path).realpath.to_s
-      self.whitelist_paths << path
-      self.whitelist_paths.uniq!
+      whitelist_paths << path
+      whitelist_paths.uniq!
       path
     end
 
@@ -165,8 +154,8 @@ module RocketJob
     def self.delete_whitelist_path(path)
       # Confirms that path exists
       path = Pathname.new(path).realpath.to_s
-      self.whitelist_paths.delete(path)
-      self.whitelist_paths.uniq!
+      whitelist_paths.delete(path)
+      whitelist_paths.uniq!
       path
     end
 
@@ -193,30 +182,8 @@ module RocketJob
       counts
     end
 
-    # The default archive directory that is used when the job being queued does not respond
-    # to #upload, and does not have an `archive_directory` specified in this entry
-    class_attribute :default_archive_directory
-
-    self.default_archive_directory = '_archive'.freeze
-
-    # Returns [Pathname] the archive_directory if set, otherwise the default_archive_directory
-    # Creates the archive directory if one is set
-    def archive_pathname(file_pathname)
-      if archive_directory
-        path = Pathname.new(archive_directory)
-        begin
-          path.mkpath unless path.exist?
-        rescue Errno::ENOENT => exc
-          raise(Errno::ENOENT, "DirmonJob failed to create archive directory: #{path}, #{exc.message}")
-        end
-        path.realpath
-      else
-        file_pathname.dirname.join(self.class.default_archive_directory).realdirpath
-      end
-    end
-
     # Passes each filename [Pathname] found that matches the pattern into the supplied block
-    def each(&block)
+    def each
       SemanticLogger.named_tagged(dirmon_entry: id.to_s) do
         # Case insensitive filename matching
         Pathname.glob(pattern, File::FNM_CASEFOLD).each do |pathname|
@@ -234,7 +201,7 @@ module RocketJob
           next if file_name.include?(self.class.default_archive_directory)
 
           # Security check?
-          if (whitelist_paths.size > 0) && whitelist_paths.none? { |whitepath| file_name.to_s.start_with?(whitepath) }
+          if whitelist_paths.size.positive? && whitelist_paths.none? { |whitepath| file_name.to_s.start_with?(whitepath) }
             logger.error "Skipping file: #{file_name} since it is not in any of the whitelisted paths: #{whitelist_paths.join(', ')}"
             next
           end
@@ -244,7 +211,7 @@ module RocketJob
             logger.error "Skipping file: #{file_name} since it is not writable by the current user. Must be able to delete/move the file after queueing the job"
             next
           end
-          block.call(pathname)
+          yield(pathname)
         end
       end
     end
@@ -264,7 +231,7 @@ module RocketJob
       end
     end
 
-    # Returns the Job to be queued
+    # Returns the Job to be created.
     def job_class
       return if job_class_name.nil?
       job_class_name.constantize
@@ -272,47 +239,44 @@ module RocketJob
       nil
     end
 
-    # Queues the job for the supplied pathname
+    # Archives the file and kicks off a proxy job to upload the file.
     def later(pathname)
-      if klass = job_class
-        logger.measure_info "Enqueued: #{name}, Job class: #{job_class_name}" do
-          job = klass.new(properties)
-          upload_file(job, pathname)
-          job.save!
-          job
-        end
-      else
-        raise(ArgumentError, "Cannot instantiate a class for: #{job_class_name.inspect}")
-      end
+      job_id             = BSON::ObjectId.new
+      archived_file_name = archive_file(job_id, pathname)
+
+      job = RocketJob::Jobs::UploadFileJob.create!(
+        job_class_name:     job_class_name,
+        properties:         properties,
+        description:        "#{name}: #{pathname.basename}",
+        upload_file_name:   archived_file_name.to_s,
+        original_file_name: pathname.to_s,
+        job_id:             job_id
+      )
+
+      logger.info(
+        message: 'Created RocketJob::Jobs::UploadFileJob',
+        payload: {
+          dirmon_entry_name:  name,
+          upload_file_name:   archived_file_name.to_s,
+          original_file_name: pathname.to_s,
+          job_class_name:     job_class_name,
+          job_id:             job_id.to_s,
+          upload_job_id:      job.id.to_s
+        }
+      )
+      job
     end
 
     private
 
-    class_attribute :whitelist_paths
-    self.whitelist_paths = Concurrent::Array.new
-
-    # Upload the file to the job
-    def upload_file(job, pathname)
-      if job.respond_to?(:upload)
-        # With RocketJob Pro the file can be uploaded directly into the Job itself
-        job.upload(pathname.to_s)
-        archive_directory ? archive_file(job, pathname) : pathname.unlink
-      else
-        upload_default(job, pathname)
-      end
+    # strip whitespaces from all variables that reference paths or patterns
+    def strip_whitespace
+      self.pattern           = pattern.strip unless pattern.nil?
+      self.archive_directory = archive_directory.strip unless archive_directory.nil?
     end
 
-    # Archives the file for a job where there was no #upload method
-    def upload_default(job, pathname)
-      full_file_name = archive_file(job, pathname)
-      if job.respond_to?(:upload_file_name=)
-        job.upload_file_name = full_file_name
-      elsif job.respond_to?(:full_file_name=)
-        job.full_file_name = full_file_name
-      else
-        raise(ArgumentError, "#{job_class_name} must either have attribute 'upload_file_name' or 'full_file_name'")
-      end
-    end
+    class_attribute :whitelist_paths
+    self.whitelist_paths = Concurrent::Array.new
 
     # Move the file to the archive directory
     #
@@ -322,14 +286,47 @@ module RocketJob
     #
     # Note:
     # - Works across partitions when the file and the archive are on different partitions
-    def archive_file(job, pathname)
+    def archive_file(job_id, pathname)
       target_path = archive_pathname(pathname)
       target_path.mkpath
-      target_file_name = target_path.join("#{job.id}_#{pathname.basename}")
+      target_file_name = target_path.join("#{job_id}_#{pathname.basename}")
       # In case the file is being moved across partitions
       FileUtils.move(pathname.to_s, target_file_name.to_s)
       target_file_name.to_s
     end
 
+    # Returns [Pathname] to the archive directory, and creates it if it does not exist.
+    #
+    # If `archive_directory` is a relative path, it is appended to the `file_pathname`.
+    # If `archive_directory` is an absolute path, it is returned as-is.
+    def archive_pathname(file_pathname)
+      path = Pathname.new(archive_directory)
+      path = file_pathname.dirname.join(archive_directory) if path.relative?
+
+      begin
+        path.mkpath unless path.exist?
+      rescue Errno::ENOENT => exc
+        raise(Errno::ENOENT, "DirmonJob failed to create archive directory: #{path}, #{exc.message}")
+      end
+      path.realpath
+    end
+
+    # Validates job_class is a Rocket Job
+    def job_is_a_rocket_job
+      klass = job_class
+      return if job_class_name.nil? || klass&.ancestors&.include?(RocketJob::Job)
+      errors.add(:job_class_name, "Job #{job_class_name} must be defined and inherit from RocketJob::Job")
+    end
+
+    # Does the job have all the supplied properties
+    def job_has_properties
+      klass = job_class
+      return unless klass
+
+      properties.each_pair do |k, _v|
+        next if klass.public_method_defined?("#{k}=".to_sym)
+        errors.add(:properties, "Unknown Property: Attempted to set a value for #{k.inspect} which is not allowed on the job #{job_class_name}")
+      end
+    end
   end
 end