rocketjob 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4f0d0679cdb6e1b6fb284b1dfd2c90b5b201ffe7
-  data.tar.gz: ea135d888befde2dd286242a8c5bb1c338a45128
+  metadata.gz: 9720cccfb90fd2afdf83b04785f5133c97078fbd
+  data.tar.gz: 352782e7591790ada1f3ab8b584202f1d666b65d
 SHA512:
-  metadata.gz: c8640b98d22541f3739a4a61fd726981a3f349c0b57ecdaf88e006839e5ac98d3505144723df65a7f98505951a9f971c0d393669fa655e63b445221a64d8e485
-  data.tar.gz: 88665fc325e161c4cb3d4dbdf506250c3891742199c6e3d9c0b5f74064f245faf08b21213222ff0679fff24ee2cea9bef4b2af3ae7da1f146dac57e472800d30
+  metadata.gz: 47cfcc68c411faace343f526240c036e1655c6d55e9de1cdbaee5292ba09400de0b442db667a7a894a6489d96d2494e721d5cbcdec5c9fab0358e158592dfc52
+  data.tar.gz: 431732d7fe08636f15654108b8d6efcc134f588fcedf93cc9b635b5ed7205d2d7add0b80c27f14cb85c3b792b115e9ec2eb39b39756bac2fc0fd09b26c36eb04

data/Rakefile

@@ -1,21 +1,20 @@
 require 'rake/clean'
 require 'rake/testtask'
 
-$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
-require 'rocket_job/version'
+require_relative 'lib/rocket_job/version'
 
 task :gem do
-  system "gem build rocketjob.gemspec"
+  system 'gem build rocketjob.gemspec'
 end
 
-task :publish => :gem do
+task publish: :gem do
   system "git tag -a v#{RocketJob::VERSION} -m 'Tagging #{RocketJob::VERSION}'"
-  system "git push --tags"
+  system 'git push --tags'
   system "gem push rocketjob-#{RocketJob::VERSION}.gem"
   system "rm rocketjob-#{RocketJob::VERSION}.gem"
 end
 
-desc "Run Test Suite"
+desc 'Run Test Suite'
 task :test do
   Rake::TestTask.new(:functional) do |t|
     t.test_files = FileList['test/**/*_test.rb']
@@ -25,4 +24,4 @@ task :test do
   Rake::Task['functional'].invoke
 end
 
-task :default => :test
+task default: :test

data/lib/rocket_job/cli.rb

@@ -5,25 +5,24 @@ module RocketJob
     attr_reader :name, :threads, :environment, :pidfile, :directory, :quiet
 
     def initialize(argv)
-      @name             = nil
-      @threads          = nil
-
-      @quiet            = false
-      @environment      = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
-      @pidfile          = nil
-      @directory        = '.'
+      @name        = nil
+      @threads     = nil
+      @quiet       = false
+      @environment = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      @pidfile     = nil
+      @directory   = '.'
       parse(argv)
     end
 
     # Run a RocketJob::Worker from the command line
     def run
-      SemanticLogger.add_appender(STDOUT,  &SemanticLogger::Appender::Base.colorized_formatter) unless quiet
+      SemanticLogger.add_appender(STDOUT, &SemanticLogger::Appender::Base.colorized_formatter) unless quiet
       boot_rails if defined?(:Rails)
       write_pidfile
 
-      opts = {}
-      opts[:name]             = name if name
-      opts[:max_threads]      = threads if threads
+      opts               = {}
+      opts[:name]        = name if name
+      opts[:max_threads] = threads if threads
       Worker.run(opts)
     end
 
@@ -33,7 +32,7 @@ module RocketJob
       if Rails.configuration.eager_load
         RocketJob::Worker.logger.benchmark_info('Eager loaded Rails and all Engines') do
           Rails.application.eager_load!
-          Rails::Engine.subclasses.each { |engine| engine.eager_load! }
+          Rails::Engine.subclasses.each(&:eager_load!)
         end
       end
     end
@@ -41,18 +40,18 @@ module RocketJob
     # Create a PID file if requested
     def write_pidfile
       return unless pidfile
-      pid = $$
+      pid = $PID
       File.open(pidfile, 'w') { |f| f.puts(pid) }
 
       # Remove pidfile on exit
       at_exit do
-        File.delete(pidfile) if pid == $$
+        File.delete(pidfile) if pid == $PID
       end
     end
 
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
-      parser = OptionParser.new do |o|
+      parser        = OptionParser.new do |o|
         o.on('-n', '--name NAME', 'Unique Name of this worker instance (Default: hostname:PID)') { |arg| @name = arg }
         o.on('-t', '--threads COUNT', 'Number of worker threads to start') { |arg| @threads = arg.to_i }
         o.on('-q', '--quiet', 'Do not write to stdout, only to logfile. Necessary when running as a daemon') { @quiet = true }

data/lib/rocket_job/concerns/singleton.rb

@@ -0,0 +1,33 @@
+# encoding: UTF-8
+require 'active_support/concern'
+
+# Worker behavior for a job
+module RocketJob
+  module Concerns
+    module Singleton
+      extend ActiveSupport::Concern
+
+      included do
+        # Start the single instance of this job
+        #
+        # Returns true if the job was started
+        # Returns false if the job is already running and doe not need to be started
+        def self.start(*args, &block)
+          # Prevent multiple Jobs of the same class from running at the same time
+          return false if where(state: [:running, :queued]).count > 0
+
+          perform_later(*args, &block)
+          true
+        end
+
+        # TODO Make :perform_later, :perform_now, :perform, :now protected/private
+        #      class << self
+        #        # Ensure that only one instance of the job is running.
+        #        protected :perform_later, :perform_now, :perform, :now
+        #      end
+        #self.send(:protected, :perform_later)
+
+      end
+    end
+  end
+end

data/lib/rocket_job/concerns/worker.rb

@@ -7,9 +7,6 @@ module RocketJob
       def self.included(base)
         base.extend ClassMethods
         base.class_eval do
-          # While working on a slice, the current slice is available via this reader
-          attr_reader :rocket_job_slice
-
           @rocket_job_defaults = nil
         end
       end
@@ -28,7 +25,7 @@ module RocketJob
 
         # Create a job and process it immediately in-line by this thread
         def now(method, *args, &block)
-          job = build(method, *args, &block)
+          job    = build(method, *args, &block)
           worker = RocketJob::Worker.new(name: 'inline')
           worker.started
           job.start
@@ -71,11 +68,61 @@ module RocketJob
           @rocket_job_defaults = block
           self
         end
-      end
 
-      def rocket_job_csv_parser
-        # TODO Change into an instance variable once CSV handling has been re-worked
-        RocketJob::Utility::CSVRow.new
+        # Returns the next job to work on in priority based order
+        # Returns nil if there are currently no queued jobs, or processing batch jobs
+        #   with records that require processing
+        #
+        # Parameters
+        #   worker_name [String]
+        #     Name of the worker that will be processing this job
+        #
+        #   skip_job_ids [Array<BSON::ObjectId>]
+        #     Job ids to exclude when looking for the next job
+        #
+        # Note:
+        #   If a job is in queued state it will be started
+        def next_job(worker_name, skip_job_ids = nil)
+          query        = {
+            '$and' => [
+              {
+                '$or' => [
+                  {'state' => 'queued'}, # Jobs
+                  {'state' => 'running', 'sub_state' => :processing} # Slices
+                ]
+              },
+              {
+                '$or' => [
+                  {run_at: {'$exists' => false}},
+                  {run_at: {'$lte' => Time.now}}
+                ]
+              }
+            ]
+          }
+          query['_id'] = {'$nin' => skip_job_ids} if skip_job_ids && skip_job_ids.size > 0
+
+          while (doc = find_and_modify(
+            query:  query,
+            sort:   [['priority', 'asc'], ['created_at', 'asc']],
+            update: {'$set' => {'worker_name' => worker_name, 'state' => 'running'}}
+          ))
+            job = load(doc)
+            if job.running?
+              return job
+            else
+              if job.expired?
+                job.destroy
+                logger.info "Destroyed expired job #{job.class.name}, id:#{job.id}"
+              else
+                # Also update in-memory state and run call-backs
+                job.start
+                job.set(started_at: job.started_at)
+                return job
+              end
+            end
+          end
+        end
+
       end
 
       # Works on this job
@@ -87,22 +134,24 @@ module RocketJob
       #
       # Thread-safe, can be called by multiple threads at the same time
       def work(worker)
-        raise 'Job must be started before calling #work' unless running?
+        raise(ArgumentError, 'Job must be started before calling #work') unless running?
         begin
           # before_perform
           call_method(perform_method, arguments, event: :before, log_level: log_level)
 
           # perform
-          call_method(perform_method, arguments, log_level: log_level)
+          ret = call_method(perform_method, arguments, log_level: log_level)
           if self.collect_output?
-            self.output = (result.is_a?(Hash) || result.is_a?(BSON::OrderedHash)) ? result : { result: result }
+            self.result = (ret.is_a?(Hash) || ret.is_a?(BSON::OrderedHash)) ? ret : {result: ret}
           end
 
           # after_perform
           call_method(perform_method, arguments, event: :after, log_level: log_level)
+
           complete!
-        rescue Exception => exc
-          set_exception(worker.name, exc)
+        rescue StandardError => exc
+          fail!(worker.name, exc) unless failed?
+          logger.error("Exception running #{self.class.name}##{perform_method}", exc)
           raise exc if RocketJob::Config.inline_mode
         end
         false
@@ -131,27 +180,28 @@ module RocketJob
       #       Log level to apply to silence logging during the call
       #       Default: nil ( no change )
       #
-      def call_method(method, arguments, options={})
-        options               = options.dup
-        event                 = options.delete(:event)
-        log_level             = options.delete(:log_level)
+      def call_method(method, arguments, options = {})
+        options   = options.dup
+        event     = options.delete(:event)
+        log_level = options.delete(:log_level)
         raise(ArgumentError, "Unknown #{self.class.name}#call_method options: #{options.inspect}") if options.size > 0
 
         the_method = event.nil? ? method : "#{event}_#{method}".to_sym
         if respond_to?(the_method)
           method_name = "#{self.class.name}##{the_method}"
           logger.info "Start #{method_name}"
-          logger.benchmark_info("Completed #{method_name}",
+          logger.benchmark_info(
+            "Completed #{method_name}",
             metric:             "rocketjob/#{self.class.name.underscore}/#{the_method}",
             log_exception:      :full,
             on_exception_level: :error,
             silence:            log_level
           ) do
-            self.send(the_method, *arguments)
+            send(the_method, *arguments)
           end
         end
       end
 
     end
   end
-end
+end

data/lib/rocket_job/config.rb

@@ -14,7 +14,7 @@ module RocketJob
     sync_cattr_reader(:instance) do
       begin
         first || create
-      rescue Exception => exc
+      rescue StandardError
         # In case another process has already created the first document
         first
       end
@@ -24,6 +24,7 @@ module RocketJob
     # No worker processes will be created, nor threads created
     sync_cattr_accessor(:inline_mode) { false }
 
+    # @formatter:off
     # The maximum number of worker threads to create on any one worker
     key :max_worker_threads,         Integer, default: 10
 
@@ -47,6 +48,7 @@ module RocketJob
     # Limit the number of workers per job class per worker
     #    'class_name' / group => 100
     #key :limits, Hash
+    # @formatter:on
 
     # Replace the MongoMapper default mongo connection for holding jobs
     def self.mongo_connection=(connection)

data/lib/rocket_job/dirmon_entry.rb

@@ -1,28 +1,39 @@
+require 'thread_safe'
+require 'pathname'
+require 'fileutils'
 module RocketJob
   class DirmonEntry
     include MongoMapper::Document
+    include AASM
 
-    # Name for this path entry used to identify this DirmonEntry
-    # in the user interface
+    # @formatter:off
+    # User defined name used to identify this DirmonEntry in Mission Control
     key :name,               String
 
-    # Wildcard path to search for files in
+    # Pattern for finding files
     #
-    # Example:
-    #   input_files/process1/*.csv*
+    # Example: All files ending in '.csv' in the input_files/process1 directory
+    #   input_files/process1/*.csv
+    #
+    # Example: All files in the input_files/process1 directory and all sub-directories
     #   input_files/process2/**/*
     #
-    # For details on valid path values, see: http://ruby-doc.org/core-2.2.2/Dir.html#method-c-glob
+    # Example: All files in the input_files/process2 directory with .csv or .txt extensions
+    #   input_files/process2/*.{csv,txt}
+    #
+    # For details on valid pattern values, see: http://ruby-doc.org/core-2.2.2/Dir.html#method-c-glob
     #
     # Note
-    # - If there are no '*' in the path then an exact filename match is expected
-    key :path,               String
+    # - If there is no '*' in the pattern then an exact filename match is expected
+    # - The pattern is not validated to ensure the path exists, it will be validated against the
+    #   `whitelist_paths` when processed by DirmonJob
+    key :pattern,            String
 
-    # Job to start
+    # Job to enqueue for processing for every file that matches the pattern
     #
     # Example:
     #   "ProcessItJob"
-    key :job_name,          String
+    key :job_class_name,     String
 
     # Any user supplied arguments for the method invocation
     # All keys must be UTF-8 strings. The values can be any valid BSON type:
@@ -52,46 +63,265 @@ module RocketJob
     #
     # If supplied, the file will be moved to this directory before the job is started
     # If the file was in a sub-directory, the corresponding sub-directory will
-    # be created in the archive directory, if the path being scanned for files
-    # is a relative path. (I.e. Does not start with '/') .
+    # be created in the archive directory.
     key :archive_directory,  String
 
-    # Allow a monitoring path to be temporarily disabled
-    key :enabled,            Boolean, default: true
-
     # Method to perform on the job, usually :perform
     key :perform_method,     Symbol, default: :perform
 
-    # Returns the Job to be queued
-    def job_class
-      job_name.nil? ? nil : job_name.constantize
+    # If this DirmonEntry is in the failed state, exception contains the cause
+    one :exception,          class_name: 'RocketJob::JobException'
+
+    # The maximum number of files that should ever match during a single poll of the pattern.
+    #
+    # Too many files could be as a result of an invalid pattern specification.
+    # Exceeding this number will result in an exception being logged in a failed Dirmon instance.
+    # Dirmon processing will continue with new instances.
+    # TODO: Implement max_hits
+    #key :max_hits,           Integer, default: 100
+
+    #
+    # Read-only attributes
+    #
+
+    # Current state, as set by AASM
+    key :state,              Symbol, default: :pending
+
+    # State Machine events and transitions
+    #
+    # :pending -> :enabled  -> :disabled
+    #                       -> :failed
+    #          -> :failed   -> :active
+    #                       -> :disabled
+    #          -> :disabled -> :active
+    aasm column: :state do
+      # DirmonEntry is `pending` until it is approved
+      state :pending, initial: true
+
+      # DirmonEntry is Enabled and will be included by DirmonJob
+      state :enabled
+
+      # DirmonEntry failed during processing and requires manual intervention
+      # See the exception for the reason for failing this entry
+      # For example: access denied, whitelist_path security violation, etc.
+      state :failed
+
+      # DirmonEntry has been manually disabled
+      state :disabled
+
+      event :enable do
+        transitions from: :pending,  to: :enabled
+        transitions from: :disabled, to: :enabled
+      end
+
+      event :disable do
+        transitions from: :enabled, to: :disabled
+        transitions from: :failed,  to: :disabled
+      end
+
+      event :fail do
+        transitions from: :enabled, to: :failed
+      end
     end
 
-    validates_presence_of :path, :job_name
+    # @formatter:on
+    validates_presence_of :pattern, :job_class_name, :perform_method
 
-    validates_each :job_name do |record, attr, value|
-      exists = false
-      begin
-        exists = value.nil? ? false : value.constantize.ancestors.include?(RocketJob::Job)
-      rescue NameError => exc
+    validates_each :perform_method do |record, attr, value|
+      if (klass = record.job_class) && !klass.instance_method(value)
+        record.errors.add(attr, "Method not implemented by #{record.job_class_name}")
       end
-      record.errors.add(attr, 'job_name must be defined and must be derived from RocketJob::Job') unless exists
+    end
+
+    validates_each :job_class_name do |record, attr, value|
+      exists =
+        begin
+          value.nil? ? false : record.job_class.ancestors.include?(RocketJob::Job)
+        rescue NameError
+          false
+        end
+      record.errors.add(attr, 'job_class_name must be defined and must be derived from RocketJob::Job') unless exists
     end
 
     validates_each :arguments do |record, attr, value|
-      if klass = record.job_class
+      if (klass = record.job_class)
         count = klass.argument_count(record.perform_method)
-        record.errors.add(attr, "There must be #{count} argument(s)") if  value.size != count
+        record.errors.add(attr, "There must be #{count} argument(s)") if value.size != count
       end
     end
 
     validates_each :properties do |record, attr, value|
-      if record.job_name && (methods = record.job_class.instance_methods)
-        value.each_pair do |key, value|
-          record.errors.add(attr, "Unknown property: #{key.inspect} with value: #{value}") unless methods.include?("#{key}=".to_sym)
+      if record.job_class && (methods = record.job_class.instance_methods)
+        value.each_pair do |k, v|
+          record.errors.add(attr, "Unknown property: #{k.inspect} with value: #{v}") unless methods.include?("#{k}=".to_sym)
         end
       end
     end
 
+    # Create indexes
+    def self.create_indexes
+      # Unique index on pattern to help prevent two entries from scanning the same files
+      ensure_index({pattern: 1}, background: true, unique: true)
+    end
+
+    # Security Settings
+    #
+    # A whitelist of paths from which to process files.
+    # This prevents accidental or malicious `pattern`s from processing files from anywhere
+    # in the system that the user under which Dirmon is running can access.
+    #
+    # All resolved `pattern`s must start with one of the whitelisted path, otherwise they will be rejected
+    #
+    # Note:
+    # - If no whitelist paths have been added, then a whitelist check is _not_ performed
+    # - Relative paths can be used, but are not considered safe since they can be manipulated
+    # - These paths should be assigned in an initializer and not editable via the Web UI to ensure
+    #   that they are not tampered with
+    #
+    # Default: [] ==> Do not enforce whitelists
+    #
+    # Returns [Array<String>] a copy of the whitelisted paths
+    def self.whitelist_paths
+      @@whitelist_paths.dup
+    end
+
+    # Add a path to the whitelist
+    # Raises: Errno::ENOENT: No such file or directory
+    def self.add_whitelist_path(path)
+      # Confirms that path exists
+      path = Pathname.new(path).realpath.to_s
+      @@whitelist_paths << path
+      @@whitelist_paths.uniq!
+      path
+    end
+
+    # Deletes a path from the whitelist paths
+    # Raises: Errno::ENOENT: No such file or directory
+    def self.delete_whitelist_path(path)
+      # Confirms that path exists
+      path = Pathname.new(path).realpath.to_s
+      @@whitelist_paths.delete(path)
+      @@whitelist_paths.uniq!
+      path
+    end
+
+    # The default archive directory that is used when the job being queued does not respond
+    # to #file_store_upload or #upload, and do not have an `archive_directory` specified in this entry
+    cattr_accessor :default_archive_directory
+
+    @@default_archive_directory = '_archive'.freeze
+
+    # Returns [Pathname] the archive_directory if set, otherwise the default_archive_directory
+    def archive_pathname
+      Pathname.new(archive_directory || self.class.default_archive_directory)
+    end
+
+    # Passes each filename [Pathname] found that matches the pattern into the supplied block
+    def each(&block)
+      logger.tagged("DirmonEntry:#{id}") do
+        Pathname.glob(pattern).each do |pathname|
+          next if pathname.directory?
+          pathname  = pathname.realpath
+          file_name = pathname.to_s
+
+          # Skip archive directories
+          next if file_name.start_with?(archive_pathname.realpath.to_s)
+
+          # Security check?
+          if (@@whitelist_paths.size > 0) && @@whitelist_paths.none? { |whitepath| file_name.start_with?(whitepath) }
+            logger.warn "Ignoring file: #{file_name} since it is not in any of the whitelisted paths: #{whitelist_paths.join(', ')}"
+            next
+          end
+
+          # File must be writable so it can be removed after processing
+          unless pathname.writable?
+            logger.warn "Ignoring file: #{file_name} since it is not writable by the current user. Must be able to delete/move the file after queueing the job"
+            next
+          end
+          block.call(pathname)
+        end
+      end
+    end
+
+    # Set exception information for this DirmonEntry and fail it
+    def fail_with_exception!(worker_name, exc_or_message)
+      if exc_or_message.is_a?(Exception)
+        self.exception        = JobException.from_exception(exc_or_message)
+        exception.worker_name = worker_name
+      else
+        build_exception(
+          class_name:  'RocketJob::DirmonEntryException',
+          message:     exc_or_message,
+          backtrace:   [],
+          worker_name: worker_name
+        )
+      end
+      fail!
+    end
+
+    @@whitelist_paths = ThreadSafe::Array.new
+
+    # Returns the Job to be queued
+    def job_class
+      return if job_class_name.nil?
+      job_class_name.constantize
+    rescue NameError
+      nil
+    end
+
+    # Queues the job for the supplied pathname
+    def later(pathname)
+      job_class.perform_later(*arguments) do |job|
+        job.perform_method = perform_method
+        # Set properties
+        properties.each_pair { |k, v| job.send("#{k}=".to_sym, v) }
+
+        upload_file(job, pathname)
+      end
+    end
+
+    protected
+
+    # Upload the file to the job
+    def upload_file(job, pathname)
+      if job.respond_to?(:file_store_upload)
+        # Allow the job to determine what to do with the file
+        # Pass the pathname as a string, not a Pathname (IO) instance
+        # so that it can read the file directly
+        job.file_store_upload(pathname.to_s)
+        archive_directory ? archive_file(job, pathname) : pathname.unlink
+      elsif job.respond_to?(:upload)
+        # With RocketJob Pro the file can be uploaded directly into the Job itself
+        job.upload(pathname.to_s)
+        archive_directory ? archive_file(job, pathname) : pathname.unlink
+      else
+        upload_default(job, pathname)
+      end
+    end
+
+    # Archives the file for a job where there was no #file_store_upload or #upload method
+    def upload_default(job, pathname)
+      # The first argument must be a hash
+      job.arguments << {} if job.arguments.size == 0
+      job.arguments.first[:full_file_name] = archive_file(job, pathname)
+    end
+
+    # Move the file to the archive directory
+    #
+    # The archived file name is prefixed with the job id
+    #
+    # Returns [String] the fully qualified archived file name
+    #
+    # Note:
+    # - Works across partitions when the file and the archive are on different partitions
+    def archive_file(job, pathname)
+      target_path = archive_pathname
+      target_path.mkpath
+      target_file_name = target_path.join("#{job.id}_#{pathname.basename}")
+      # In case the file is being moved across partitions
+      FileUtils.move(pathname.to_s, target_file_name.to_s)
+      target_file_name.to_s
+    end
+
   end
 end