rocketjob 6.0.0 → 6.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e313f192b854d066258a614ceac1131851c8df94c7e08f7cea6681fff6946d69
-  data.tar.gz: 10804682bee08715671696db4610ce4f93679398398bc385e93619f5a3aca715
+  metadata.gz: fc1f3d316be0aa058cee5b22c116872a80ea5bdaf7d72db447ae13781ba6b370
+  data.tar.gz: 87f56a455792e45ab2ad706f0c8bd625590df9942806bbd308c5145d07e94973
 SHA512:
-  metadata.gz: 158675e5ddec87a8b277708887b037746e3f1573569edd5a8959eabdf5668b144553cbe6164844f2cf69bc603798b3b5b052697506dad8a29e8477afc62cc45f
-  data.tar.gz: 680efe5603de3649b7e09340a545d2e1df1e02697d34af9451869310e9f2a87bbd05423686bb7d40d60f104553f7311721870e52ac2171bb1491b3a8decaf439
+  metadata.gz: f928a3afb886c39893ef5b5d2e13f71405c83df53c3b02826e474326c9171f87c84e595fc179a0e22cbc69d846f1da12eb268026fc06bcaf609920326159c6f4
+  data.tar.gz: 3b90508c45e7a09739ae53e5bb66992d80251a5b5668eb568eeba30a8da8d109bbb162b2db66e93f82ce3deb4f6cc9425175d0f9c41adb351f6c3885768db3dc

data/lib/rocket_job/batch/categories.rb

@@ -75,27 +75,26 @@ module RocketJob
         return category_name if category_name.is_a?(Category::Input)
         raise(ArgumentError, "Cannot supply Output Category to input category") if category_name.is_a?(Category::Output)
 
+        # Initialize categories when this method is called before initialization is complete
+        rocketjob_categories_assign if input_categories.empty?
+
         category_name = category_name.to_sym
         # find does not work against this association
         input_categories.each { |category| return category if category.name == category_name }
 
-        unless category_name == :main
-          raise(
-            ArgumentError,
-            "Unknown Input Category: #{category_name.inspect}. Registered categories: #{input_categories.collect(&:name).join(',')}"
-          )
-        end
-
-        # Auto-register main input category when not defined
-        category = Category::Input.new(job: self)
-        self.input_categories << category
-        category
+        raise(
+          ArgumentError,
+          "Unknown Input Category: #{category_name.inspect}. Registered categories: #{input_categories.collect(&:name).join(',')}"
+        )
       end
 
       def output_category(category_name = :main)
         return category_name if category_name.is_a?(Category::Output)
         raise(ArgumentError, "Cannot supply Input Category to output category") if category_name.is_a?(Category::Input)
 
+        # Initialize categories when this method is called before initialization is complete
+        rocketjob_categories_assign if output_categories.empty? && self.class.defined_output_categories
+
         category_name = category_name.to_sym
         # .find does not work against this association
         output_categories.each { |category| return category if category.name == category_name }
@@ -154,7 +153,7 @@ module RocketJob
             end
         end
 
-        return if !self.class.defined_output_categories || !output_categories.empty?
+        return if !output_categories.empty? || !self.class.defined_output_categories
 
         # Input categories defaults to nil if none was set in the class
         self.output_categories = self.class.defined_output_categories.deep_dup
@@ -164,7 +163,6 @@ module RocketJob
       def rocketjob_categories_output_render
         return if @rocket_job_output.nil?
 
-        # TODO: ..
         return unless output_categories
         return if output_categories.empty?
 

data/lib/rocket_job/dirmon_entry.rb

@@ -71,6 +71,7 @@ module RocketJob
     validates_presence_of :pattern, :job_class_name, :archive_directory
     validate :job_is_a_rocket_job
     validate :job_has_properties
+    validates_uniqueness_of :pattern, :name
 
     # State Machine events and transitions
     #
@@ -97,6 +98,7 @@ module RocketJob
       event :enable do
         transitions from: :pending, to: :enabled
         transitions from: :disabled, to: :enabled
+        transitions from: :failed, to: :enabled
       end
 
       event :disable do

data/lib/rocket_job/job_exception.rb

@@ -23,7 +23,7 @@ module RocketJob
       new(
         args.merge(
           class_name: exc.class.name,
-          message:    exc.message,
+          message:    exc.message.to_s.encode("UTF-8", replace: ""),
           backtrace:  exc.backtrace || []
         )
       )

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -59,23 +59,31 @@ module RocketJob
       def check_directories
         new_file_names = {}
         DirmonEntry.enabled.each do |dirmon_entry|
-          dirmon_entry.each do |path|
-            # Skip file size checking since S3 files are only visible once completely uploaded.
-            unless path.partial_files_visible?
-              logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
-              dirmon_entry.later(path)
-              next
-            end
+          check_entry(dirmon_entry, new_file_names)
+        end
+        self.previous_file_names = new_file_names
+      end
 
-            # BSON Keys cannot contain periods
-            key           = path.to_s.tr(".", "_")
-            previous_size = previous_file_names[key]
-            # Check every few minutes for a file size change before trying to process the file.
-            size                = check_file(dirmon_entry, path, previous_size)
-            new_file_names[key] = size if size
+      def check_entry(dirmon_entry, file_names)
+        dirmon_entry.each do |path|
+          # Skip file size checking since S3 files are only visible once completely uploaded.
+          unless path.partial_files_visible?
+            logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
+            dirmon_entry.later(path)
+            next
           end
+
+          # BSON Keys cannot contain periods
+          key           = path.to_s.tr(".", "_")
+          previous_size = previous_file_names[key]
+          # Check every few minutes for a file size change before trying to process the file.
+          size            = check_file(dirmon_entry, path, previous_size)
+          file_names[key] = size if size
         end
-        self.previous_file_names = new_file_names
+      rescue StandardError => e
+        logger.error("Dirmon Entry: #{dirmon_entry.id} failed. Moved to `failed` state to prevent processing again without manual intervention.", e)
+        dirmon_entry.fail(worker_name, e)
+        dirmon_entry.save(validate: false)
       end
 
       # Checks if a file should result in starting a job

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -145,6 +145,10 @@ module RocketJob
       rescue Exception => e
         errors.add(field, "Failed to load :#{field}, #{e.inspect}")
       end
+
+      # Allow multiple instances of this job to run with the same cron schedule
+      def rocket_job_cron_singleton_check
+      end
     end
   end
 end

data/lib/rocket_job/jobs/on_demand_job.rb

@@ -97,6 +97,10 @@ module RocketJob
       rescue Exception => e
         errors.add(:code, "Failed to parse :code, #{e.inspect}")
       end
+
+      # Allow multiple instances of this job to run with the same cron schedule
+      def rocket_job_cron_singleton_check
+      end
     end
   end
 end

data/lib/rocket_job/plugins/throttle_dependent_jobs.rb

@@ -13,14 +13,20 @@ module RocketJob
       included do
         field :dependent_jobs, type: Array, class_attribute: true, user_editable: true, copy_on_restart: true
 
-        define_throttle :dependent_job_exists?
-        define_batch_throttle :dependent_job_exists? if respond_to?(:define_batch_throttle)
+        define_throttle :dependent_jobs_running?
+        define_batch_throttle :dependent_jobs_running? if respond_to?(:define_batch_throttle)
+      end
+
+      class_methods do
+        def depends_on_job(*jobs)
+          self.dependent_jobs = Array(jobs).collect(&:to_s)
+        end
       end
 
       private
 
       # Checks if there are any dependent jobs are running
-      def dependent_job_exists?
+      def dependent_jobs_running?
         return false if dependent_jobs.blank?
 
         jobs_count = RocketJob::Job.running.where(:_type.in => dependent_jobs).count

data/lib/rocket_job/subscribers/secret_config.rb

@@ -0,0 +1,17 @@
+module RocketJob
+  module Subscribers
+    # Cause all instances to refresh their in-memory copy
+    # of the Secret Config Registry
+    #
+    #   RocketJob::Subscribers::SecretConfig.publish(:refresh)
+    class SecretConfig
+      include RocketJob::Subscriber
+
+      def refresh
+        logger.measure_info "Refreshed Secret Config" do
+          ::SecretConfig.refresh!
+        end
+      end
+    end
+  end
+end

data/lib/rocket_job/supervisor.rb

@@ -34,6 +34,7 @@ module RocketJob
       logger.info "Rocket Job Server started"
 
       event_listener = Thread.new { Event.listener }
+      Subscribers::SecretConfig.subscribe if defined?(SecretConfig)
       Subscribers::Server.subscribe(self) do
         Subscribers::Worker.subscribe(self) do
           Subscribers::Logger.subscribe do

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "6.0.0".freeze
+  VERSION = "6.1.0".freeze
 end

data/lib/rocketjob.rb

@@ -83,12 +83,13 @@ module RocketJob
     autoload :UploadFileJob,         "rocket_job/jobs/upload_file_job"
 
     module ReEncrypt
-      autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
+      autoload :RelationalJob,       "rocket_job/jobs/re_encrypt/relational_job"
     end
   end
 
   module Subscribers
     autoload :Logger,                "rocket_job/subscribers/logger"
+    autoload :SecretConfig,          "rocket_job/subscribers/secret_config"
     autoload :Server,                "rocket_job/subscribers/server"
     autoload :Worker,                "rocket_job/subscribers/worker"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 6.0.0
+  version: 6.1.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-25 00:00:00.000000000 Z
+date: 2021-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -84,14 +84,14 @@ dependencies:
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.7'
 - !ruby/object:Gem::Dependency
@@ -205,6 +205,7 @@ files:
 - lib/rocket_job/sliced/writer/output.rb
 - lib/rocket_job/subscriber.rb
 - lib/rocket_job/subscribers/logger.rb
+- lib/rocket_job/subscribers/secret_config.rb
 - lib/rocket_job/subscribers/server.rb
 - lib/rocket_job/subscribers/worker.rb
 - lib/rocket_job/supervisor.rb