rocketjob 6.0.2 → 6.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10ef66caa28110774987fdeff06af1a4906ed724bc307bf55d012e1104a9eca8
-  data.tar.gz: b0384cccd5d694a3431c6c9cb7d004603c4b79865929b8c2bb249b4b66e937a2
+  metadata.gz: ca9a7af0f4448efdb418e738f95bc5f885d2399bdfd50ce3d0533c6928f3f78f
+  data.tar.gz: e0f1bc3fb3c60989f22ff4d75c1ed86c8f7a998bdc5bc02f0f8616dd9631029e
 SHA512:
-  metadata.gz: 331a1c21ad9c2b481e67d5d3857e3158dd23a7bb6524dd872773e0aad0e0c5edb4e02d4a60be6498bc212e2517c81c027ce7f22d6b378e9d5772c391669c4347
-  data.tar.gz: e46cdffa443cc75d5dbed5f396ec84fb19fd12bad5b48bbddad7d310526a941bd837f778da8391460e40c575caed62c6e0329adc99ae8c58b07ada694cb1684c
+  metadata.gz: 79e914a57bbdae9026cbe783d8a1743e538bb2b1ffebacb527561ef961174eaa24693e850e0b72ee9ae113611db96fe1e0e2edb85d6395cbc057b7fc17bf11e9
+  data.tar.gz: f6a8ba0b5060f54072a68fc396e43a3b9ba1c53b58e6b282006e111792e612d301b737279a344be252fc659724b70e12cff5400f49eb47316d38eda0571788f3

data/lib/rocket_job/batch/io.rb

@@ -466,6 +466,8 @@ module RocketJob
             output_collection.download(header_line: binary_header_line) { |record| io.write(record) }
           end
         else
+          # TODO: Add category to named tags to aid problem determination
+          # And RJ Download metric with duration
           IOStreams.new(stream || category.file_name).writer(:line, **args) do |io|
             output_collection.download(header_line: header_line) { |record| io << record }
           end

data/lib/rocket_job/dirmon_entry.rb

@@ -71,6 +71,7 @@ module RocketJob
     validates_presence_of :pattern, :job_class_name, :archive_directory
     validate :job_is_a_rocket_job
     validate :job_has_properties
+    validates_uniqueness_of :pattern, :name
 
     # State Machine events and transitions
     #
@@ -97,6 +98,7 @@ module RocketJob
       event :enable do
         transitions from: :pending, to: :enabled
         transitions from: :disabled, to: :enabled
+        transitions from: :failed, to: :enabled
       end
 
       event :disable do

data/lib/rocket_job/extensions/mongo/logging.rb

@@ -6,22 +6,26 @@ module Mongo
       include SemanticLogger::Loggable
       logger.name = "Mongo"
 
+      undef :started
       def started(event)
         @event_command = event.command
       end
 
+      undef :succeeded
       def succeeded(event)
         logger.debug(message:  prefix(event),
                      duration: (event.duration * 1000),
                      payload:  @event_command)
       end
 
+      undef :failed
       def failed(event)
         logger.debug(message:  "#{prefix(event)} Failed: #{event.message}",
                      duration: (event.duration * 1000),
                      payload:  @event_command)
       end
 
+      undef :prefix
       def prefix(event)
         "#{event.address} | #{event.database_name}.#{event.command_name}"
       end

data/lib/rocket_job/jobs/copy_file_job.rb

@@ -82,7 +82,10 @@ module RocketJob
       end
 
       def apply_streams(path, streams)
-        streams.each_pair { |stream, args| path.stream(stream.to_sym, args.nil? ? {} : decode_args(args)) }
+        streams.each_pair do |stream, args|
+          stream_args = args.nil? ? {} : decode_args(args)
+          path.stream(stream.to_sym, **stream_args)
+        end
       end
 
       def decode_args(args)

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -59,23 +59,31 @@ module RocketJob
       def check_directories
         new_file_names = {}
         DirmonEntry.enabled.each do |dirmon_entry|
-          dirmon_entry.each do |path|
-            # Skip file size checking since S3 files are only visible once completely uploaded.
-            unless path.partial_files_visible?
-              logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
-              dirmon_entry.later(path)
-              next
-            end
+          check_entry(dirmon_entry, new_file_names)
+        end
+        self.previous_file_names = new_file_names
+      end
 
-            # BSON Keys cannot contain periods
-            key           = path.to_s.tr(".", "_")
-            previous_size = previous_file_names[key]
-            # Check every few minutes for a file size change before trying to process the file.
-            size                = check_file(dirmon_entry, path, previous_size)
-            new_file_names[key] = size if size
+      def check_entry(dirmon_entry, file_names)
+        dirmon_entry.each do |path|
+          # Skip file size checking since S3 files are only visible once completely uploaded.
+          unless path.partial_files_visible?
+            logger.info("File: #{path}. Starting: #{dirmon_entry.job_class_name}")
+            dirmon_entry.later(path)
+            next
           end
+
+          # BSON Keys cannot contain periods
+          key           = path.to_s.tr(".", "_")
+          previous_size = previous_file_names[key]
+          # Check every few minutes for a file size change before trying to process the file.
+          size            = check_file(dirmon_entry, path, previous_size)
+          file_names[key] = size if size
         end
-        self.previous_file_names = new_file_names
+      rescue StandardError => e
+        logger.error("Dirmon Entry: #{dirmon_entry.id} failed. Moved to `failed` state to prevent processing again without manual intervention.", e)
+        dirmon_entry.fail(worker_name, e)
+        dirmon_entry.save(validate: false)
       end
 
       # Checks if a file should result in starting a job

data/lib/rocket_job/jobs/on_demand_batch_job.rb

@@ -145,6 +145,10 @@ module RocketJob
       rescue Exception => e
         errors.add(field, "Failed to load :#{field}, #{e.inspect}")
       end
+
+      # Allow multiple instances of this job to run with the same cron schedule
+      def rocket_job_cron_singleton_check
+      end
     end
   end
 end

data/lib/rocket_job/jobs/on_demand_job.rb

@@ -97,6 +97,10 @@ module RocketJob
       rescue Exception => e
         errors.add(:code, "Failed to parse :code, #{e.inspect}")
       end
+
+      # Allow multiple instances of this job to run with the same cron schedule
+      def rocket_job_cron_singleton_check
+      end
     end
   end
 end

data/lib/rocket_job/sliced/output.rb

@@ -12,6 +12,7 @@ module RocketJob
         # Call the supplied block for every record returned
         record_count = 0
         each do |slice|
+          # TODO: Add slice_id to named tags to aid problem determination
           slice.each do |record|
             record_count += 1
             yield(record)

data/lib/rocket_job/supervisor.rb

@@ -14,7 +14,6 @@ module RocketJob
       Thread.current.name = "rocketjob main"
       RocketJob.create_indexes
       register_signal_handlers
-      subscribe_to_events
 
       server = Server.create!
       new(server).run
@@ -35,8 +34,15 @@ module RocketJob
       logger.info "Rocket Job Server started"
 
       event_listener = Thread.new { Event.listener }
-      supervise_pool
-      stop!
+      Subscribers::SecretConfig.subscribe if defined?(SecretConfig)
+      Subscribers::Server.subscribe(self) do
+        Subscribers::Worker.subscribe(self) do
+          Subscribers::Logger.subscribe do
+            supervise_pool
+            stop!
+          end
+        end
+      end
     rescue ::Mongoid::Errors::DocumentNotFound
       logger.info("Server has been destroyed. Going down hard!")
     rescue Exception => e
@@ -89,12 +95,5 @@ module RocketJob
     def synchronize(&block)
       @mutex.synchronize(&block)
     end
-
-    def subscribe_to_events
-      Subscribers::Logger.subscribe
-      Subscribers::SecretConfig.subscribe if defined?(SecretConfig)
-      Subscribers::Server.subscribe
-      Subscribers::Worker.subscribe
-    end
   end
 end

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "6.0.2".freeze
+  VERSION = "6.1.1".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 6.0.2
+  version: 6.1.1
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-25 00:00:00.000000000 Z
+date: 2022-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -236,7 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Ruby's missing batch processing system.