rocketjob 6.1.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc1f3d316be0aa058cee5b22c116872a80ea5bdaf7d72db447ae13781ba6b370
-  data.tar.gz: 87f56a455792e45ab2ad706f0c8bd625590df9942806bbd308c5145d07e94973
+  metadata.gz: d12a5469219af4b5781c67cf65670879f1fa349e42509431e5924015a52b4542
+  data.tar.gz: 745bbcfb5cc47a3413edf874b088b81382ccc185218aac3648adb1250e51b803
 SHA512:
-  metadata.gz: f928a3afb886c39893ef5b5d2e13f71405c83df53c3b02826e474326c9171f87c84e595fc179a0e22cbc69d846f1da12eb268026fc06bcaf609920326159c6f4
-  data.tar.gz: 3b90508c45e7a09739ae53e5bb66992d80251a5b5668eb568eeba30a8da8d109bbb162b2db66e93f82ce3deb4f6cc9425175d0f9c41adb351f6c3885768db3dc
+  metadata.gz: 012b3adb126e452c3ab4409d38d1fe02499cfc5049d5cc30074a1f607245e8da1327ad1a0a4b738ec290bcaf233cb7388132828d6f8f1684bdd70a1e17eb7c55
+  data.tar.gz: f8a1899798671265395f72d259afce338bea6e555755c33fdfa11928e10b14f29cc868570caa39f091f6863fa3c8809ab8874a6ba81b8b210c36c63e4454cf2b

data/README.md

@@ -1,5 +1,5 @@
 # Rocket Job
-[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Support](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
+[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) 
 
 Ruby's missing batch system
 
@@ -14,7 +14,7 @@ Checkout https://rocketjob.io/
 
 ## Support
 
-* Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
+* Ask questions in [Rocket Job Discussions](https://github.com/reidmorrison/rocketjob/discussions)
 * [Report bugs](https://github.com/rocketjob/rocketjob/issues)
 
 ## Rocket Job v6
@@ -106,8 +106,8 @@ These fields have been removed from the job itself:
 ~~~ruby
 class MyJob < RocketJob::Job
   include RocketJob::Batch
-  
-  self.slice_sice = 1_000
+
+  self.slice_size = 1_000
   self.encrypt    = true
   self.compress   = true
 end
@@ -123,8 +123,8 @@ If the serializer is set to `encrypt` then it is automatically compressed.
 ~~~ruby
 class MyJob < RocketJob::Job
   include RocketJob::Batch
-  
-  input_category slice_sice: 1_000, serializer: :encrypt
+
+  input_category slice_size: 1_000, serializer: :encrypt
 end
 ~~~
 

data/lib/rocket_job/batch/io.rb

@@ -466,6 +466,8 @@ module RocketJob
             output_collection.download(header_line: binary_header_line) { |record| io.write(record) }
           end
         else
+          # TODO: Add category to named tags to aid problem determination
+          # And RJ Download metric with duration
           IOStreams.new(stream || category.file_name).writer(:line, **args) do |io|
             output_collection.download(header_line: header_line) { |record| io << record }
           end

data/lib/rocket_job/config.rb

@@ -1,3 +1,4 @@
+require "logger/severity"
 require "yaml"
 module RocketJob
   # Rocket Job Configuration

data/lib/rocket_job/extensions/mongo/logging.rb

@@ -6,22 +6,26 @@ module Mongo
       include SemanticLogger::Loggable
       logger.name = "Mongo"
 
+      undef :started
       def started(event)
         @event_command = event.command
       end
 
+      undef :succeeded
       def succeeded(event)
         logger.debug(message:  prefix(event),
                      duration: (event.duration * 1000),
                      payload:  @event_command)
       end
 
+      undef :failed
       def failed(event)
         logger.debug(message:  "#{prefix(event)} Failed: #{event.message}",
                      duration: (event.duration * 1000),
                      payload:  @event_command)
       end
 
+      undef :prefix
       def prefix(event)
         "#{event.address} | #{event.database_name}.#{event.command_name}"
       end

data/lib/rocket_job/jobs/copy_file_job.rb

@@ -82,7 +82,10 @@ module RocketJob
       end
 
       def apply_streams(path, streams)
-        streams.each_pair { |stream, args| path.stream(stream.to_sym, args.nil? ? {} : decode_args(args)) }
+        streams.each_pair do |stream, args|
+          stream_args = args.nil? ? {} : decode_args(args)
+          path.stream(stream.to_sym, **stream_args)
+        end
       end
 
       def decode_args(args)

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -42,6 +42,10 @@ module RocketJob
       self.description   = "Directory Monitor"
       self.priority      = 30
 
+      # Prevent multiple dirmon jobs from running at the same time by only
+      # creating the next instance when this one finishes.
+      self.cron_after_start = false
+
       # Hash[file_name, size]
       field :previous_file_names, type: Hash, default: {}, copy_on_restart: true
 

data/lib/rocket_job/plugins/retry.rb

@@ -26,14 +26,51 @@ module RocketJob
     # # Disable retries for this job instance.
     # MyCronJob.create!(retry_limit: 0)
     #
+    # Below is a table of the delay for each retry attempt, as well as the _total_ duration spent retrying a job for
+    # the specified number of retries, excluding actual processing time.
+    #
+    # |---------|------------|----------------|
+    # | Attempt |      Delay | Total Duration |
+    # |---------|------------|----------------|
+    # |      01 |     6.000s |         6.000s |
+    # |      02 |    21.000s |        27.000s |
+    # |      03 |     1m 26s |         1m 53s |
+    # |      04 |     4m 21s |         6m 14s |
+    # |      05 |    10m 30s |        16m 44s |
+    # |      06 |    21m 41s |        38m 25s |
+    # |      07 |     40m 6s |        20h 18m |
+    # |      08 |     20h 8m |        21h 26m |
+    # |      09 |    20h 49m |        23h 16m |
+    # |      10 |    21h 46m |          2h 3m |
+    # |      11 |     23h 4m |          6h 7m |
+    # |      12 |     0h 45m |        11h 52m |
+    # |      13 |     2h 56m |     1d 19h 48m |
+    # |      14 |     5h 40m |      1d 6h 29m |
+    # |      15 |      9h 3m |     2d 20h 33m |
+    # |      16 |    13h 12m |     2d 14h 45m |
+    # |      17 |    18h 12m |     3d 13h 57m |
+    # |      18 |   1d 0h 9m |      5d 19h 7m |
+    # |      19 |  1d 7h 12m |      6d 7h 19m |
+    # |      20 | 1d 15h 26m |      8d 3h 46m |
+    # |      21 |   2d 1h 1m |     10d 9h 47m |
+    # |      22 |  2d 12h 4m |     13d 2h 51m |
+    # |      23 |  3d 0h 44m |     16d 8h 35m |
+    # |      24 |  3d 15h 9m |     20d 4h 45m |
+    # |      25 |  4d 7h 30m |     24d 17h 16 |
+    # |      26 |  5d 1h 56m |     30d 0h 12m |
+    # |      27 | 6d 22h 37m |     36d 3h 49m |
+    # |      28 | 7d 21h 44m |     43d 6h 34m |
+    # |      29 | 8d 23h 28m |     51d 11h 2m |
+    # |      30 |   9d 4h 0m |     61d 20h 2m |
+    # |---------|------------|----------------|
     module Retry
       extend ActiveSupport::Concern
 
       included do
         after_fail :rocket_job_retry
 
-        # Maximum number of times to retry this job
-        # 25 is approximately 3 weeks of retries
+        # Maximum number of times to retry this job.
+        # The default of 25 is almost 25 days of retries.
         field :retry_limit, type: Integer, default: 25, class_attribute: true, user_editable: true, copy_on_restart: true
 
         # List of times when this job failed
@@ -73,10 +110,23 @@ module RocketJob
         self.worker_name  = nil
       end
 
-      # Returns [Time] when to retry this job at
-      # Same basic formula as Delayed Job
+      # Returns [Integer] the number of seconds after which to retry this failed job.
+      # Uses an exponential back-off algorithm to prevent overloading the failed resource.
+      #
+      # For example, to see the durations for the first 25 retries:
+      #   count = 25
+      #   intervals = (1..count).map { |attempt| attempt**4 + 5 }
+      #
+      # Display the above intervals as human readable durations:
+      #   intervals.map { |seconds| RocketJob.seconds_as_duration(seconds) }
+      #
+      # Then sum the total duration in seconds:
+      #   RocketJob.seconds_as_duration(intervals.sum)
+      #
+      # Or, to see the total durations based on the number of retries:
+      #   (0..count).map{|i| "#{i+1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}"}
       def rocket_job_retry_seconds_to_delay
-        (rocket_job_failure_count**4) + 15 + (rand(30) * (rocket_job_failure_count + 1))
+        rocket_job_failure_count**4 + 5
       end
     end
   end

data/lib/rocket_job/rocket_job.rb

@@ -28,11 +28,11 @@ module RocketJob
     return nil unless seconds
 
     if seconds >= 86_400.0 # 1 day
-      "#{(seconds / 86_400).to_i}d #{Time.at(seconds).strftime('%-Hh %-Mm')}"
+      "#{(seconds / 86_400).to_i}d #{Time.at(seconds).utc.strftime('%-Hh %-Mm')}"
     elsif seconds >= 3600.0 # 1 hour
-      Time.at(seconds).strftime("%-Hh %-Mm")
+      Time.at(seconds).utc.strftime("%-Hh %-Mm")
     elsif seconds >= 60.0 # 1 minute
-      Time.at(seconds).strftime("%-Mm %-Ss")
+      Time.at(seconds).utc.strftime("%-Mm %-Ss")
     elsif seconds >= 1.0 # 1 second
       format("%.3fs", seconds)
     else

data/lib/rocket_job/sliced/output.rb

@@ -12,6 +12,7 @@ module RocketJob
         # Call the supplied block for every record returned
         record_count = 0
         each do |slice|
+          # TODO: Add slice_id to named tags to aid problem determination
           slice.each do |record|
             record_count += 1
             yield(record)

data/lib/rocket_job/sliced/slices.rb

@@ -92,7 +92,7 @@ module RocketJob
 
       def insert_many(slices)
         documents = slices.collect(&:as_document)
-        all.collection.insert_many(documents)
+        all.collection.insert_many(documents) if documents.present?
       end
 
       # Append to an existing slice if already present

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "6.1.0".freeze
+  VERSION = "6.2.0".freeze
 end

data/lib/rocketjob.rb

@@ -1,6 +1,6 @@
-require "iostreams"
 require "semantic_logger"
 require "symmetric-encryption"
+require "iostreams"
 require "mongoid"
 require "rocket_job/extensions/mongo/logging"
 require "rocket_job/extensions/iostreams/path"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 6.1.0
+  version: 6.2.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-04 00:00:00.000000000 Z
+date: 2023-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -28,58 +28,58 @@ dependencies:
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   name: fugit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.4'
 - !ruby/object:Gem::Dependency
   name: iostreams
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '7.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '7.5'
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
@@ -98,14 +98,14 @@ dependencies:
   name: symmetric-encryption
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.3'
 description:
@@ -229,14 +229,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.4.9
 signing_key:
 specification_version: 4
 summary: Ruby's missing batch processing system.