rocketjob 6.1.1 → 6.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca9a7af0f4448efdb418e738f95bc5f885d2399bdfd50ce3d0533c6928f3f78f
-  data.tar.gz: e0f1bc3fb3c60989f22ff4d75c1ed86c8f7a998bdc5bc02f0f8616dd9631029e
+  metadata.gz: a90e66a2d150e503e939735d7913b016e3cea3a9a43b6ae0751b101768f10c5c
+  data.tar.gz: bc0029bffda1dfa575984352c1284aaec8e93d8cb0f81789ae253740de3cc4d9
 SHA512:
-  metadata.gz: 79e914a57bbdae9026cbe783d8a1743e538bb2b1ffebacb527561ef961174eaa24693e850e0b72ee9ae113611db96fe1e0e2edb85d6395cbc057b7fc17bf11e9
-  data.tar.gz: f6a8ba0b5060f54072a68fc396e43a3b9ba1c53b58e6b282006e111792e612d301b737279a344be252fc659724b70e12cff5400f49eb47316d38eda0571788f3
+  metadata.gz: 4350c6360df190ccb0c251944cf4bf5bc20de512779b5f6e0107571c049f7adc2e589de63ba350d1c179505e3af2ff5610220edbfc9e43f4c0abff4943448ee6
+  data.tar.gz: 9924be8c32392448e6285e9f2a94c1da26abd5b29a2d2aa7d60ff0cee8e34c61cedd6c69cc109894dfeaa9e5ce5dcfea0b2084bacce0493b2e44c5c6f2c813fc

data/README.md

@@ -1,5 +1,5 @@
 # Rocket Job
-[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) [![Support](https://img.shields.io/badge/IRC%20(gitter)-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
+[![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg) 
 
 Ruby's missing batch system
 
@@ -14,7 +14,7 @@ Checkout https://rocketjob.io/
 
 ## Support
 
-* Questions? Join the chat room on Gitter for [rocketjob support](https://gitter.im/rocketjob/support)
+* Ask questions in [Rocket Job Discussions](https://github.com/reidmorrison/rocketjob/discussions)
 * [Report bugs](https://github.com/rocketjob/rocketjob/issues)
 
 ## Rocket Job v6
@@ -106,8 +106,8 @@ These fields have been removed from the job itself:
 ~~~ruby
 class MyJob < RocketJob::Job
   include RocketJob::Batch
-  
-  self.slice_sice = 1_000
+
+  self.slice_size = 1_000
   self.encrypt    = true
   self.compress   = true
 end
@@ -123,8 +123,8 @@ If the serializer is set to `encrypt` then it is automatically compressed.
 ~~~ruby
 class MyJob < RocketJob::Job
   include RocketJob::Batch
-  
-  input_category slice_sice: 1_000, serializer: :encrypt
+
+  input_category slice_size: 1_000, serializer: :encrypt
 end
 ~~~
 

data/lib/rocket_job/batch/worker.rb

@@ -226,7 +226,7 @@ module RocketJob
         result = self.class.with(write: {w: 1}) do |query|
           query.
             where(id: id, state: :running, sub_state: :processing).
-            update("$set" => {sub_state: :after, worker_name: worker_name})
+            update("$set" => {sub_state: "after", worker_name: worker_name})
         end
 
         # Reload to pull in any counters or other data that was modified.
@@ -251,7 +251,7 @@ module RocketJob
           result   = self.class.with(write: {w: 1}) do |query|
             query.
               where(id: id, state: :running, sub_state: :processing).
-              update({"$set" => {state: :failed, worker_name: worker_name}})
+              update({"$set" => {state: "failed", worker_name: worker_name}})
           end
           fail_job = false unless result.modified_count.positive?
         end

data/lib/rocket_job/category/base.rb

@@ -61,7 +61,7 @@ module RocketJob
         @tabular ||= IOStreams::Tabular.new(
           columns:        columns,
           format:         format == :auto ? nil : format,
-          format_options: format_options&.deep_symbolize_keys,
+          format_options: format_options&.to_h&.deep_symbolize_keys,
           file_name:      file_name
         )
       end

data/lib/rocket_job/category/input.rb

@@ -99,7 +99,7 @@ module RocketJob
         @tabular ||= IOStreams::Tabular.new(
           columns:          columns,
           format:           format == :auto ? nil : format,
-          format_options:   format_options&.deep_symbolize_keys,
+          format_options:   format_options&.to_h&.deep_symbolize_keys,
           file_name:        file_name,
           allowed_columns:  allowed_columns,
           required_columns: required_columns,

data/lib/rocket_job/config.rb

@@ -1,3 +1,4 @@
+require "logger/severity"
 require "yaml"
 module RocketJob
   # Rocket Job Configuration

data/lib/rocket_job/jobs/copy_file_job.rb

@@ -26,6 +26,7 @@ module RocketJob
       self.destroy_on_complete = false
       # Number of times to automatically retry the copy. Set to `0` for no retry attempts.
       self.retry_limit = 10
+      self.prioriy = 30
 
       # File names in IOStreams URL format.
       field :source_url, type: String, user_editable: true

data/lib/rocket_job/jobs/dirmon_job.rb

@@ -42,6 +42,10 @@ module RocketJob
       self.description   = "Directory Monitor"
       self.priority      = 30
 
+      # Prevent multiple dirmon jobs from running at the same time by only
+      # creating the next instance when this one finishes.
+      self.cron_after_start = false
+
       # Hash[file_name, size]
       field :previous_file_names, type: Hash, default: {}, copy_on_restart: true
 

data/lib/rocket_job/jobs/upload_file_job.rb

@@ -59,7 +59,14 @@ module RocketJob
         if job.respond_to?(:upload)
           # Return the database connection for this thread back to the connection pool
           # in case the upload takes a long time and the database connection expires.
-          ActiveRecord::Base.clear_active_connections! if defined?(ActiveRecord::Base)
+          if defined?(ActiveRecord::Base)
+            if ActiveRecord::Base.respond_to?(:connection_handler)
+              # Rails 7
+              ActiveRecord::Base.connection_handler.clear_active_connections!
+            else
+              ActiveRecord::Base.connection_pool.release_connection
+            end
+          end
 
           if original_file_name
             job.upload(upload_file_name, file_name: original_file_name)

data/lib/rocket_job/plugins/retry.rb

@@ -26,14 +26,51 @@ module RocketJob
     # # Disable retries for this job instance.
     # MyCronJob.create!(retry_limit: 0)
     #
+    # Below is a table of the delay for each retry attempt, as well as the _total_ duration spent retrying a job for
+    # the specified number of retries, excluding actual processing time.
+    #
+    # |---------|------------|----------------|
+    # | Attempt |      Delay | Total Duration |
+    # |---------|------------|----------------|
+    # |      01 |     6.000s |         6.000s |
+    # |      02 |    21.000s |        27.000s |
+    # |      03 |     1m 26s |         1m 53s |
+    # |      04 |     4m 21s |         6m 14s |
+    # |      05 |    10m 30s |        16m 44s |
+    # |      06 |    21m 41s |        38m 25s |
+    # |      07 |     40m 6s |        20h 18m |
+    # |      08 |     20h 8m |        21h 26m |
+    # |      09 |    20h 49m |        23h 16m |
+    # |      10 |    21h 46m |          2h 3m |
+    # |      11 |     23h 4m |          6h 7m |
+    # |      12 |     0h 45m |        11h 52m |
+    # |      13 |     2h 56m |     1d 19h 48m |
+    # |      14 |     5h 40m |      1d 6h 29m |
+    # |      15 |      9h 3m |     2d 20h 33m |
+    # |      16 |    13h 12m |     2d 14h 45m |
+    # |      17 |    18h 12m |     3d 13h 57m |
+    # |      18 |   1d 0h 9m |      5d 19h 7m |
+    # |      19 |  1d 7h 12m |      6d 7h 19m |
+    # |      20 | 1d 15h 26m |      8d 3h 46m |
+    # |      21 |   2d 1h 1m |     10d 9h 47m |
+    # |      22 |  2d 12h 4m |     13d 2h 51m |
+    # |      23 |  3d 0h 44m |     16d 8h 35m |
+    # |      24 |  3d 15h 9m |     20d 4h 45m |
+    # |      25 |  4d 7h 30m |     24d 17h 16 |
+    # |      26 |  5d 1h 56m |     30d 0h 12m |
+    # |      27 | 6d 22h 37m |     36d 3h 49m |
+    # |      28 | 7d 21h 44m |     43d 6h 34m |
+    # |      29 | 8d 23h 28m |     51d 11h 2m |
+    # |      30 |   9d 4h 0m |     61d 20h 2m |
+    # |---------|------------|----------------|
     module Retry
       extend ActiveSupport::Concern
 
       included do
         after_fail :rocket_job_retry
 
-        # Maximum number of times to retry this job
-        # 25 is approximately 3 weeks of retries
+        # Maximum number of times to retry this job.
+        # The default of 25 is almost 25 days of retries.
         field :retry_limit, type: Integer, default: 25, class_attribute: true, user_editable: true, copy_on_restart: true
 
         # List of times when this job failed
@@ -73,10 +110,23 @@ module RocketJob
         self.worker_name  = nil
       end
 
-      # Returns [Time] when to retry this job at
-      # Same basic formula as Delayed Job
+      # Returns [Integer] the number of seconds after which to retry this failed job.
+      # Uses an exponential back-off algorithm to prevent overloading the failed resource.
+      #
+      # For example, to see the durations for the first 25 retries:
+      #   count = 25
+      #   intervals = (1..count).map { |attempt| attempt**4 + 5 }
+      #
+      # Display the above intervals as human readable durations:
+      #   intervals.map { |seconds| RocketJob.seconds_as_duration(seconds) }
+      #
+      # Then sum the total duration in seconds:
+      #   RocketJob.seconds_as_duration(intervals.sum)
+      #
+      # Or, to see the total durations based on the number of retries:
+      #   (0..count).map{|i| "#{i+1} ==> #{RocketJob.seconds_as_duration(intervals[0..i].sum)}"}
       def rocket_job_retry_seconds_to_delay
-        (rocket_job_failure_count**4) + 15 + (rand(30) * (rocket_job_failure_count + 1))
+        rocket_job_failure_count**4 + 5
       end
     end
   end

data/lib/rocket_job/rocket_job.rb

@@ -28,11 +28,11 @@ module RocketJob
     return nil unless seconds
 
     if seconds >= 86_400.0 # 1 day
-      "#{(seconds / 86_400).to_i}d #{Time.at(seconds).strftime('%-Hh %-Mm')}"
+      "#{(seconds / 86_400).to_i}d #{Time.at(seconds).utc.strftime('%-Hh %-Mm')}"
     elsif seconds >= 3600.0 # 1 hour
-      Time.at(seconds).strftime("%-Hh %-Mm")
+      Time.at(seconds).utc.strftime("%-Hh %-Mm")
     elsif seconds >= 60.0 # 1 minute
-      Time.at(seconds).strftime("%-Mm %-Ss")
+      Time.at(seconds).utc.strftime("%-Mm %-Ss")
     elsif seconds >= 1.0 # 1 second
       format("%.3fs", seconds)
     else
@@ -58,3 +58,6 @@ module RocketJob
     @rails = true
   end
 end
+
+# Slice is a reserved word in Rails 7, but already being used in RocketJob long before that.
+Mongoid.destructive_fields.delete(:slice) if Mongoid.respond_to?(:destructive_fields)

data/lib/rocket_job/sliced/input.rb

@@ -105,7 +105,7 @@ module RocketJob
       def requeue_failed
         failed.update_all(
           "$unset" => {worker_name: nil, started_at: nil},
-          "$set"   => {state: :queued}
+          "$set"   => {state: "queued"}
         )
       end
 
@@ -113,7 +113,7 @@ module RocketJob
       def requeue_running(worker_name)
         running.where(worker_name: /\A#{worker_name}/).update_all(
           "$unset" => {worker_name: nil, started_at: nil},
-          "$set"   => {state: :queued}
+          "$set"   => {state: "queued"}
         )
       end
 

data/lib/rocket_job/sliced/slices.rb

@@ -92,7 +92,7 @@ module RocketJob
 
       def insert_many(slices)
         documents = slices.collect(&:as_document)
-        all.collection.insert_many(documents)
+        all.collection.insert_many(documents) if documents.present?
       end
 
       # Append to an existing slice if already present

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = "6.1.1".freeze
+  VERSION = "6.3.0".freeze
 end

data/lib/rocketjob.rb

@@ -1,6 +1,6 @@
-require "iostreams"
 require "semantic_logger"
 require "symmetric-encryption"
+require "iostreams"
 require "mongoid"
 require "rocket_job/extensions/mongo/logging"
 require "rocket_job/extensions/iostreams/path"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 6.1.1
+  version: 6.3.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-27 00:00:00.000000000 Z
+date: 2023-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -28,58 +28,58 @@ dependencies:
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   name: fugit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.4'
 - !ruby/object:Gem::Dependency
   name: iostreams
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.9'
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '7.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '7.5'
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
@@ -98,14 +98,14 @@ dependencies:
   name: symmetric-encryption
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.3'
 description:
@@ -229,14 +229,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.9
 signing_key:
 specification_version: 4
 summary: Ruby's missing batch processing system.