rocketjob 6.3.2 → 7.1.0

data/lib/rocket_job/worker.rb

@@ -63,25 +63,29 @@ module RocketJob
       logger.info "Started"
 
       until shutdown?
-        sleep_seconds = Config.max_poll_seconds
         reset_filter_if_expired
         job = next_available_job
 
-        # Returns true when work was completed, but no other work is available
-        if job&.rocket_job_work(self, false)
-          # Return the database connections for this fiber back to the connection pool
-          if defined?(ActiveRecord::Base)
-            if ActiveRecord::Base.respond_to?(:connection_handler)
-              # Rails 7 .2
-              ActiveRecord::Base.connection_handler.clear_active_connections!(:all)
+        if job
+          # Returns true when this job has no more work immediately available,
+          # for example a batch job that has run out of slices, or was throttled.
+          no_immediate_work = job.rocket_job_work(self, false)
+
+          # Return the database connections for this thread back to the connection pool.
+          release_active_record_connections
+
+          sleep_seconds =
+            if no_immediate_work
+              # Stagger workers so that they don't all poll at the same time.
+              random_wait_interval
             else
-              ActiveRecord::Base.clear_active_connections!
+              # Work was performed and more queued work is likely available,
+              # so poll again immediately rather than waiting a full interval.
+              0
             end
-          end
-          
-
-          # Stagger workers so that they don't all poll at the same time.
-          sleep_seconds = random_wait_interval
+        else
+          # No work was found, so wait the full poll interval before checking again.
+          sleep_seconds = Config.max_poll_seconds
         end
 
         wait_for_shutdown?(sleep_seconds)
@@ -91,12 +95,18 @@ module RocketJob
     rescue Exception => e
       logger.fatal("Unhandled exception in job processing thread", e)
     ensure
-      if defined?(ActiveRecord::Base)
-        if ActiveRecord::Base.respond_to?(:connection_handler)
-          ActiveRecord::Base.connection_handler.clear_active_connections!(:all)
-        else
-          ActiveRecord::Base.clear_active_connections!
-        end
+      release_active_record_connections
+    end
+
+    # Return the database connections for this thread back to the connection pool.
+    def release_active_record_connections
+      return unless defined?(ActiveRecord::Base)
+
+      if ActiveRecord::Base.respond_to?(:connection_handler)
+        # Rails 7.2+
+        ActiveRecord::Base.connection_handler.clear_active_connections!(:all)
+      else
+        ActiveRecord::Base.clear_active_connections!
       end
     end
 
@@ -149,12 +159,19 @@ module RocketJob
     # Whether the supplied job has been throttled and should be ignored.
     def throttled_job?(job)
       # Evaluate job throttles, if any.
-      filter = job.rocket_job_throttles.matching_filter(job)
-      return false unless filter
-
-      add_to_current_filter(filter)
-      # Restore retrieved job so that other workers can process it later
-      job.set(worker_name: nil, state: :queued)
+      throttle = job.rocket_job_throttles.matching_throttle(job)
+      return false unless throttle
+
+      add_to_current_filter(throttle.extract_filter(job))
+      # Restore retrieved job so that other workers can process it later, recording
+      # why it was throttled so it is visible in Mission Control. This reuses the
+      # write that requeues the job, so it adds no extra database round trip.
+      job.set(
+        worker_name:  nil,
+        state:        :queued,
+        throttled_by: throttle.extract_description(job),
+        throttled_at: Time.now
+      )
       true
     end
 
@@ -164,14 +181,46 @@ module RocketJob
     # Applies the current filter to exclude filtered jobs.
     #
     # Returns nil if no jobs are available for processing.
+    #
+    # Notes:
+    # - An already running batch job is _joined_ with a read-only query, since
+    #   concurrency for a batch job is coordinated per-slice (see
+    #   `Sliced::Input#next_slice`), not on the job document. Writing
+    #   `worker_name`/`state` to the shared job document on every poll turns it
+    #   into a write-contention hotspot: with many workers MongoDB serializes the
+    #   updates and retries the write conflicts (server log id 46404), which is
+    #   what makes a large batch job slow down as workers are added.
+    # - Only a queued job is claimed with a write, transitioning it to running.
+    #   That write is naturally distributed: each queued job is claimed once.
     def find_and_assign_job
       SemanticLogger.silence(:info) do
         scheduled = RocketJob::Job.where(run_at: nil).or(:run_at.lte => Time.now)
         working   = RocketJob::Job.queued.or(state: "running", sub_state: "processing")
         query     = RocketJob::Job.and(working, scheduled)
         query     = query.and(current_filter) unless current_filter.blank?
-        update    = {"$set" => {"worker_name" => name, "state" => "running"}}
-        query.sort(priority: 1, _id: 1).find_one_and_update(update, bypass_document_validation: true)
+        query     = query.sort(priority: 1, _id: 1)
+
+        # Retry only when a queued job was claimed by another worker first.
+        loop do
+          job = query.first
+          return nil unless job
+
+          # Already running batch job: join it without writing to the job document.
+          return job if job.running?
+
+          # Queued job: claim it atomically, guarding on it still being queued.
+          # find_one_and_update returns the pre-image (state: queued) so that the
+          # caller still runs throttles and `start!`.
+          # Clear any stale throttle reason from a previous throttled poll so a job
+          # that is now allowed to run no longer reports as throttled in Mission Control.
+          update = {
+            "$set"   => {"worker_name" => name, "state" => "running"},
+            "$unset" => {"throttled_by" => "", "throttled_at" => ""}
+          }
+          claimed = RocketJob::Job.queued.where(id: job.id).
+                    find_one_and_update(update, bypass_document_validation: true)
+          return claimed if claimed
+        end
       end
     end
 

data/lib/rocketjob.rb

@@ -23,6 +23,7 @@ module RocketJob
   autoload :ActiveWorker,            "rocket_job/active_worker"
   autoload :Batch,                   "rocket_job/batch"
   autoload :CLI,                     "rocket_job/cli"
+  autoload :Deprecation,             "rocket_job/deprecation"
   autoload :DirmonEntry,             "rocket_job/dirmon_entry"
   autoload :Event,                   "rocket_job/event"
   autoload :Heartbeat,               "rocket_job/heartbeat"
@@ -31,7 +32,6 @@ module RocketJob
   autoload :LookupCollection,        "rocket_job/lookup_collection"
   autoload :Worker,                  "rocket_job/worker"
   autoload :Performance,             "rocket_job/performance"
-  autoload :RactorWorker,            "rocket_job/ractor_worker"
   autoload :Server,                  "rocket_job/server"
   autoload :Sliced,                  "rocket_job/sliced"
   autoload :Subscriber,              "rocket_job/subscriber"
@@ -83,7 +83,7 @@ module RocketJob
     autoload :UploadFileJob,         "rocket_job/jobs/upload_file_job"
 
     module ReEncrypt
-      autoload :RelationalJob,       "rocket_job/jobs/re_encrypt/relational_job"
+      autoload :RelationalJob, "rocket_job/jobs/re_encrypt/relational_job"
     end
   end
 

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 6.3.2
+  version: 7.1.0
 platform: ruby
 authors:
 - Reid Morrison
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-01-31 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -56,62 +55,65 @@ dependencies:
   name: iostreams
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '2.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.5'
+        version: '8.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.5'
+        version: '8.1'
 - !ruby/object:Gem::Dependency
   name: semantic_logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: symmetric-encryption
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '4.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
-description:
-email:
+        version: '4.6'
+description: Rocket Job is a distributed, priority-based, MongoDB-backed batch processing
+  system for Ruby. Run conventional background jobs, or split a single job's input
+  into slices and process it concurrently across thousands of workers, spilling from
+  memory to disk so very large files never fall over.
 executables:
 - rocketjob
+- rocketjob_batch_perf
 - rocketjob_perf
 extensions: []
 extra_rdoc_files: []
@@ -143,6 +145,7 @@ files:
 - lib/rocket_job/category/output.rb
 - lib/rocket_job/cli.rb
 - lib/rocket_job/config.rb
+- lib/rocket_job/deprecation.rb
 - lib/rocket_job/dirmon_entry.rb
 - lib/rocket_job/event.rb
 - lib/rocket_job/extensions/iostreams/path.rb
@@ -186,7 +189,6 @@ files:
 - lib/rocket_job/plugins/state_machine.rb
 - lib/rocket_job/plugins/throttle_dependent_jobs.rb
 - lib/rocket_job/plugins/transaction.rb
-- lib/rocket_job/ractor_worker.rb
 - lib/rocket_job/railtie.rb
 - lib/rocket_job/rocket_job.rb
 - lib/rocket_job/server.rb
@@ -217,11 +219,16 @@ files:
 - lib/rocket_job/worker.rb
 - lib/rocket_job/worker_pool.rb
 - lib/rocketjob.rb
-homepage: https://rocketjob.io
+homepage: https://rocketjob.reidmorrison.com
 licenses:
 - Apache-2.0
-metadata: {}
-post_install_message:
+metadata:
+  bug_tracker_uri: https://github.com/reidmorrison/rocketjob/issues
+  changelog_uri: https://github.com/reidmorrison/rocketjob/blob/main/CHANGELOG.md
+  documentation_uri: https://rocketjob.reidmorrison.com
+  homepage_uri: https://rocketjob.reidmorrison.com
+  source_code_uri: https://github.com/reidmorrison/rocketjob/tree/v7.1.0
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -229,15 +236,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
-summary: Ruby's missing batch processing system.
+summary: Process millions of records across thousands of workers. A distributed, MongoDB-backed
+  batch processing system for Ruby.
 test_files: []

data/lib/rocket_job/ractor_worker.rb

@@ -1,42 +0,0 @@
-module RocketJob
-  # Run each worker in its own "Ractor".
-  class RactorWorker < Worker
-    attr_reader :thread
-
-    def initialize(id:, server_name:)
-      super(id: id, server_name: server_name)
-      @shutdown = Concurrent::Event.new
-      @thread   = Ractor.new(name: "rocketjob-#{id}") { run }
-    end
-
-    def alive?
-      @thread.alive?
-    end
-
-    def backtrace
-      @thread.backtrace
-    end
-
-    def join(*args)
-      @thread.join(*args)
-    end
-
-    # Send each active worker the RocketJob::ShutdownException so that stops processing immediately.
-    def kill
-      @thread.raise(Shutdown, "Shutdown due to kill request for worker: #{name}") if @thread.alive?
-    end
-
-    def shutdown?
-      @shutdown.set?
-    end
-
-    def shutdown!
-      @shutdown.set
-    end
-
-    # Returns [true|false] whether the shutdown indicator was set
-    def wait_for_shutdown?(timeout = nil)
-      @shutdown.wait(timeout)
-    end
-  end
-end