workhorse 1.2.17.rc0 → 1.2.17.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25908f294cd9623ebde44ec8bba5181ee54965703f4fb93df18876b216718eb8
-  data.tar.gz: 568e620571a9ded165fee1fc4420efbb6810f073edd4365c6c8efbe7c3676052
+  metadata.gz: 2bcfdbfa710579ee64f436236ed5c4214404a82211ae7471513ba141e7e03eaf
+  data.tar.gz: 680b91c6bbdbc8473fdc454175bdf6a2f4b14a0831162ab02e92208914f69f7a
 SHA512:
-  metadata.gz: 83b6ac441e3762f251e35f04dd213044d6fb904a005155235546391cdecd4f9c0c1db24be7c30371da4704b27e6f18aa6ce0b448b6c6c1dbc8c77eeb81faa9e3
-  data.tar.gz: 9c2940a6bfb7cd0be49170d81af2378f27395dbf597f6d8fa2c777bdc812b26d06cc849454ba5925fcf5f26b7724779b3d6e941bc6c9237d6dc251b7d071188f
+  metadata.gz: 29fe50d7eb989a0d51cdcbd23adb7d42ce56fccf553448c7baf112bae2dd678a81613d593637fe6f168ff3620d9b31048abcdd0f8674b4db2244a59a82865779
+  data.tar.gz: '094cb3bb85f2be4bd2575f6441e2abc6ef17d522796c586dca69f864031fde2b0095bc8ce2c3d8bee6507ab799297fd8e79c86fbbadd28467495a39cc7031694'

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Workhorse Changelog
 
+## 1.2.17.rc1 - 2024-02-05
+
+* Revamp memory handling:
+
+  * Change memory handling for workers to automatically shut down themselves
+    upon exceeding `config.max_worker_memory_mb` (if configured and > 0),
+    triggering the creation of a shutdown file
+    (`tmp/pids/workhorse.<pid>.shutdown`).
+  * Have the `watch` command, if scheduled, silently restart the shutdown worker
+    and remove the shutdown file.
+  * The presence of the shutdown file informs the watcher to produce output or
+    remain silent.
+  * Implement this adjustment to limit `watch` command output to error cases,
+    facilitating seamless cron integration for notification purposes.
+
+  Sitrox reference: #121312.
+
 ## 1.2.17.rc0 - 2024-02-05
 
 * Add option `config.max_worker_memory_mb` for automatic restart of workers

data/README.md

@@ -466,17 +466,18 @@ succeeded jobs. You can run this using your scheduler in a specific interval.
 
 ## Memory handling
 
-When dealing with jobs that may exhibit a large memory footprint, it's important
-to note that Ruby might not release consumed memory back to the operating
-system. Consequently, your job workers could accumulate a significant amount of
-memory over time. To address this, Workhorse provides the
-`config.max_worker_memory_mb` option.
-
-If `config.max_worker_memory_mb` is set to a value above `0`, the `watch`
-command will check the memory footprint (RSS / resident size) of all worker
-processes. If any worker exceeds the specified footprint, Workhorse will
-silently restart it to ensure proper memory release. This process does not
-produce any output in the `watch` command.
+When a worker exceeds the memory limit specified by
+`config.max_worker_memory_mb` (assuming it is configured and > 0), it initiates
+a graceful shutdown process by creating a shutdown file named
+`tmp/pids/workhorse.<pid>.shutdown`.
+
+Simultaneously, the `watch` command, if scheduled, monitors the presence of this
+shutdown file. Upon detecting its existence, it silently triggers the restart of
+the shutdown worker and removes the shutdown file to signify that the restart
+process has begun.
+
+This mechanism ensures that workers are automatically restarted without manual
+intervention when memory limits are exceeded.
 
 Example configuration:
 

data/VERSION

@@ -1 +1 @@
-1.2.17.rc0
+1.2.17.rc1

data/lib/workhorse/daemon.rb

@@ -46,15 +46,20 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
+        if pid_file && pid && active
           warn "Worker ##{worker.id} (#{worker.name}): Already started (PID #{pid})" unless quiet
           code = 2
         elsif pid_file
           File.delete pid_file
-          puts "Worker ##{worker.id} (#{worker.name}): Starting (stale pid file)" unless quiet
+
+          shutdown_file = pid ? Workhorse::Worker.shutdown_file_for(pid) : nil
+          shutdown_file = nil if shutdown_file && !File.exist?(shutdown_file)
+
+          puts "Worker ##{worker.id} (#{worker.name}): Starting (stale pid file)" unless quiet || shutdown_file
           start_worker worker
+          FileUtils.rm(shutdown_file) if shutdown_file
         else
           warn "Worker ##{worker.id} (#{worker.name}): Starting" unless quiet
           start_worker worker
@@ -68,9 +73,9 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
+        if pid_file && pid && active
           puts "Worker (#{worker.name}) ##{worker.id}: Stopping"
           stop_worker pid_file, pid, kill: kill
         elsif pid_file
@@ -89,9 +94,9 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
+        if pid_file && pid && active
           puts "Worker ##{worker.id} (#{worker.name}): Running" unless quiet
         elsif pid_file
           warn "Worker ##{worker.id} (#{worker.name}): Not running (stale PID file)" unless quiet
@@ -113,14 +118,10 @@ module Workhorse
       end
 
       if should_be_running && status(quiet: true) != 0
-        code = start(quiet: Workhorse.silence_watcher)
+        return start(quiet: Workhorse.silence_watcher)
       else
-        code = 0
+        return 0
       end
-
-      watch_memory! if should_be_running
-
-      return code
     end
 
     def restart
@@ -132,7 +133,9 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        _pid_file, pid = read_pid(worker)
+        _pid_file, pid, active = read_pid(worker)
+
+        next unless pid && active
 
         begin
           Process.kill 'HUP', pid
@@ -148,30 +151,6 @@ module Workhorse
 
     private
 
-    def watch_memory!
-      return if Workhorse.max_worker_memory_mb == 0
-
-      for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
-        next unless pid_file && pid
-
-        memory = memory_for(pid)
-        next unless memory
-
-        if memory > Workhorse.max_worker_memory_mb
-          stop_worker pid_file, pid
-          start_worker worker
-        end
-      end
-    end
-
-    # Returns the memory (RSS) in MB for the given process.
-    def memory_for(pid)
-      mem = `ps -p #{pid} -o rss=`&.strip
-      return nil if mem.blank?
-      return mem.to_i / 1024
-    end
-
     def for_each_worker(&block)
       @workers.each(&block)
     end
@@ -237,16 +216,21 @@ module Workhorse
 
     def read_pid(worker)
       file = pid_file_for(worker)
+      pid = nil
+      active = false
 
       if File.exist?(file)
         raw_pid = File.read(file)
-        return nil, nil if raw_pid.blank?
 
-        pid = Integer(raw_pid)
-        return file, process?(pid) ? pid : nil
+        unless raw_pid.blank?
+          pid = Integer(raw_pid)
+          active = process?(pid)
+        end
       else
         return nil, nil
       end
+
+      return file, pid, active
     end
   end
 end

data/lib/workhorse/performer.rb

@@ -22,7 +22,7 @@ module Workhorse
       Thread.current[:workhorse_current_performer] = self
 
       ActiveRecord::Base.connection_pool.with_connection do
-        if defined?(Rails) && Rails.application && Rails.application.respond_to?(:executor)
+        if defined?(Rails) && Rails.respond_to?(:application) && Rails.application && Rails.application.respond_to?(:executor)
           Rails.application.executor.wrap do
             perform_wrapped
           end
@@ -63,7 +63,6 @@ module Workhorse
 
       inner_job_class = deserialized_job.try(:job_class) || deserialized_job.class
       skip_tx = inner_job_class.try(:skip_tx?)
-      log "SKIP TX: #{skip_tx.inspect}".red, :error
 
       if Workhorse.perform_jobs_in_tx && !skip_tx
         Workhorse.tx_callback.call do

data/lib/workhorse/poller.rb

@@ -9,7 +9,7 @@ module Workhorse
     attr_reader :worker
     attr_reader :table
 
-    def initialize(worker)
+    def initialize(worker, before_poll = proc { true })
       @worker = worker
       @running = false
       @table = Workhorse::DbJob.arel_table
@@ -17,6 +17,7 @@ module Workhorse
       @instant_repoll = Concurrent::AtomicBoolean.new(false)
       @global_lock_fails = 0
       @max_global_lock_fails_reached = false
+      @before_poll = before_poll
     end
 
     def running?
@@ -34,6 +35,12 @@ module Workhorse
           break unless running?
 
           begin
+            unless @before_poll.call
+              Thread.new { worker.shutdown }
+              sleep
+              next
+            end
+
             poll
             sleep
           rescue Exception => e

data/lib/workhorse/pool.rb

@@ -2,6 +2,7 @@ module Workhorse
   # Abstraction layer of a simple thread pool implementation used by the worker.
   class Pool
     attr_reader :mutex
+    attr_reader :active_threads
 
     def initialize(size)
       @size = size

data/lib/workhorse/worker.rb

@@ -20,6 +20,12 @@ module Workhorse
       worker.wait
     end
 
+    # @private
+    def self.shutdown_file_for(pid)
+      return nil unless defined?(Rails)
+      Rails.root.join('tmp', 'pids', "workhorse.#{pid}.shutdown")
+    end
+
     # Instantiates a new worker. The worker is not automatically started.
     #
     # @param queues [Array] The queues you want this worker to process. If an
@@ -51,7 +57,7 @@ module Workhorse
 
       @mutex = Mutex.new
       @pool = Pool.new(@pool_size)
-      @poller = Workhorse::Poller.new(self)
+      @poller = Workhorse::Poller.new(self, proc { check_memory })
       @logger = logger
 
       unless (@polling_interval / 0.1).round(2).modulo(1).zero?
@@ -155,6 +161,37 @@ module Workhorse
 
     private
 
+    def check_memory
+      mem = current_memory_consumption
+
+      unless mem
+        log "Could not determine memory consumption of worker with pid #{pid}"
+        return false
+      end
+
+      max = Workhorse.max_worker_memory_mb
+      exceeded = max > 0 && current_memory_consumption > max
+
+      return true unless exceeded
+
+      if defined?(Rails)
+        FileUtils.touch self.class.shutdown_file_for(pid)
+      end
+
+      log "Worker process #{id.inspect} memory consumption (RSS) of #{mem}MB exceeds "\
+          "configured per-worker limit of #{max}MB and is now being shut down. Make sure "\
+          'that your worker processes are watched (e.g. using the "watch"-command) for ' \
+          'this worker to be restarted automatically.'
+
+      return false
+    end
+
+    def current_memory_consumption
+      mem = `ps -p #{pid} -o rss=`&.strip
+      return nil if mem.blank?
+      return mem.to_i / 1024
+    end
+
     def check_rails_env
       unless Rails.env.production?
         warn 'WARNING: Always run workhorse workers in production environment. Other environments can lead to unexpected behavior.'

data/test/lib/jobs.rb

@@ -36,6 +36,15 @@ class SyntaxErrorJob
   end
 end
 
+class MemHungryJob
+  class_attribute :data
+
+  # Should consume roughly 1GB of memory.
+  def perform
+    self.class.data = 'x' * 250.megabytes
+  end
+end
+
 class DummyRailsOpsOp
   class_attribute :results
   self.results = Concurrent::Array.new

data/test/lib/test_helper.rb

@@ -5,8 +5,33 @@ require 'pry'
 require 'colorize'
 require 'mysql2'
 require 'benchmark'
+require 'concurrent'
 require 'jobs'
 
+class MockRailsEnv < String
+  def production?
+    self == 'production'
+  end
+
+  def test?
+    self == 'test'
+  end
+
+  def development?
+    self == 'development'
+  end
+end
+
+class Rails
+  def self.root
+    Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../../')))
+  end
+
+  def self.env
+    MockRailsEnv.new('production')
+  end
+end
+
 class WorkhorseTest < ActiveSupport::TestCase
   def setup
     Workhorse::DbJob.delete_all
@@ -31,6 +56,14 @@ class WorkhorseTest < ActiveSupport::TestCase
     end
   end
 
+  def work_until(max: 50, interval: 0.1, **options, &block)
+    w = Workhorse::Worker.new(**options)
+    w.start
+    return with_retries(max, interval: interval, &block)
+  ensure
+    w.shutdown
+  end
+
   def with_worker(options = {})
     w = Workhorse::Worker.new(**options)
     w.start
@@ -40,6 +73,18 @@ class WorkhorseTest < ActiveSupport::TestCase
       w.shutdown
     end
   end
+
+  def with_retries(max = 50, interval: 0.1, &_block)
+    runs = 0
+
+    loop do
+      return yield
+    rescue Minitest::Assertion => e
+      fail if runs > max
+      sleep interval
+      runs += 1
+    end
+  end
 end
 
 ActiveRecord::Base.establish_connection(

data/test/workhorse/worker_test.rb

@@ -180,8 +180,86 @@ class Workhorse::WorkerTest < WorkhorseTest
     assert_equal 'waiting',   jobs[1].state
   end
 
+  def test_controlled_shutdown
+    remove_pids!
+
+    Workhorse.max_worker_memory_mb = 50
+
+    daemon = start_daemon
+
+    pid = with_retries do
+      pid = daemon.workers.first.pid
+      assert_process(pid)
+      pid
+    end
+
+    10.times do
+      Workhorse.enqueue BasicJob.new(sleep_time: 0.1)
+
+      with_retries do
+        assert_equal 'succeeded', Workhorse::DbJob.first.state
+        Workhorse::DbJob.delete_all
+      end
+    end
+
+    Workhorse.enqueue MemHungryJob.new
+
+    with_retries do
+      assert_equal 'succeeded', Workhorse::DbJob.first.state
+
+      assert File.exist?("tmp/pids/workhorse.#{pid}.shutdown")
+      assert_not_process pid
+    end
+
+    daemon.watch
+
+    with_retries do
+      assert_not File.exist?("tmp/pids/workhorse.#{pid}.shutdown")
+    end
+  ensure
+    daemon.stop
+    Workhorse.max_worker_memory_mb = 0
+  end
+
   private
 
+  def remove_pids!
+    Dir[Rails.root.join('tmp', 'pids', '*')].each do |file|
+      FileUtils.rm file
+    end
+  end
+
+  def start_daemon
+    daemon = Workhorse::Daemon.new(pidfile: 'tmp/pids/test%s.pid') do |d|
+      d.worker 'Test Worker' do
+        begin
+          Workhorse::Worker.start_and_wait(
+            pool_size:        1,
+            polling_interval: 0.1,
+            logger:           ActiveSupport::Logger.new('tmp/log.log')
+          )
+        end
+      end
+    end
+    daemon.start
+    return daemon
+  end
+
+  def assert_process(pid)
+    assert process?(pid), "Process #{pid} expected to be running"
+  end
+
+  def assert_not_process(pid)
+    assert_not process?(pid), "Process #{pid} expected to be stopped"
+  end
+
+  def process?(pid)
+    Process.kill(0, pid)
+    true
+  rescue Errno::EPERM, Errno::ESRCH
+    false
+  end
+
   def enqueue_in_multiple_queues
     Workhorse.enqueue BasicJob.new(some_param: nil)
     Workhorse.enqueue BasicJob.new(some_param: :q1), queue: :q1

data/workhorse.gemspec

@@ -1,14 +1,14 @@
 # -*- encoding: utf-8 -*-
-# stub: workhorse 1.2.17.rc0 ruby lib
+# stub: workhorse 1.2.17.rc1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "workhorse".freeze
-  s.version = "1.2.17.rc0"
+  s.version = "1.2.17.rc1"
 
   s.required_rubygems_version = Gem::Requirement.new("> 1.3.1".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Sitrox".freeze]
-  s.date = "2024-02-05"
+  s.date = "2024-02-08"
   s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/active_job_extension.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
   s.rubygems_version = "3.4.6".freeze
   s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 1.2.17.rc0
+  version: 1.2.17.rc1
 platform: ruby
 authors:
 - Sitrox
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-05 00:00:00.000000000 Z
+date: 2024-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler