workhorse 1.2.16 → 1.2.17.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c50e38fd05785d30e0dfa783d732f46df2be9b980ac0a54dca1a2d91264ff9d
-  data.tar.gz: a4b282a25d39429da15c5380a41a1786b54b24b2b8afb4961786881380eecacf
+  metadata.gz: 2bcfdbfa710579ee64f436236ed5c4214404a82211ae7471513ba141e7e03eaf
+  data.tar.gz: 680b91c6bbdbc8473fdc454175bdf6a2f4b14a0831162ab02e92208914f69f7a
 SHA512:
-  metadata.gz: f8fefeb28bb6651e93163e5d9b8744994f681064bf55168b73d8ca5e9c72aee9e3f7642d03a86eb496ebd81bf15e724757cb4f0a6790dcb87e3ae5f3ef09344d
-  data.tar.gz: '090f112b83a2a69015e0244345d30616af8701bf99f2a6814238006efa2cca49850dfe024b26d1359b2863d01bf2c84d0bbf84a7122419f540969ba7f8d66dea'
+  metadata.gz: 29fe50d7eb989a0d51cdcbd23adb7d42ce56fccf553448c7baf112bae2dd678a81613d593637fe6f168ff3620d9b31048abcdd0f8674b4db2244a59a82865779
+  data.tar.gz: '094cb3bb85f2be4bd2575f6441e2abc6ef17d522796c586dca69f864031fde2b0095bc8ce2c3d8bee6507ab799297fd8e79c86fbbadd28467495a39cc7031694'

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Workhorse Changelog
 
+## 1.2.17.rc1 - 2024-02-05
+
+* Revamp memory handling:
+
+  * Change memory handling for workers to automatically shut down themselves
+    upon exceeding `config.max_worker_memory_mb` (if configured and > 0),
+    triggering the creation of a shutdown file
+    (`tmp/pids/workhorse.<pid>.shutdown`).
+  * Have the `watch` command, if scheduled, silently restart the shutdown worker
+    and remove the shutdown file.
+  * The presence of the shutdown file informs the watcher to produce output or
+    remain silent.
+  * Implement this adjustment to limit `watch` command output to error cases,
+    facilitating seamless cron integration for notification purposes.
+
+  Sitrox reference: #121312.
+
+## 1.2.17.rc0 - 2024-02-05
+
+* Add option `config.max_worker_memory_mb` for automatic restart of workers
+  exceeding the specified memory threshold using the `watch` command. Default is
+  `0`, deactivating this feature. See [memory
+  handling](README.md#memory-handling) for more information.
+
+  Sitrox reference: #121312.
+
 ## 1.2.16 - 2023-09-18
 
 * Add support for `--skip-initializer` flag to install generator.

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 - 2023 Sitrox
+Copyright (c) 2017 - 2024 Sitrox
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -464,6 +464,30 @@ jobs database on a regular interval. Workhorse provides the job
 `Workhorse::Jobs::CleanupSucceededJobs` for this purpose that cleans up all
 succeeded jobs. You can run this using your scheduler in a specific interval.
 
+## Memory handling
+
+When a worker exceeds the memory limit specified by
+`config.max_worker_memory_mb` (assuming it is configured and > 0), it initiates
+a graceful shutdown process by creating a shutdown file named
+`tmp/pids/workhorse.<pid>.shutdown`.
+
+Simultaneously, the `watch` command, if scheduled, monitors the presence of this
+shutdown file. Upon detecting its existence, it silently triggers the restart of
+the shutdown worker and removes the shutdown file to signify that the restart
+process has begun.
+
+This mechanism ensures that workers are automatically restarted without manual
+intervention when memory limits are exceeded.
+
+Example configuration:
+
+```ruby
+# config/initializers/workhorse.rb
+Workhorse.setup do |config|
+  config.max_worker_memory_mb = 512 # Set the memory threshold to 512 megabytes
+end
+```
+
 ## Load hooks
 
 Using the load hook `:workhorse_db_job`, you can inject custom code into the
@@ -540,4 +564,4 @@ Please consult the [FAQ](FAQ.md).
 
 ## Copyright
 
-Copyright © 2017 - 2023 Sitrox. See `LICENSE` for further details.
+Copyright © 2017 - 2024 Sitrox. See `LICENSE` for further details.

data/RUBY_VERSION

@@ -1 +1 @@
-ruby-2.7.1-p83
+ruby-3.2.1-p31

data/VERSION

@@ -1 +1 @@
-1.2.16
+1.2.17.rc1

data/lib/workhorse/daemon.rb

@@ -46,15 +46,20 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
+        if pid_file && pid && active
           warn "Worker ##{worker.id} (#{worker.name}): Already started (PID #{pid})" unless quiet
           code = 2
         elsif pid_file
           File.delete pid_file
-          puts "Worker ##{worker.id} (#{worker.name}): Starting (stale pid file)" unless quiet
+
+          shutdown_file = pid ? Workhorse::Worker.shutdown_file_for(pid) : nil
+          shutdown_file = nil if shutdown_file && !File.exist?(shutdown_file)
+
+          puts "Worker ##{worker.id} (#{worker.name}): Starting (stale pid file)" unless quiet || shutdown_file
           start_worker worker
+          FileUtils.rm(shutdown_file) if shutdown_file
         else
           warn "Worker ##{worker.id} (#{worker.name}): Starting" unless quiet
           start_worker worker
@@ -68,9 +73,9 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
+        if pid_file && pid && active
           puts "Worker (#{worker.name}) ##{worker.id}: Stopping"
           stop_worker pid_file, pid, kill: kill
         elsif pid_file
@@ -89,9 +94,9 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        pid_file, pid = read_pid(worker)
+        pid_file, pid, active = read_pid(worker)
 
-        if pid_file && pid
+        if pid_file && pid && active
           puts "Worker ##{worker.id} (#{worker.name}): Running" unless quiet
         elsif pid_file
           warn "Worker ##{worker.id} (#{worker.name}): Not running (stale PID file)" unless quiet
@@ -128,7 +133,9 @@ module Workhorse
       code = 0
 
       for_each_worker do |worker|
-        _pid_file, pid = read_pid(worker)
+        _pid_file, pid, active = read_pid(worker)
+
+        next unless pid && active
 
         begin
           Process.kill 'HUP', pid
@@ -209,16 +216,21 @@ module Workhorse
 
     def read_pid(worker)
       file = pid_file_for(worker)
+      pid = nil
+      active = false
 
       if File.exist?(file)
         raw_pid = File.read(file)
-        return nil, nil if raw_pid.blank?
 
-        pid = Integer(raw_pid)
-        return file, process?(pid) ? pid : nil
+        unless raw_pid.blank?
+          pid = Integer(raw_pid)
+          active = process?(pid)
+        end
       else
         return nil, nil
       end
+
+      return file, pid, active
     end
   end
 end

data/lib/workhorse/performer.rb

@@ -22,7 +22,7 @@ module Workhorse
       Thread.current[:workhorse_current_performer] = self
 
       ActiveRecord::Base.connection_pool.with_connection do
-        if defined?(Rails) && Rails.application && Rails.application.respond_to?(:executor)
+        if defined?(Rails) && Rails.respond_to?(:application) && Rails.application && Rails.application.respond_to?(:executor)
           Rails.application.executor.wrap do
             perform_wrapped
           end
@@ -63,7 +63,6 @@ module Workhorse
 
       inner_job_class = deserialized_job.try(:job_class) || deserialized_job.class
       skip_tx = inner_job_class.try(:skip_tx?)
-      log "SKIP TX: #{skip_tx.inspect}".red, :error
 
       if Workhorse.perform_jobs_in_tx && !skip_tx
         Workhorse.tx_callback.call do

data/lib/workhorse/poller.rb

@@ -9,7 +9,7 @@ module Workhorse
     attr_reader :worker
     attr_reader :table
 
-    def initialize(worker)
+    def initialize(worker, before_poll = proc { true })
       @worker = worker
       @running = false
       @table = Workhorse::DbJob.arel_table
@@ -17,6 +17,7 @@ module Workhorse
       @instant_repoll = Concurrent::AtomicBoolean.new(false)
       @global_lock_fails = 0
       @max_global_lock_fails_reached = false
+      @before_poll = before_poll
     end
 
     def running?
@@ -34,6 +35,12 @@ module Workhorse
           break unless running?
 
           begin
+            unless @before_poll.call
+              Thread.new { worker.shutdown }
+              sleep
+              next
+            end
+
             poll
             sleep
           rescue Exception => e

data/lib/workhorse/pool.rb

@@ -2,6 +2,7 @@ module Workhorse
   # Abstraction layer of a simple thread pool implementation used by the worker.
   class Pool
     attr_reader :mutex
+    attr_reader :active_threads
 
     def initialize(size)
       @size = size

data/lib/workhorse/worker.rb

@@ -20,6 +20,12 @@ module Workhorse
       worker.wait
     end
 
+    # @private
+    def self.shutdown_file_for(pid)
+      return nil unless defined?(Rails)
+      Rails.root.join('tmp', 'pids', "workhorse.#{pid}.shutdown")
+    end
+
     # Instantiates a new worker. The worker is not automatically started.
     #
     # @param queues [Array] The queues you want this worker to process. If an
@@ -51,7 +57,7 @@ module Workhorse
 
       @mutex = Mutex.new
       @pool = Pool.new(@pool_size)
-      @poller = Workhorse::Poller.new(self)
+      @poller = Workhorse::Poller.new(self, proc { check_memory })
       @logger = logger
 
       unless (@polling_interval / 0.1).round(2).modulo(1).zero?
@@ -155,6 +161,37 @@ module Workhorse
 
     private
 
+    def check_memory
+      mem = current_memory_consumption
+
+      unless mem
+        log "Could not determine memory consumption of worker with pid #{pid}"
+        return false
+      end
+
+      max = Workhorse.max_worker_memory_mb
+      exceeded = max > 0 && current_memory_consumption > max
+
+      return true unless exceeded
+
+      if defined?(Rails)
+        FileUtils.touch self.class.shutdown_file_for(pid)
+      end
+
+      log "Worker process #{id.inspect} memory consumption (RSS) of #{mem}MB exceeds "\
+          "configured per-worker limit of #{max}MB and is now being shut down. Make sure "\
+          'that your worker processes are watched (e.g. using the "watch"-command) for ' \
+          'this worker to be restarted automatically.'
+
+      return false
+    end
+
+    def current_memory_consumption
+      mem = `ps -p #{pid} -o rss=`&.strip
+      return nil if mem.blank?
+      return mem.to_i / 1024
+    end
+
     def check_rails_env
       unless Rails.env.production?
         warn 'WARNING: Always run workhorse workers in production environment. Other environments can lead to unexpected behavior.'

data/lib/workhorse.rb

@@ -75,6 +75,12 @@ module Workhorse
   mattr_accessor :stale_detection_run_time_threshold
   self.stale_detection_run_time_threshold = 12 * 60
 
+  # Maximum memory for a worker in MB. If this memory limit (RSS / resident
+  # size) is reached for a worker process, the 'watch' command will restart said
+  # worker. Set this to 0 disable this feature.
+  mattr_accessor :max_worker_memory_mb
+  self.max_worker_memory_mb = 0
+
   def self.setup
     yield self
   end

data/test/lib/jobs.rb

@@ -36,6 +36,15 @@ class SyntaxErrorJob
   end
 end
 
+class MemHungryJob
+  class_attribute :data
+
+  # Should consume roughly 1GB of memory.
+  def perform
+    self.class.data = 'x' * 250.megabytes
+  end
+end
+
 class DummyRailsOpsOp
   class_attribute :results
   self.results = Concurrent::Array.new

data/test/lib/test_helper.rb

@@ -5,8 +5,33 @@ require 'pry'
 require 'colorize'
 require 'mysql2'
 require 'benchmark'
+require 'concurrent'
 require 'jobs'
 
+class MockRailsEnv < String
+  def production?
+    self == 'production'
+  end
+
+  def test?
+    self == 'test'
+  end
+
+  def development?
+    self == 'development'
+  end
+end
+
+class Rails
+  def self.root
+    Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../../')))
+  end
+
+  def self.env
+    MockRailsEnv.new('production')
+  end
+end
+
 class WorkhorseTest < ActiveSupport::TestCase
   def setup
     Workhorse::DbJob.delete_all
@@ -31,6 +56,14 @@ class WorkhorseTest < ActiveSupport::TestCase
     end
   end
 
+  def work_until(max: 50, interval: 0.1, **options, &block)
+    w = Workhorse::Worker.new(**options)
+    w.start
+    return with_retries(max, interval: interval, &block)
+  ensure
+    w.shutdown
+  end
+
   def with_worker(options = {})
     w = Workhorse::Worker.new(**options)
     w.start
@@ -40,6 +73,18 @@ class WorkhorseTest < ActiveSupport::TestCase
       w.shutdown
     end
   end
+
+  def with_retries(max = 50, interval: 0.1, &_block)
+    runs = 0
+
+    loop do
+      return yield
+    rescue Minitest::Assertion => e
+      fail if runs > max
+      sleep interval
+      runs += 1
+    end
+  end
 end
 
 ActiveRecord::Base.establish_connection(

data/test/workhorse/worker_test.rb

@@ -180,8 +180,86 @@ class Workhorse::WorkerTest < WorkhorseTest
     assert_equal 'waiting',   jobs[1].state
   end
 
+  def test_controlled_shutdown
+    remove_pids!
+
+    Workhorse.max_worker_memory_mb = 50
+
+    daemon = start_daemon
+
+    pid = with_retries do
+      pid = daemon.workers.first.pid
+      assert_process(pid)
+      pid
+    end
+
+    10.times do
+      Workhorse.enqueue BasicJob.new(sleep_time: 0.1)
+
+      with_retries do
+        assert_equal 'succeeded', Workhorse::DbJob.first.state
+        Workhorse::DbJob.delete_all
+      end
+    end
+
+    Workhorse.enqueue MemHungryJob.new
+
+    with_retries do
+      assert_equal 'succeeded', Workhorse::DbJob.first.state
+
+      assert File.exist?("tmp/pids/workhorse.#{pid}.shutdown")
+      assert_not_process pid
+    end
+
+    daemon.watch
+
+    with_retries do
+      assert_not File.exist?("tmp/pids/workhorse.#{pid}.shutdown")
+    end
+  ensure
+    daemon.stop
+    Workhorse.max_worker_memory_mb = 0
+  end
+
   private
 
+  def remove_pids!
+    Dir[Rails.root.join('tmp', 'pids', '*')].each do |file|
+      FileUtils.rm file
+    end
+  end
+
+  def start_daemon
+    daemon = Workhorse::Daemon.new(pidfile: 'tmp/pids/test%s.pid') do |d|
+      d.worker 'Test Worker' do
+        begin
+          Workhorse::Worker.start_and_wait(
+            pool_size:        1,
+            polling_interval: 0.1,
+            logger:           ActiveSupport::Logger.new('tmp/log.log')
+          )
+        end
+      end
+    end
+    daemon.start
+    return daemon
+  end
+
+  def assert_process(pid)
+    assert process?(pid), "Process #{pid} expected to be running"
+  end
+
+  def assert_not_process(pid)
+    assert_not process?(pid), "Process #{pid} expected to be stopped"
+  end
+
+  def process?(pid)
+    Process.kill(0, pid)
+    true
+  rescue Errno::EPERM, Errno::ESRCH
+    false
+  end
+
   def enqueue_in_multiple_queues
     Workhorse.enqueue BasicJob.new(some_param: nil)
     Workhorse.enqueue BasicJob.new(some_param: :q1), queue: :q1

data/workhorse.gemspec

@@ -1,48 +1,31 @@
 # -*- encoding: utf-8 -*-
-# stub: workhorse 1.2.16 ruby lib
+# stub: workhorse 1.2.17.rc1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "workhorse".freeze
-  s.version = "1.2.16"
+  s.version = "1.2.17.rc1"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Sitrox".freeze]
-  s.date = "2023-09-18"
+  s.date = "2024-02-08"
   s.files = [".github/workflows/ruby.yml".freeze, ".gitignore".freeze, ".releaser_config".freeze, ".rubocop.yml".freeze, "CHANGELOG.md".freeze, "FAQ.md".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "RUBY_VERSION".freeze, "Rakefile".freeze, "VERSION".freeze, "bin/rubocop".freeze, "lib/active_job/queue_adapters/workhorse_adapter.rb".freeze, "lib/generators/workhorse/install_generator.rb".freeze, "lib/generators/workhorse/templates/bin/workhorse.rb".freeze, "lib/generators/workhorse/templates/config/initializers/workhorse.rb".freeze, "lib/generators/workhorse/templates/create_table_jobs.rb".freeze, "lib/workhorse.rb".freeze, "lib/workhorse/active_job_extension.rb".freeze, "lib/workhorse/daemon.rb".freeze, "lib/workhorse/daemon/shell_handler.rb".freeze, "lib/workhorse/db_job.rb".freeze, "lib/workhorse/enqueuer.rb".freeze, "lib/workhorse/jobs/cleanup_succeeded_jobs.rb".freeze, "lib/workhorse/jobs/detect_stale_jobs_job.rb".freeze, "lib/workhorse/jobs/run_active_job.rb".freeze, "lib/workhorse/jobs/run_rails_op.rb".freeze, "lib/workhorse/performer.rb".freeze, "lib/workhorse/poller.rb".freeze, "lib/workhorse/pool.rb".freeze, "lib/workhorse/scoped_env.rb".freeze, "lib/workhorse/worker.rb".freeze, "test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze, "workhorse.gemspec".freeze]
-  s.rubygems_version = "3.1.2".freeze
+  s.rubygems_version = "3.4.6".freeze
   s.summary = "Multi-threaded job backend with database queuing for ruby.".freeze
   s.test_files = ["test/active_job/queue_adapters/workhorse_adapter_test.rb".freeze, "test/lib/db_schema.rb".freeze, "test/lib/jobs.rb".freeze, "test/lib/test_helper.rb".freeze, "test/workhorse/db_job_test.rb".freeze, "test/workhorse/enqueuer_test.rb".freeze, "test/workhorse/performer_test.rb".freeze, "test/workhorse/poller_test.rb".freeze, "test/workhorse/pool_test.rb".freeze, "test/workhorse/worker_test.rb".freeze]
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 4
-  end
+  s.specification_version = 4
 
-  if s.respond_to? :add_runtime_dependency then
-    s.add_development_dependency(%q<bundler>.freeze, [">= 0"])
-    s.add_development_dependency(%q<rake>.freeze, [">= 0"])
-    s.add_development_dependency(%q<rubocop>.freeze, ["~> 1.28.0"])
-    s.add_development_dependency(%q<minitest>.freeze, [">= 0"])
-    s.add_development_dependency(%q<mysql2>.freeze, [">= 0"])
-    s.add_development_dependency(%q<colorize>.freeze, [">= 0"])
-    s.add_development_dependency(%q<benchmark-ips>.freeze, [">= 0"])
-    s.add_development_dependency(%q<activejob>.freeze, [">= 0"])
-    s.add_development_dependency(%q<pry>.freeze, [">= 0"])
-    s.add_runtime_dependency(%q<activesupport>.freeze, [">= 0"])
-    s.add_runtime_dependency(%q<activerecord>.freeze, [">= 0"])
-    s.add_runtime_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
-  else
-    s.add_dependency(%q<bundler>.freeze, [">= 0"])
-    s.add_dependency(%q<rake>.freeze, [">= 0"])
-    s.add_dependency(%q<rubocop>.freeze, ["~> 1.28.0"])
-    s.add_dependency(%q<minitest>.freeze, [">= 0"])
-    s.add_dependency(%q<mysql2>.freeze, [">= 0"])
-    s.add_dependency(%q<colorize>.freeze, [">= 0"])
-    s.add_dependency(%q<benchmark-ips>.freeze, [">= 0"])
-    s.add_dependency(%q<activejob>.freeze, [">= 0"])
-    s.add_dependency(%q<pry>.freeze, [">= 0"])
-    s.add_dependency(%q<activesupport>.freeze, [">= 0"])
-    s.add_dependency(%q<activerecord>.freeze, [">= 0"])
-    s.add_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
-  end
+  s.add_development_dependency(%q<bundler>.freeze, [">= 0"])
+  s.add_development_dependency(%q<rake>.freeze, [">= 0"])
+  s.add_development_dependency(%q<rubocop>.freeze, ["~> 1.28.0"])
+  s.add_development_dependency(%q<minitest>.freeze, [">= 0"])
+  s.add_development_dependency(%q<mysql2>.freeze, [">= 0"])
+  s.add_development_dependency(%q<colorize>.freeze, [">= 0"])
+  s.add_development_dependency(%q<benchmark-ips>.freeze, [">= 0"])
+  s.add_development_dependency(%q<activejob>.freeze, [">= 0"])
+  s.add_development_dependency(%q<pry>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<activesupport>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<activerecord>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<concurrent-ruby>.freeze, [">= 0"])
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workhorse
 version: !ruby/object:Gem::Version
-  version: 1.2.16
+  version: 1.2.17.rc1
 platform: ruby
 authors:
 - Sitrox
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-18 00:00:00.000000000 Z
+date: 2024-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -242,11 +242,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Multi-threaded job backend with database queuing for ruby.