delayed_job_master 2.0.3 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2352f0e35b9bf06ef9362e797ea35d8b848583d03d8aea62235497dc8b938cc9
-  data.tar.gz: 205e1a4b04a8ad361ff0122acc65bcaa02439eaf5bbe91a258652dfa02d79eb9
+  metadata.gz: dbc7748a97c2f7aa0869b23cc1e00fbc8682d9f527e9c151d77a018c227c218e
+  data.tar.gz: aa7ba20301d0881e0a76ae66e70a72a10b9baed00cd0137c0a7118f6fbdd1ceb
 SHA512:
-  metadata.gz: f02b0683502467943f0e1e81543705b7d69febdcf22a846d05525a95fa13844552c6d3215df4f2b87fbfdf859d9178eb18350d746a8f92743752ac0cfa2f44f2
-  data.tar.gz: 104a01e051fce6f6fba061c38e1674a8c3a8bd23d9ae367399a9825fc0a560fc8657dd863d6dca053c734761b93160d70b50c5356343ec12320b572b8937f019
+  metadata.gz: 52b34fa14373943cdaf4265a6076781d3e032916e8cc3b1f33c8da1a7fa4986309f99a329798213212dec17c1e0a8ba5c3ab4947b54802156e6dce30e6d044ea
+  data.tar.gz: 63ef8e75c2ee50e90e173a77b3d8a466208da366decf2caf199104e3468ce126cb86f4d2b8fdc3066795c8e699c41b8bc61b3475ab23d61ffc02c8ae582b75bf

data/.github/workflows/ci.yml

@@ -5,48 +5,41 @@ on: [push, pull_request]
 jobs:
   test:
     runs-on: ubuntu-20.04
+    services:
+      postgres:
+        image: postgres:9.5
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        ports:
+          - 3306:3306
+        options: --health-cmd "mysqladmin ping -h localhost" --health-interval 20s --health-timeout 10s --health-retries 10
     strategy:
       fail-fast: false
       matrix:
-        ruby: [2.3, 2.4, 2.5, 2.6, 2.7, '3.0', 3.1]
-        gemfile: ['rails50', 'rails51', 'rails52', 'rails60', 'rails61', 'rails70']
-        exclude:
-          - ruby: 2.3
-            gemfile: rails60
-          - ruby: 2.3
-            gemfile: rails61
-          - ruby: 2.3
-            gemfile: rails70
-          - ruby: 2.4
-            gemfile: rails60
-          - ruby: 2.4
-            gemfile: rails61
-          - ruby: 2.4
-            gemfile: rails70
-          - ruby: 2.5
-            gemfile: rails70
-          - ruby: 2.6
-            gemfile: rails70
-          - ruby: 3.0
-            gemfile: rails50
-          - ruby: 3.0
-            gemfile: rails51
-          - ruby: 3.0
-            gemfile: rails52
-          - ruby: 3.1
-            gemfile: rails50
-          - ruby: 3.1
-            gemfile: rails51
-          - ruby: 3.1
-            gemfile: rails52
+        ruby: [2.7, '3.0', 3.1, 3.2]
+        gemfile: ['rails60', 'rails61', 'rails70']
+        database: ['postgresql', 'mysql']
+        database_config: ['default', 'multi']
 
-    name: ruby ${{ matrix.ruby }}, ${{ matrix.gemfile }}
+    name: ruby ${{ matrix.ruby }}, ${{ matrix.gemfile }}, ${{ matrix.database }}
 
     env:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      DATABASE: ${{ matrix.database }}
+      DATABASE_CONFIG: ${{ matrix.database_config }}
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
@@ -54,8 +47,12 @@ jobs:
     - name: Prepare test
       run: |
         cd spec/dummy
-        BUNDLE_GEMFILE=../../${{ env.BUNDLE_GEMFILE }} RAILS_ENV=test bundle exec rake dbs:migrate
+        BUNDLE_GEMFILE=../../${{ env.BUNDLE_GEMFILE }} RAILS_ENV=test bundle exec rake db:create db:migrate
         cd ../..
     - name: Run test
       run: |
         bundle exec rspec
+    - name: Show log
+      run: |
+        cat spec/dummy/log/test.log
+      if: ${{ failure() }}

data/.gitignore

@@ -1,11 +1,9 @@
 /.bundle/
-/.project
 /Gemfile.lock
 /gemfiles/*gemfile.lock
 /coverage/
 /pkg/
 /spec/reports/
-/spec/**/db/*.sqlite3
 /spec/**/log/*.log
 /spec/**/tmp/*
 /vendor/

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # CHANGELOG
 
+## 3.0.0
+
+* Add multithread feature for workers.
+* Add graceful stop feature by WINCH signal.
+* Add command line options.
+* Add listen/notify feature for postgresql.
+* Support multiple callbacks.
+* Separate waitpid thread and job checker thread.
+* Rename some configurations (`worker.count` to `worker.max_processes`, `monitor_wait` to `monitor_interval`).
+* Use LIMIT query instead of COUNT query for finding jobs.
+* Patch default SQL of delayed_job for postgresql.
+* Drop support for ruby < 2.7 and rails < 6.0.
+
 ## 2.0.3
 
 * Fix database config detection for rails 7.0.

data/README.md

@@ -4,26 +4,26 @@ A simple delayed_job master process to control multiple workers.
 
 ## Features
 
-* Preload application and fork workers fastly.
-* Monitor job queues and fork new workers on demand.
-* Trap signals to restart / reopen log files.
-* Support multiple databases.
+* Support concurrent workers with multiprocess and multithread.
+* Preload application and fork worker processes on demand.
+* Check memory usage after processing a job to kill workers consuming large memory.
+* Support signals for restarting master process / reopening log files.
+* Check new jobs by polling multiple databases having delayed_jobs table.
+* Detect new jobs quickly by using listen/notify feature. (only postgresql)
 
 ## Dependencies
 
-* ruby 2.3+
+* ruby 2.7+
+* activesupport 6.0+
 * delayed_job 4.1
-
-## Supported delayed_job backends
-
-* delayed_job_active_record 4.1
+* delayed_job_active_record 4.1 (execlude sqlite)
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'delayed_job_master', require: false
+gem 'delayed_job_master'
 ```
 
 And then execute:
@@ -36,7 +36,7 @@ Generate `bin/delayed_job_master` and `config/delayed_job_master.rb`:
 
     $ rails generate delayed_job_master:config
 
-## Configuration
+## Configurations
 
 Edit `config/delayed_job_master.rb`:
 
@@ -44,8 +44,11 @@ Edit `config/delayed_job_master.rb`:
 # working directory
 working_directory Dir.pwd
 
-# monitor wait time in second
-monitor_wait 5
+# monitor interval for events (in seconds)
+monitor_interval 5
+
+# polling interval for new jobs (in seconds)
+polling_interval 5
 
 # path to pid file
 pid_file "#{Dir.pwd}/tmp/pids/delayed_job_master.pid"
@@ -56,16 +59,19 @@ log_file "#{Dir.pwd}/log/delayed_job_master.log"
 # log level
 log_level :info
 
-# databases for checking queued jobs if you have multiple databases
-# databases [:production]
+# databases for checking new jobs in case multiple databases
+# databases [:primary, :secondary]
 
 # worker1
 add_worker do |worker|
   # queue name for the worker
   worker.queues %w(queue1)
 
-  # worker count
-  worker.count 1
+  # max process count
+  worker.max_processes 1
+
+  # max thread count for each worker
+  worker.max_threads 1
 
   # max memory in MB
   worker.max_memory 300
@@ -83,22 +89,27 @@ end
 # worker2
 add_worker do |worker|
   worker.queues %w(queue2)
-  worker.count 2
+  worker.max_processes 2
+  worker.max_threads 2
 end
 
 before_fork do |master, worker|
-  Delayed::Worker.before_fork if defined?(Delayed::Worker)
+  ActiveRecord::Base.connection.disconnect!
 end
 
 after_fork do |master, worker|
-  Delayed::Worker.after_fork if defined?(Delayed::Worker)
+  ActiveRecord::Base.establish_connection
 end
+```
 
-before_monitor do |master|
-  ActiveRecord::Base.connection.verify! if defined?(ActiveRecord::Base)
-end
+### Listen/notify
+
+Listen/notify feature is enabled by default if `pg` gem is detected.
+To disable this feature, put `config/initializers/delayed_job_master.rb` as follows:
 
-after_monitor do |master|
+```ruby
+DelayedJobMaster.configure do |config|
+  config.listener = nil
 end
 ```
 
@@ -113,11 +124,15 @@ Command line options:
 * -c, --config: Specify configuration file.
 * -D, --daemon: Start master as a daemon.
 
-Stop master and workers gracefully:
+Stop master immediately, stop workers gracefully:
 
     $ kill -TERM `cat tmp/pids/delayed_job_master.pid`
 
-Stop master and workers forcefully:
+Stop master and workers gracefully:
+
+    $ kill -WINCH `cat tmp/pids/delayed_job_master.pid`
+
+Stop master and workers immediately:
 
     $ kill -QUIT `cat tmp/pids/delayed_job_master.pid`
 
@@ -125,31 +140,30 @@ Reopen log files:
 
     $ kill -USR1 `cat tmp/pids/delayed_job_master.pid`
 
-Restart gracefully:
+Restart master immediately, stop workers gracefully:
 
     $ kill -USR2 `cat tmp/pids/delayed_job_master.pid`
 
 Workers handle each signal as follows:
 
-* TERM: Workers stop after finishing current jobs.
+* TERM/WINCH/USR2: Workers stop after finishing current jobs.
 * QUIT: Workers are killed immediately.
 * USR1: Workers reopen log files.
-* USR2: New workers start, old workers stop after finishing current jobs.
 
-## Worker status
+### Worker status
 
 `ps` command shows worker status as follows:
 
 ```
 $ ps aux
-... delayed_job.0: worker[0] (queue1) [BUSY]  # BUSY process is currently proceeding some jobs
+... delayed_job: worker[0] (queue1) [BUSY]  # BUSY process is currently proceeding some jobs
 ```
 
 After graceful restart, you may find OLD process.
 
 ```
 $ ps aux
-... delayed_job.0: worker[0] (queue1) [BUSY] [OLD]  # OLD process will stop after finishing current jobs.
+... delayed_job: worker[0] (queue1) [BUSY] [OLD]  # OLD process will stop after finishing current jobs.
 ```
 
 ## Contributing
@@ -159,4 +173,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/kanety
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/delayed_job_master.gemspec

@@ -1,11 +1,11 @@
 # coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'delayed/master/version'
+require 'delayed_job_master/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "delayed_job_master"
-  spec.version       = Delayed::Master::VERSION
+  spec.version       = DelayedJobMaster::VERSION
   spec.authors       = ["Yoshikazu Kaneta"]
   spec.email         = ["kaneta@sitebridge.co.jp"]
   spec.summary       = %q{A simple delayed_job master process to control multiple workers}
@@ -18,15 +18,18 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = '>= 2.3'
+  spec.required_ruby_version = '>= 2.7'
 
   spec.add_dependency "delayed_job", ">= 4.1"
   spec.add_dependency "get_process_mem"
+  spec.add_dependency "activesupport", ">= 6.0"
 
-  spec.add_development_dependency "rails"
+  spec.add_development_dependency "rails", ">= 6.0"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec-rails"
   spec.add_development_dependency "simplecov"
-  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "pg"
+  spec.add_development_dependency "mysql2"
   spec.add_development_dependency "delayed_job_active_record", ">= 4.1"
+  spec.add_development_dependency "delayed_job_bulk", ">= 1.0.0"
 end

data/lib/delayed/master/callbacks.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Delayed
+  module Master
+    class Callbacks
+      def initialize(config)
+        @config = config
+      end
+
+      def call(event, *args, &block)
+        run(:"before_#{event}", *args)
+        result = run(:"around_#{event}", *args, &block)
+        run(:"after_#{event}", *args)
+        result
+      end
+
+      def run(name, *args, &block)
+        callbacks = get_callbacks(name)
+        if block
+          callbacks.reverse.reduce(block) { |ret, c| -> { c.call(*args, &ret) } }.call
+        else
+          callbacks.each { |c| c.call(*args) }
+        end
+      end
+
+      private
+
+      def get_callbacks(name)
+        if @config.respond_to?(name)
+          @config.send(name).to_a
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/delayed/master/command.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require 'optparse'
 require_relative 'config'
 
 module Delayed
-  class Master
+  module Master
     class Command
       attr_reader :config
 
@@ -11,7 +13,7 @@ module Delayed
 
         OptionParser.new { |opt|
           opt.banner = <<-EOS
-            #{File.basename($PROGRAM_NAME)} #{Delayed::Master::VERSION}
+            #{File.basename($PROGRAM_NAME)} #{DelayedJobMaster::VERSION}
             Usage: #{File.basename($PROGRAM_NAME)} [options]
           EOS
 
@@ -19,12 +21,33 @@ module Delayed
             puts opt
             exit
           end
-          opt.on('-D', '--daemon', 'Start master as a daemon') do |boolean|
-            @config.daemon(boolean)
-          end
           opt.on('-c', '--config=FILE', 'Specify config file') do |file|
             @config.read(file)
           end
+          opt.on('-D', '--daemon', 'Start master as a daemon') do |boolean|
+            @config.daemon = boolean
+          end
+          opt.on('--working-directory=DIR', 'Path to working directory') do |dir|
+            @config.working_directory = dir
+          end
+          opt.on('--pid-file=FILE', 'Path to pid file') do |file|
+            @config.pid_file = file
+          end
+          opt.on('--log-file=FILE', 'Path to log file') do |file|
+            @config.log_file = file
+          end
+          opt.on('--log-level=LEVEL', 'Log level') do |level|
+            @config.log_level = level.to_sym
+          end
+          opt.on('--monitor-interval=INTERVAL', Numeric, 'Monitor interval') do |interval|
+            @config.monitor_interval = interval
+          end
+          opt.on('--polling-interval=INTERVAL', Numeric, 'Polling interval') do |interval|
+            @config.polling_interval = interval
+          end
+          opt.on('--databases=DB1,DB2', Array, 'Database spec name to check delayed_jobs table') do |databases|
+            @config.databases = databases.map(&:to_sym)
+          end
         }.parse(args)
       end
     end

data/lib/delayed/master/config.rb

@@ -1,45 +1,57 @@
+# frozen_string_literal: true
+
+require_relative 'worker_setting'
+
 module Delayed
-  class Master
+  module Master
     class Config
-      SIMPLE_CONFIGS   = [:working_directory, :log_file, :log_level, :pid_file, :monitor_wait, :daemon, :databases]
-      CALLBACK_CONFIGS = [:before_fork, :after_fork, :before_monitor, :after_monitor]
+      SIMPLE_CONFIGS   = [:daemon, :working_directory, :log_file, :log_level, :pid_file,
+                          :monitor_interval, :polling_interval, :databases]
+      CALLBACK_CONFIGS = [:before_fork, :after_fork,
+                          :before_monitor, :after_monitor, :around_monitor,
+                          :before_polling, :after_polling, :around_polling]
 
-      attr_reader :data, :workers
+      attr_accessor *SIMPLE_CONFIGS
+      attr_accessor *CALLBACK_CONFIGS
+      attr_reader :workers
 
       def initialize(file = nil)
-        @data = {}
+        @daemon = false
+        @working_directory = Dir.pwd
+        @pid_file = "#{@working_directory}/tmp/pids/delayed_job_master.pid"
+        @log_file = "#{@working_directory}/log/delayed_job_master.log"
+        @log_level = :info
+        @monitor_interval = 5
+        @polling_interval = 5
+        @databases = []
+        CALLBACK_CONFIGS.each do |name|
+          send("#{name}=", [])
+        end
         @workers = []
         read(file) if file
       end
 
-      def worker_settings
-        @workers
-      end
-
       def read(file)
         instance_eval(File.read(file), file)
       end
 
-      def add_worker
-        worker = WorkerSetting.new(id: @workers.size, queues: [], count: 1, exit_on_complete: true)
-        yield worker
-        @workers << worker
-      end
-
-      def callbacks
-        @data.select { |k, _| CALLBACK_CONFIGS.include?(k) }
+      def worker_settings
+        @workers
       end
 
-      def run_callback(key, *args)
-        @data[key].call(*args)
+      def add_worker
+        worker = WorkerSetting.new(id: @workers.size)
+        yield worker if block_given?
+        @workers << worker
+        worker
       end
 
       SIMPLE_CONFIGS.each do |key|
-        define_method(key) do |value = nil|
-          if !value.nil?
-            @data[key] = value
+        define_method(key) do |*args|
+          if args.size > 0
+            instance_variable_set("@#{key}", args[0])
           else
-            @data[key]
+            instance_variable_get("@#{key}")
           end
         end
       end
@@ -47,48 +59,29 @@ module Delayed
       CALLBACK_CONFIGS.each do |key|
         define_method(key) do |&block|
           if block
-            @data[key] = block
+            instance_variable_get("@#{key}") << block
           else
-            @data[key]
+            instance_variable_get("@#{key}")
           end
         end
       end
 
-      class WorkerSetting
-        SIMPLE_CONFIGS = [:id, :count, :max_memory,
-                          :min_priority, :max_priority, :sleep_delay, :read_ahead, :exit_on_complete,
-                          :max_attempts, :max_run_time, :destroy_failed_jobs]
-        ARRAY_CONFIGS  = [:queues]
-
-        attr_reader :data
-
-        def initialize(default = {})
-          @data = default
-        end
-
-        def control(value = nil)
-          puts "DEPRECATION WARNING: deprecated control setting is called from #{caller(1, 1).first}. Remove it from your config file."
-        end
-
-        SIMPLE_CONFIGS.each do |key|
-          define_method(key) do |value = nil|
-            if !value.nil?
-              @data[key] = value
-            else
-              @data[key]
-            end
-          end
-        end
+      def monitor_wait(value = nil)
+        ActiveSupport::Deprecation.warn <<-TEXT.squish
+          deprecated 'monitor_wait' setting was used. Use 'monitor_interval' instead."
+        TEXT
+        @monitor_interval = @polling_interval = value
+      end
 
-        ARRAY_CONFIGS.each do |key|
-          define_method(key) do |value = nil|
-            if value
-              @data[key] = Array(value)
-            else
-              @data[key]
-            end
-          end
+      def abstract_texts
+        texts = []
+        texts << "databases: #{@databases.join(', ')}" if @databases.present?
+        texts += @workers.map do |worker|
+          str = "worker[#{worker.id}]: #{worker.max_processes} processes, #{worker.max_threads} threads"
+          str += " (#{worker.queues.join(', ')})" if worker.queues.present?
+          str
         end
+        texts
       end
     end
   end