delayed_job_master 2.0.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +31 -34
  3. data/.gitignore +0 -2
  4. data/CHANGELOG.md +13 -0
  5. data/README.md +46 -33
  6. data/delayed_job_master.gemspec +8 -5
  7. data/lib/delayed/master/callbacks.rb +37 -0
  8. data/lib/delayed/master/command.rb +28 -5
  9. data/lib/delayed/master/config.rb +51 -58
  10. data/lib/delayed/master/core.rb +146 -0
  11. data/lib/delayed/master/database.rb +72 -0
  12. data/lib/delayed/master/file_reopener.rb +18 -0
  13. data/lib/delayed/master/forker.rb +30 -19
  14. data/lib/delayed/master/job_checker.rb +91 -47
  15. data/lib/delayed/master/job_finder.rb +31 -0
  16. data/lib/delayed/master/job_listener.rb +31 -0
  17. data/lib/delayed/master/monitoring.rb +37 -36
  18. data/lib/delayed/master/postgresql/job_listener.rb +73 -0
  19. data/lib/delayed/master/postgresql/job_notifier.rb +45 -0
  20. data/lib/delayed/master/safe_array.rb +30 -0
  21. data/lib/delayed/master/signaler.rb +17 -7
  22. data/lib/delayed/master/sleep.rb +18 -0
  23. data/lib/delayed/master/worker/backend/active_record.rb +41 -0
  24. data/lib/delayed/master/worker/extension.rb +14 -0
  25. data/lib/delayed/master/worker/lifecycle.rb +10 -0
  26. data/lib/delayed/master/worker/plugins/all.rb +17 -0
  27. data/lib/delayed/master/worker/plugins/executor_wrapper.rb +23 -0
  28. data/lib/delayed/master/worker/plugins/memory_checker.rb +28 -0
  29. data/lib/delayed/master/worker/plugins/signal_handler.rb +35 -0
  30. data/lib/delayed/master/worker/plugins/status_notifier.rb +21 -0
  31. data/lib/delayed/master/worker/thread_pool.rb +64 -0
  32. data/lib/delayed/master/worker/thread_worker.rb +65 -0
  33. data/lib/delayed/master/worker.rb +8 -7
  34. data/lib/delayed/master/worker_setting.rb +72 -0
  35. data/lib/delayed/master.rb +7 -100
  36. data/lib/delayed_job_master/railtie.rb +16 -0
  37. data/lib/delayed_job_master/version.rb +5 -0
  38. data/lib/delayed_job_master.rb +15 -1
  39. data/lib/generators/delayed_job_master/templates/config.rb +14 -14
  40. data/lib/generators/delayed_job_master/templates/script +1 -1
  41. metadata +71 -17
  42. data/gemfiles/rails50.gemfile +0 -7
  43. data/gemfiles/rails51.gemfile +0 -7
  44. data/gemfiles/rails52.gemfile +0 -7
  45. data/lib/delayed/master/database_detector.rb +0 -37
  46. data/lib/delayed/master/job_counter.rb +0 -26
  47. data/lib/delayed/master/plugins/memory_checker.rb +0 -24
  48. data/lib/delayed/master/plugins/signal_handler.rb +0 -31
  49. data/lib/delayed/master/plugins/status_notifier.rb +0 -17
  50. data/lib/delayed/master/util/file_reopener.rb +0 -18
  51. data/lib/delayed/master/version.rb +0 -5
  52. data/lib/delayed/master/worker_extension.rb +0 -19
@@ -1,105 +1,12 @@
1
- require 'fileutils'
2
- require 'logger'
3
- require_relative 'master/version'
4
- require_relative 'master/command'
5
- require_relative 'master/worker'
6
- require_relative 'master/monitoring'
7
- require_relative 'master/signaler'
8
- require_relative 'master/util/file_reopener'
1
+ # frozen_string_literal: true
9
2
 
10
- module Delayed
11
- class Master
12
- attr_reader :config, :logger, :workers
13
-
14
- def initialize(argv)
15
- @config = Command.new(argv).config
16
- @logger = setup_logger(@config.log_file, @config.log_level)
17
- @workers = []
18
-
19
- @signaler = Signaler.new(self)
20
- @monitoring = Monitoring.new(self)
21
- end
22
-
23
- def run
24
- print_config
25
- daemonize if @config.daemon
3
+ require_relative 'master/core'
26
4
 
27
- @logger.info "started master #{Process.pid}".tap { |msg| puts msg }
28
-
29
- handle_pid_file do
30
- @signaler.register
31
- @prepared = true
32
- @monitoring.monitor_while { stop? }
33
- end
34
-
35
- @logger.info "shut down master"
36
- end
37
-
38
- def prepared?
39
- @prepared
40
- end
41
-
42
- def quit
43
- @signaler.dispatch(:KILL)
44
- @stop = true
45
- end
46
-
47
- def stop
48
- @signaler.dispatch(:TERM)
49
- @stop = true
50
- end
51
-
52
- def stop?
53
- @stop == true
54
- end
55
-
56
- def reopen_files
57
- @signaler.dispatch(:USR1)
58
- @logger.info "reopening files..."
59
- Util::FileReopener.reopen
60
- @logger.info "reopened"
61
- end
62
-
63
- def restart
64
- @signaler.dispatch(:USR2)
65
- @logger.info "restarting master..."
66
- exec(*([$0] + ARGV))
67
- end
68
-
69
- private
70
-
71
- def setup_logger(log_file, log_level)
72
- FileUtils.mkdir_p(File.dirname(log_file)) if log_file.is_a?(String)
73
- logger = Logger.new(log_file)
74
- logger.level = log_level
75
- logger
76
- end
77
-
78
- def daemonize
79
- Process.daemon(true)
80
- end
81
-
82
- def handle_pid_file
83
- create_pid_file
84
- yield
85
- remove_pid_file
86
- end
87
-
88
- def create_pid_file
89
- FileUtils.mkdir_p(File.dirname(@config.pid_file))
90
- File.write(@config.pid_file, Process.pid)
91
- end
92
-
93
- def remove_pid_file
94
- File.delete(@config.pid_file) if File.exist?(@config.pid_file)
95
- end
96
-
97
- def print_config
98
- @logger.info "databases: #{@config.databases.join(', ')}" if @config.databases
99
- @config.worker_settings.each do |setting|
100
- message = "worker[#{setting.id}]: #{setting.count} processes"
101
- message << " (#{setting.queues.join(', ')})" if setting.queues.respond_to?(:join)
102
- @logger.info message
5
+ module Delayed
6
+ module Master
7
+ class << self
8
+ def new(argv)
9
+ Core.new(argv)
103
10
  end
104
11
  end
105
12
  end
@@ -0,0 +1,16 @@
1
+ module DelayedJobMaster
2
+ class Railtie < Rails::Railtie
3
+ config.after_initialize do
4
+ case DelayedJobMaster.config.listener
5
+ when :postgresql
6
+ require_relative '../delayed/master/postgresql/job_notifier'
7
+ if defined?(Delayed::Backend::ActiveRecord)
8
+ Delayed::Backend::ActiveRecord::Job.include Delayed::Master::Postgresql::JobNotifier
9
+ end
10
+ if defined?(Delayed::Backend::Bulk)
11
+ Delayed::Backend::Bulk.include Delayed::Master::Postgresql::BulkJobNotifier
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DelayedJobMaster
4
+ VERSION = "3.0.0"
5
+ end
@@ -1 +1,15 @@
1
- require 'delayed/master'
1
+ require 'active_support'
2
+
3
+ require_relative 'delayed_job_master/version'
4
+ require_relative 'delayed_job_master/railtie' if defined?(Rails)
5
+
6
+ module DelayedJobMaster
7
+ mattr_accessor :config, default: ActiveSupport::InheritableOptions.new
8
+ config.listener = :postgresql if defined?(PG)
9
+
10
+ class << self
11
+ def configure
12
+ yield config
13
+ end
14
+ end
15
+ end
@@ -1,8 +1,11 @@
1
1
  # working directory
2
2
  working_directory Dir.pwd
3
3
 
4
- # monitor wait time in second
5
- monitor_wait 5
4
+ # monitor interval for events (in seconds)
5
+ monitor_interval 5
6
+
7
+ # polling interval for new jobs (in seconds)
8
+ polling_interval 5
6
9
 
7
10
  # path to pid file
8
11
  pid_file "#{Dir.pwd}/tmp/pids/delayed_job_master.pid"
@@ -18,9 +21,12 @@ add_worker do |worker|
18
21
  # queue name for the worker
19
22
  worker.queues %w(queue1)
20
23
 
21
- # worker count
22
- worker.count 1
24
+ # max process count
25
+ worker.max_processes 1
23
26
 
27
+ # max thread count for each worker
28
+ worker.max_threads 1
29
+
24
30
  # max memory in MB
25
31
  worker.max_memory 300
26
32
 
@@ -37,20 +43,14 @@ end
37
43
  # worker2
38
44
  add_worker do |worker|
39
45
  worker.queues %w(queue2)
40
- worker.count 2
46
+ worker.max_processes 2
47
+ worker.max_threads 2
41
48
  end
42
49
 
43
50
  before_fork do |master, worker|
44
- Delayed::Worker.before_fork if defined?(Delayed::Worker)
51
+ ActiveRecord::Base.connection.disconnect!
45
52
  end
46
53
 
47
54
  after_fork do |master, worker|
48
- Delayed::Worker.after_fork if defined?(Delayed::Worker)
49
- end
50
-
51
- before_monitor do |master|
52
- ActiveRecord::Base.connection.verify! if defined?(ActiveRecord::Base)
53
- end
54
-
55
- after_monitor do |master|
55
+ ActiveRecord::Base.establish_connection
56
56
  end
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
3
  require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
4
- require 'delayed_job_master'
4
+ require 'delayed/master'
5
5
 
6
6
  Delayed::Master.new(ARGV).run
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: delayed_job_master
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.3
4
+ version: 3.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yoshikazu Kaneta
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-08-04 00:00:00.000000000 Z
11
+ date: 2023-05-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: delayed_job
@@ -38,20 +38,34 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: activesupport
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '6.0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '6.0'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: rails
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
45
59
  - - ">="
46
60
  - !ruby/object:Gem::Version
47
- version: '0'
61
+ version: '6.0'
48
62
  type: :development
49
63
  prerelease: false
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
66
  - - ">="
53
67
  - !ruby/object:Gem::Version
54
- version: '0'
68
+ version: '6.0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: rake
57
71
  requirement: !ruby/object:Gem::Requirement
@@ -95,7 +109,21 @@ dependencies:
95
109
  - !ruby/object:Gem::Version
96
110
  version: '0'
97
111
  - !ruby/object:Gem::Dependency
98
- name: sqlite3
112
+ name: pg
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: mysql2
99
127
  requirement: !ruby/object:Gem::Requirement
100
128
  requirements:
101
129
  - - ">="
@@ -122,6 +150,20 @@ dependencies:
122
150
  - - ">="
123
151
  - !ruby/object:Gem::Version
124
152
  version: '4.1'
153
+ - !ruby/object:Gem::Dependency
154
+ name: delayed_job_bulk
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: 1.0.0
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: 1.0.0
125
167
  description: A simple delayed_job master process to control multiple workers
126
168
  email:
127
169
  - kaneta@sitebridge.co.jp
@@ -139,29 +181,41 @@ files:
139
181
  - README.md
140
182
  - Rakefile
141
183
  - delayed_job_master.gemspec
142
- - gemfiles/rails50.gemfile
143
- - gemfiles/rails51.gemfile
144
- - gemfiles/rails52.gemfile
145
184
  - gemfiles/rails60.gemfile
146
185
  - gemfiles/rails61.gemfile
147
186
  - gemfiles/rails70.gemfile
148
187
  - lib/delayed/master.rb
188
+ - lib/delayed/master/callbacks.rb
149
189
  - lib/delayed/master/command.rb
150
190
  - lib/delayed/master/config.rb
151
- - lib/delayed/master/database_detector.rb
191
+ - lib/delayed/master/core.rb
192
+ - lib/delayed/master/database.rb
193
+ - lib/delayed/master/file_reopener.rb
152
194
  - lib/delayed/master/forker.rb
153
195
  - lib/delayed/master/job_checker.rb
154
- - lib/delayed/master/job_counter.rb
196
+ - lib/delayed/master/job_finder.rb
197
+ - lib/delayed/master/job_listener.rb
155
198
  - lib/delayed/master/monitoring.rb
156
- - lib/delayed/master/plugins/memory_checker.rb
157
- - lib/delayed/master/plugins/signal_handler.rb
158
- - lib/delayed/master/plugins/status_notifier.rb
199
+ - lib/delayed/master/postgresql/job_listener.rb
200
+ - lib/delayed/master/postgresql/job_notifier.rb
201
+ - lib/delayed/master/safe_array.rb
159
202
  - lib/delayed/master/signaler.rb
160
- - lib/delayed/master/util/file_reopener.rb
161
- - lib/delayed/master/version.rb
203
+ - lib/delayed/master/sleep.rb
162
204
  - lib/delayed/master/worker.rb
163
- - lib/delayed/master/worker_extension.rb
205
+ - lib/delayed/master/worker/backend/active_record.rb
206
+ - lib/delayed/master/worker/extension.rb
207
+ - lib/delayed/master/worker/lifecycle.rb
208
+ - lib/delayed/master/worker/plugins/all.rb
209
+ - lib/delayed/master/worker/plugins/executor_wrapper.rb
210
+ - lib/delayed/master/worker/plugins/memory_checker.rb
211
+ - lib/delayed/master/worker/plugins/signal_handler.rb
212
+ - lib/delayed/master/worker/plugins/status_notifier.rb
213
+ - lib/delayed/master/worker/thread_pool.rb
214
+ - lib/delayed/master/worker/thread_worker.rb
215
+ - lib/delayed/master/worker_setting.rb
164
216
  - lib/delayed_job_master.rb
217
+ - lib/delayed_job_master/railtie.rb
218
+ - lib/delayed_job_master/version.rb
165
219
  - lib/generators/delayed_job_master/config_generator.rb
166
220
  - lib/generators/delayed_job_master/templates/config.rb
167
221
  - lib/generators/delayed_job_master/templates/script
@@ -177,7 +231,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
177
231
  requirements:
178
232
  - - ">="
179
233
  - !ruby/object:Gem::Version
180
- version: '2.3'
234
+ version: '2.7'
181
235
  required_rubygems_version: !ruby/object:Gem::Requirement
182
236
  requirements:
183
237
  - - ">="
@@ -1,7 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem "delayed_job_active_record", "~> 4.1.0"
4
- gem "rails", "~> 5.0.0"
5
- gem "sqlite3", "~> 1.3.6"
6
-
7
- gemspec path: "../"
@@ -1,7 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem "delayed_job_active_record", "~> 4.1.0"
4
- gem "rails", "~> 5.1.0"
5
- gem "sqlite3", "~> 1.3.6"
6
-
7
- gemspec path: "../"
@@ -1,7 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem "delayed_job_active_record", "~> 4.1.0"
4
- gem "rails", "~> 5.2.0"
5
- gem "sqlite3", "~> 1.3.6"
6
-
7
- gemspec path: "../"
@@ -1,37 +0,0 @@
1
- module Delayed
2
- class Master
3
- class DatabaseDetector
4
- def initialize
5
- end
6
-
7
- def call
8
- load_spec_names.select { |spec_name| has_delayed_job_table?(spec_name) }
9
- end
10
-
11
- private
12
-
13
- def load_spec_names
14
- if Rails::VERSION::MAJOR >= 6
15
- load_spec_names_from_multi_db_config
16
- else
17
- [Rails.env.to_sym]
18
- end
19
- end
20
-
21
- def load_spec_names_from_multi_db_config
22
- configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env)
23
- configs.reject!(&:replica?)
24
- if Rails::VERSION::MAJOR == 6
25
- configs.map { |c| c.spec_name.to_sym }
26
- else
27
- configs.map { |c| c.name.to_sym }
28
- end
29
- end
30
-
31
- def has_delayed_job_table?(spec_name)
32
- ActiveRecord::Base.establish_connection(spec_name)
33
- ActiveRecord::Base.connection.tables.include?('delayed_jobs')
34
- end
35
- end
36
- end
37
- end
@@ -1,26 +0,0 @@
1
- # JobCounter depends on delayed_job_active_record.
2
- # See https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/delayed/backend/active_record.rb
3
- module Delayed
4
- class Master
5
- class JobCounter
6
- def initialize(klass)
7
- @klass = klass
8
- end
9
-
10
- def count(setting)
11
- jobs = ready_to_run(setting.max_run_time || Delayed::Worker::DEFAULT_MAX_RUN_TIME)
12
- jobs.where!("priority >= ?", setting.min_priority) if setting.min_priority
13
- jobs.where!("priority <= ?", setting.max_priority) if setting.max_priority
14
- jobs.where!(queue: setting.queues) if setting.queues.any?
15
- jobs.count
16
- end
17
-
18
- private
19
-
20
- def ready_to_run(max_run_time)
21
- db_time_now = @klass.db_time_now
22
- @klass.where("(run_at <= ? AND (locked_at IS NULL OR locked_at < ?)) AND failed_at IS NULL", db_time_now, db_time_now - max_run_time)
23
- end
24
- end
25
- end
26
- end
@@ -1,24 +0,0 @@
1
- require 'get_process_mem'
2
-
3
- module Delayed
4
- class Master
5
- module Plugins
6
- class MemoryChecker < Delayed::Plugin
7
- callbacks do |lifecycle|
8
- lifecycle.before(:perform) do |worker, job|
9
- mem = GetProcessMem.new
10
- worker.master_logger.info "performing #{job.name}, memory: #{mem.mb.to_i} MB"
11
- end
12
- lifecycle.after(:perform) do |worker, job|
13
- mem = GetProcessMem.new
14
- worker.master_logger.info "performed #{job.name}, memory: #{mem.mb.to_i} MB"
15
- if worker.max_memory && mem.mb > worker.max_memory
16
- worker.master_logger.info "shutting down worker #{Process.pid} because it consumes large memory..."
17
- worker.stop
18
- end
19
- end
20
- end
21
- end
22
- end
23
- end
24
- end
@@ -1,31 +0,0 @@
1
- module Delayed
2
- class Master
3
- module Plugins
4
- class SignalHandler < Delayed::Plugin
5
- callbacks do |lifecycle|
6
- lifecycle.before(:execute) do |worker|
7
- worker.instance_eval do
8
- trap(:USR1) do
9
- Thread.new do
10
- master_logger.info "reopening files..."
11
- Delayed::Master::Util::FileReopener.reopen
12
- master_logger.info "reopened"
13
- end
14
- end
15
- trap(:USR2) do
16
- Thread.new do
17
- $0 = "#{$0} [OLD]"
18
- master_logger.info "shutting down worker #{Process.pid}..."
19
- stop
20
- end
21
- end
22
- end
23
- end
24
- lifecycle.after(:execute) do |worker|
25
- worker.master_logger.info "shut down worker #{Process.pid}"
26
- end
27
- end
28
- end
29
- end
30
- end
31
- end
@@ -1,17 +0,0 @@
1
- module Delayed
2
- class Master
3
- module Plugins
4
- class StatusNotifier < Delayed::Plugin
5
- callbacks do |lifecycle|
6
- lifecycle.around(:perform) do |worker, job, &block|
7
- title = $0
8
- $0 = "#{title} [BUSY]"
9
- ret = block.call
10
- $0 = title
11
- ret
12
- end
13
- end
14
- end
15
- end
16
- end
17
- end
@@ -1,18 +0,0 @@
1
- module Delayed
2
- class Master
3
- class Util
4
- class FileReopener
5
- class << self
6
- def reopen
7
- ObjectSpace.each_object(File) do |file|
8
- next if file.closed? || !file.sync
9
- file.reopen file.path, 'a+'
10
- file.sync = true
11
- file.flush
12
- end
13
- end
14
- end
15
- end
16
- end
17
- end
18
- end
@@ -1,5 +0,0 @@
1
- module Delayed
2
- class Master
3
- VERSION = "2.0.3"
4
- end
5
- end
@@ -1,19 +0,0 @@
1
- module Delayed
2
- class Worker
3
- attr_accessor :master_logger, :max_memory
4
- end
5
- end
6
-
7
- require_relative 'plugins/memory_checker'
8
- require_relative 'plugins/signal_handler'
9
- require_relative 'plugins/status_notifier'
10
-
11
- [
12
- Delayed::Master::Plugins::MemoryChecker,
13
- Delayed::Master::Plugins::SignalHandler,
14
- Delayed::Master::Plugins::StatusNotifier
15
- ].each do |plugin|
16
- unless Delayed::Worker.plugins.include?(plugin)
17
- Delayed::Worker.plugins << plugin
18
- end
19
- end