delayed_job_master 2.0.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +31 -34
  3. data/.gitignore +0 -2
  4. data/CHANGELOG.md +13 -0
  5. data/README.md +46 -33
  6. data/delayed_job_master.gemspec +8 -5
  7. data/lib/delayed/master/callbacks.rb +37 -0
  8. data/lib/delayed/master/command.rb +28 -5
  9. data/lib/delayed/master/config.rb +51 -58
  10. data/lib/delayed/master/core.rb +146 -0
  11. data/lib/delayed/master/database.rb +72 -0
  12. data/lib/delayed/master/file_reopener.rb +18 -0
  13. data/lib/delayed/master/forker.rb +30 -19
  14. data/lib/delayed/master/job_checker.rb +91 -47
  15. data/lib/delayed/master/job_finder.rb +31 -0
  16. data/lib/delayed/master/job_listener.rb +31 -0
  17. data/lib/delayed/master/monitoring.rb +37 -36
  18. data/lib/delayed/master/postgresql/job_listener.rb +73 -0
  19. data/lib/delayed/master/postgresql/job_notifier.rb +45 -0
  20. data/lib/delayed/master/safe_array.rb +30 -0
  21. data/lib/delayed/master/signaler.rb +17 -7
  22. data/lib/delayed/master/sleep.rb +18 -0
  23. data/lib/delayed/master/worker/backend/active_record.rb +41 -0
  24. data/lib/delayed/master/worker/extension.rb +14 -0
  25. data/lib/delayed/master/worker/lifecycle.rb +10 -0
  26. data/lib/delayed/master/worker/plugins/all.rb +17 -0
  27. data/lib/delayed/master/worker/plugins/executor_wrapper.rb +23 -0
  28. data/lib/delayed/master/worker/plugins/memory_checker.rb +28 -0
  29. data/lib/delayed/master/worker/plugins/signal_handler.rb +35 -0
  30. data/lib/delayed/master/worker/plugins/status_notifier.rb +21 -0
  31. data/lib/delayed/master/worker/thread_pool.rb +64 -0
  32. data/lib/delayed/master/worker/thread_worker.rb +65 -0
  33. data/lib/delayed/master/worker.rb +8 -7
  34. data/lib/delayed/master/worker_setting.rb +72 -0
  35. data/lib/delayed/master.rb +7 -100
  36. data/lib/delayed_job_master/railtie.rb +16 -0
  37. data/lib/delayed_job_master/version.rb +5 -0
  38. data/lib/delayed_job_master.rb +15 -1
  39. data/lib/generators/delayed_job_master/templates/config.rb +14 -14
  40. data/lib/generators/delayed_job_master/templates/script +1 -1
  41. metadata +71 -17
  42. data/gemfiles/rails50.gemfile +0 -7
  43. data/gemfiles/rails51.gemfile +0 -7
  44. data/gemfiles/rails52.gemfile +0 -7
  45. data/lib/delayed/master/database_detector.rb +0 -37
  46. data/lib/delayed/master/job_counter.rb +0 -26
  47. data/lib/delayed/master/plugins/memory_checker.rb +0 -24
  48. data/lib/delayed/master/plugins/signal_handler.rb +0 -31
  49. data/lib/delayed/master/plugins/status_notifier.rb +0 -17
  50. data/lib/delayed/master/util/file_reopener.rb +0 -18
  51. data/lib/delayed/master/version.rb +0 -5
  52. data/lib/delayed/master/worker_extension.rb +0 -19
@@ -0,0 +1,146 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fileutils'
4
+ require 'logger'
5
+ require_relative 'safe_array'
6
+ require_relative 'command'
7
+ require_relative 'worker'
8
+ require_relative 'database'
9
+ require_relative 'callbacks'
10
+ require_relative 'monitoring'
11
+ require_relative 'job_checker'
12
+ require_relative 'job_listener'
13
+ require_relative 'signaler'
14
+ require_relative 'file_reopener'
15
+
16
+ module Delayed
17
+ module Master
18
+ class Core
19
+ attr_reader :config, :logger, :databases, :callbacks, :workers
20
+ attr_reader :monitoring, :job_checker, :job_listener
21
+
22
+ def initialize(argv)
23
+ @config = Command.new(argv).config
24
+ @logger = setup_logger(@config.log_file, @config.log_level)
25
+ @workers = SafeArray.new
26
+
27
+ @databases = Database.all(@config.databases)
28
+ @callbacks = Callbacks.new(@config)
29
+ @monitoring = Monitoring.new(self)
30
+ @job_checker = JobChecker.new(self)
31
+ @job_listener = JobListener.klass.new(self)
32
+ @signaler = Signaler.new(self)
33
+ end
34
+
35
+ def run
36
+ print_config
37
+ daemonize if @config.daemon
38
+
39
+ @logger.info { "started master #{Process.pid}".tap { |msg| puts msg } }
40
+
41
+ handle_pid_file do
42
+ @signaler.register
43
+ @prepared = true
44
+
45
+ start
46
+ wait
47
+ shutdown
48
+ end
49
+
50
+ @logger.info { "shut down master" }
51
+ end
52
+
53
+ def start
54
+ @job_checker.start
55
+ @job_listener.start
56
+ @monitoring.start
57
+ end
58
+
59
+ def wait
60
+ @job_checker.wait
61
+ @job_listener.wait
62
+ @monitoring.wait
63
+ end
64
+
65
+ def shutdown
66
+ @job_checker.shutdown
67
+ @job_listener.shutdown
68
+ @monitoring.shutdown
69
+ end
70
+
71
+ def prepared?
72
+ @prepared
73
+ end
74
+
75
+ def quit
76
+ @signaler.dispatch(:KILL)
77
+ shutdown
78
+ @workers.clear
79
+ @stop = true
80
+ end
81
+
82
+ def stop
83
+ @signaler.dispatch(:TERM)
84
+ shutdown
85
+ @workers.clear
86
+ @stop = true
87
+ end
88
+
89
+ def graceful_stop
90
+ @signaler.dispatch(:TERM)
91
+ @stop = true
92
+ end
93
+
94
+ def stop?
95
+ @stop == true
96
+ end
97
+
98
+ def reopen_files
99
+ @signaler.dispatch(:USR1)
100
+ @logger.info { "reopening files..." }
101
+ FileReopener.reopen
102
+ @logger.info { "reopened" }
103
+ end
104
+
105
+ def restart
106
+ @signaler.dispatch(:USR2)
107
+ @logger.info { "restarting master..." }
108
+ exec(*([$0] + ARGV))
109
+ end
110
+
111
+ private
112
+
113
+ def setup_logger(log_file, log_level)
114
+ FileUtils.mkdir_p(File.dirname(log_file)) if log_file.is_a?(String)
115
+ logger = Logger.new(log_file)
116
+ logger.level = log_level
117
+ logger
118
+ end
119
+
120
+ def print_config
121
+ @config.abstract_texts.each do |text|
122
+ @logger.info { text }
123
+ end
124
+ end
125
+
126
+ def daemonize
127
+ Process.daemon(true)
128
+ end
129
+
130
+ def handle_pid_file
131
+ create_pid_file
132
+ yield
133
+ remove_pid_file
134
+ end
135
+
136
+ def create_pid_file
137
+ FileUtils.mkdir_p(File.dirname(@config.pid_file))
138
+ File.write(@config.pid_file, Process.pid)
139
+ end
140
+
141
+ def remove_pid_file
142
+ File.delete(@config.pid_file) if File.exist?(@config.pid_file)
143
+ end
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class Database
6
+ class_attribute :model_cache
7
+ self.model_cache = {}
8
+
9
+ attr_accessor :spec_name
10
+
11
+ def initialize(spec_name)
12
+ @spec_name = spec_name
13
+ end
14
+
15
+ def model
16
+ cache_model do
17
+ define_model
18
+ end
19
+ end
20
+
21
+ def with_connection
22
+ model.connection_pool.with_connection do |connection|
23
+ yield connection
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ def cache_model
30
+ self.class.model_cache[@spec_name] ||= yield
31
+ end
32
+
33
+ def define_model
34
+ model = Class.new(Delayed::Job)
35
+ model_name = "DelayedJob#{@spec_name.capitalize}"
36
+ unless Delayed::Master.const_defined?(model_name)
37
+ Delayed::Master.const_set(model_name, model)
38
+ Delayed::Master.const_get(model_name).establish_connection(@spec_name)
39
+ end
40
+ Delayed::Master.const_get(model_name)
41
+ end
42
+
43
+ class << self
44
+ def all(spec_names = nil)
45
+ spec_names = spec_names.presence || spec_names_with_delayed_job_table
46
+ spec_names.map { |spec_name| new(spec_name) }
47
+ end
48
+
49
+ private
50
+
51
+ def spec_names_with_delayed_job_table
52
+ @spec_names_with_delayed_job_table ||= spec_names_without_replica.select do |spec_name|
53
+ exist_delayed_job_table?(spec_name)
54
+ end
55
+ end
56
+
57
+ def spec_names_without_replica
58
+ configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env)
59
+ configs.reject(&:replica?).map do |c|
60
+ c.respond_to?(:name) ? c.name.to_sym : c.spec_name.to_sym
61
+ end
62
+ end
63
+
64
+ def exist_delayed_job_table?(spec_name)
65
+ new(spec_name).with_connection do |connection|
66
+ connection.tables.include?('delayed_jobs')
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class FileReopener
6
+ class << self
7
+ def reopen
8
+ ObjectSpace.each_object(File) do |file|
9
+ next if file.closed? || !file.sync
10
+ file.reopen file.path, 'a+'
11
+ file.sync = true
12
+ file.flush
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -1,42 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'file_reopener'
4
+
1
5
  module Delayed
2
- class Master
6
+ module Master
3
7
  class Forker
4
8
  def initialize(master)
5
9
  @master = master
6
10
  @config = master.config
11
+ @callbacks = master.callbacks
7
12
  end
8
13
 
9
- def new_worker(worker)
10
- @master.logger.info "forking #{worker.name}..."
11
- fork_worker(worker)
12
- @master.logger.info "forked #{worker.name} with pid #{worker.pid}"
13
-
14
- @master.workers << worker
14
+ def call(worker)
15
+ around_fork(worker) do
16
+ @callbacks.run(:before_fork, @master, worker)
17
+ worker.pid = fork do
18
+ @callbacks.run(:after_fork, @master, worker)
19
+ after_fork_at_child(worker)
20
+ worker.pid = Process.pid
21
+ worker.instance = create_instance(worker)
22
+ $0 = worker.process_title
23
+ worker.instance.start
24
+ end
25
+ end
15
26
  end
16
27
 
17
28
  private
18
29
 
19
- def fork_worker(worker)
20
- @config.run_callback(:before_fork, @master, worker)
21
- worker.pid = fork do
22
- worker.pid = Process.pid
23
- worker.instance = create_instance(worker)
24
- @config.run_callback(:after_fork, @master, worker)
25
- $0 = worker.process_title
26
- worker.instance.start
27
- end
30
+ def around_fork(worker)
31
+ @master.logger.info { "forking #{worker.name}..." }
32
+ yield
33
+ @master.logger.info { "forked #{worker.name} with pid #{worker.pid}" }
34
+ end
35
+
36
+ def after_fork_at_child(worker)
37
+ FileReopener.reopen
28
38
  end
29
39
 
30
40
  def create_instance(worker)
31
- require_relative 'worker_extension'
41
+ require_relative 'worker/extension'
32
42
 
33
- instance = Delayed::Worker.new(worker.setting.data)
43
+ instance = Delayed::Worker.new
34
44
  [:max_run_time, :max_attempts, :destroy_failed_jobs].each do |key|
35
45
  if (value = worker.setting.send(key))
36
46
  Delayed::Worker.send("#{key}=", value)
37
47
  end
38
48
  end
39
- [:max_memory].each do |key|
49
+ [:min_priority, :max_priority, :sleep_delay, :read_ahead, :exit_on_complete, :queues,
50
+ :max_threads, :max_memory].each do |key|
40
51
  if (value = worker.setting.send(key))
41
52
  instance.send("#{key}=", value)
42
53
  end
@@ -1,83 +1,127 @@
1
- require_relative 'job_counter'
2
- require_relative 'database_detector'
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'database'
4
+ require_relative 'forker'
5
+ require_relative 'job_finder'
6
+ require_relative 'sleep'
3
7
 
4
8
  module Delayed
5
- class Master
9
+ module Master
6
10
  class JobChecker
11
+ include Sleep
12
+
7
13
  def initialize(master)
8
14
  @master = master
9
15
  @config = master.config
10
- @spec_names = target_spec_names
16
+ @databases = master.databases
17
+ @callbacks = master.callbacks
18
+ @queues = @databases.map { |database| [database, Queue.new] }.to_h
19
+ @threads = []
20
+ end
11
21
 
12
- define_models
13
- extend_after_fork_callback
22
+ def start
23
+ @threads << start_scheduler_thread
24
+ @threads += @databases.map do |database|
25
+ start_checker_thread(database)
26
+ end
14
27
  end
15
28
 
16
- def check
17
- workers = []
18
- mon = Monitor.new
29
+ def start_scheduler_thread
30
+ Thread.new do
31
+ loop_with_sleep @config.polling_interval do |i|
32
+ if @master.stop?
33
+ stop
34
+ break
35
+ elsif i == 0
36
+ schedule(@databases)
37
+ end
38
+ end
39
+ end
40
+ end
19
41
 
20
- threads = @spec_names.map do |spec_name|
21
- Thread.new(spec_name) do |spec_name|
22
- find_jobs_in_db(spec_name) do |setting|
23
- mon.synchronize do
24
- workers << Worker.new(index: @master.workers.size + workers.size, database: spec_name, setting: setting)
42
+ def start_checker_thread(database)
43
+ Thread.new(database) do |database|
44
+ loop do
45
+ if @queues[database].pop == :stop
46
+ break
47
+ else
48
+ @callbacks.call(:polling, @master, database) do
49
+ check(database)
25
50
  end
26
51
  end
27
52
  end
28
53
  end
29
-
30
- threads.each(&:join)
31
-
32
- workers
33
54
  end
34
55
 
35
- private
56
+ def stop
57
+ @databases.each do |database|
58
+ queue = @queues[database]
59
+ queue.clear
60
+ queue.push(:stop)
61
+ end
62
+ end
36
63
 
37
- def define_models
38
- @spec_names.each do |spec_name|
39
- klass = Class.new(Delayed::Job)
40
- klass_name = "DelayedJob#{spec_name.capitalize}"
41
- unless Delayed::Master.const_defined?(klass_name)
42
- Delayed::Master.const_set(klass_name, klass)
43
- Delayed::Master.const_get(klass_name).establish_connection(spec_name)
44
- end
64
+ def schedule(databases)
65
+ Array(databases).each do |database|
66
+ queue = @queues[database]
67
+ queue.push(database) if queue.size == 0
45
68
  end
46
69
  end
47
70
 
48
- def model_for(spec_name)
49
- Delayed::Master.const_get("DelayedJob#{spec_name.capitalize}")
71
+ def wait
72
+ @threads.each(&:join)
50
73
  end
51
74
 
52
- def extend_after_fork_callback
53
- prc = @config.after_fork
54
- @config.after_fork do |master, worker|
55
- prc.call(master, worker)
56
- ActiveRecord::Base.establish_connection(worker.database) if worker.database
75
+ def shutdown
76
+ @threads.each(&:kill)
77
+ end
78
+
79
+ private
80
+
81
+ def check(database)
82
+ free_settings = detect_free_settings(database)
83
+ return if free_settings.blank?
84
+
85
+ @master.logger.debug { "checking jobs @#{database.spec_name}..." }
86
+ settings = check_jobs(database, free_settings)
87
+ if settings.present?
88
+ @master.logger.info { "found jobs for #{settings.uniq.map(&:worker_info).join(', ')}" }
89
+ fork_workers(database, settings)
57
90
  end
91
+ rescue => e
92
+ @master.logger.warn { "#{e.class}: #{e.message}" }
93
+ @master.logger.debug { e.backtrace.join("\n") }
58
94
  end
59
95
 
60
- def target_spec_names
61
- if @config.databases.nil? || @config.databases.empty?
62
- DatabaseDetector.new.call
63
- else
64
- @config.databases
96
+ def detect_free_settings(database)
97
+ @config.worker_settings.each_with_object([]) do |setting, array|
98
+ used_count = @master.workers.count { |worker| worker.setting.queues == setting.queues }
99
+ free_count = setting.max_processes - used_count
100
+ array << [setting, free_count] if free_count > 0
65
101
  end
66
102
  end
67
103
 
68
- def find_jobs_in_db(spec_name)
69
- counter = JobCounter.new(model_for(spec_name))
104
+ def check_jobs(database, settings)
105
+ finder = JobFinder.new(database.model)
70
106
 
71
- @config.worker_settings.each do |setting|
72
- count = @master.workers.count { |worker| worker.setting.queues == setting.queues }
73
- slot = setting.count - count
74
- if slot > 0 && (job_count = counter.count(setting)) > 0
75
- [slot, job_count].min.times do
76
- yield setting
107
+ settings.each_with_object([]) do |(setting, free_count), array|
108
+ job_ids = finder.call(setting, free_count)
109
+ if job_ids.size > 0
110
+ [free_count, job_ids.size].min.times do
111
+ array << setting
77
112
  end
78
113
  end
79
114
  end
80
115
  end
116
+
117
+ def fork_workers(database, settings)
118
+ settings.each do |setting|
119
+ worker = Worker.new(database: database, setting: setting)
120
+ Forker.new(@master).call(worker)
121
+ @master.workers << worker
122
+ @master.monitoring.schedule(worker)
123
+ end
124
+ end
81
125
  end
82
126
  end
83
127
  end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ # JobFinder runs SQL query which is almost same as delayed_job_active_record.
4
+ # See https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/delayed/backend/active_record.rb
5
+ module Delayed
6
+ module Master
7
+ class JobFinder
8
+ def initialize(model)
9
+ @model = model
10
+ end
11
+
12
+ def call(setting, limit)
13
+ scope(setting).limit(limit).pluck(:id)
14
+ end
15
+
16
+ def count(setting)
17
+ scope(setting).count
18
+ end
19
+
20
+ private
21
+
22
+ def scope(setting)
23
+ @model.ready_to_run(nil, setting.max_run_time || Delayed::Worker::DEFAULT_MAX_RUN_TIME).tap do |jobs|
24
+ jobs.where!("priority >= ?", setting.min_priority) if setting.min_priority
25
+ jobs.where!("priority <= ?", setting.max_priority) if setting.max_priority
26
+ jobs.where!(queue: setting.queues) if setting.queues.present?
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class JobListener
6
+ def initialize(master)
7
+ end
8
+
9
+ def start
10
+ end
11
+
12
+ def wait
13
+ end
14
+
15
+ def shutdown
16
+ end
17
+
18
+ class << self
19
+ def klass
20
+ case DelayedJobMaster.config.listener
21
+ when :postgresql
22
+ require_relative 'postgresql/job_listener'
23
+ Postgresql::JobListener
24
+ else
25
+ self
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -1,58 +1,59 @@
1
- require_relative 'forker'
2
- require_relative 'job_checker' if defined?(Delayed::Backend::ActiveRecord)
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'safe_array'
4
+ require_relative 'sleep'
3
5
 
4
6
  module Delayed
5
- class Master
7
+ module Master
6
8
  class Monitoring
9
+ include Sleep
10
+
7
11
  def initialize(master)
8
12
  @master = master
9
13
  @config = master.config
10
- @forker = Forker.new(master)
11
- @job_checker = JobChecker.new(master)
14
+ @callbacks = master.callbacks
15
+ @threads = SafeArray.new
12
16
  end
13
17
 
14
- def monitor_while(&block)
15
- loop do
16
- break if block.call
17
- monitor do
18
- check_terminated
19
- check_queued_jobs
18
+ def start
19
+ @threads << Thread.new do
20
+ loop_with_sleep @config.monitor_interval do |i|
21
+ if @master.stop?
22
+ break
23
+ elsif i == 0
24
+ @callbacks.call(:monitor, @master) {}
25
+ end
20
26
  end
21
- sleep @config.monitor_wait.to_i
22
27
  end
23
28
  end
24
29
 
25
- private
26
-
27
- def monitor
28
- @config.run_callback(:before_monitor, @master)
29
- yield
30
- @config.run_callback(:after_monitor, @master)
31
- rescue Exception => e
32
- @master.logger.warn "#{e.class}: #{e.message} at #{__FILE__}: #{__LINE__}"
30
+ def schedule(worker)
31
+ @threads << Thread.new do
32
+ wait_pid(worker)
33
+ @threads.delete(Thread.current)
34
+ end
33
35
  end
34
36
 
35
- def check_terminated
36
- if (pid = terminated_pid)
37
- @master.logger.debug "found terminated pid: #{pid}"
38
- @master.workers.reject! { |worker| worker.pid == pid }
39
- end
37
+ def wait
38
+ @threads.each(&:join)
40
39
  end
41
40
 
42
- def terminated_pid
43
- Process.waitpid(-1, Process::WNOHANG)
44
- rescue Errno::ECHILD
45
- nil
41
+ def shutdown
42
+ @threads.each(&:kill)
46
43
  end
47
44
 
48
- def check_queued_jobs
49
- @master.logger.debug "checking jobs..."
45
+ private
50
46
 
51
- new_workers = @job_checker.check
52
- new_workers.each do |worker|
53
- @master.logger.info "found jobs for #{worker.info}"
54
- @forker.new_worker(worker)
55
- end
47
+ def wait_pid(worker)
48
+ Process.waitpid(worker.pid)
49
+ @master.logger.debug { "found terminated pid: #{worker.pid}" }
50
+ @master.workers.delete(worker)
51
+ rescue Errno::ECHILD
52
+ @master.logger.warn { "failed to waitpid: #{worker.pid}" }
53
+ @master.workers.delete(worker)
54
+ rescue => e
55
+ @master.logger.warn { "#{e.class}: #{e.message}" }
56
+ @master.logger.debug { e.backtrace.join("\n") }
56
57
  end
57
58
  end
58
59
  end