delayed_job_master 2.0.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +31 -34
  3. data/.gitignore +0 -2
  4. data/CHANGELOG.md +13 -0
  5. data/README.md +46 -33
  6. data/delayed_job_master.gemspec +8 -5
  7. data/lib/delayed/master/callbacks.rb +37 -0
  8. data/lib/delayed/master/command.rb +28 -5
  9. data/lib/delayed/master/config.rb +51 -58
  10. data/lib/delayed/master/core.rb +146 -0
  11. data/lib/delayed/master/database.rb +72 -0
  12. data/lib/delayed/master/file_reopener.rb +18 -0
  13. data/lib/delayed/master/forker.rb +30 -19
  14. data/lib/delayed/master/job_checker.rb +91 -47
  15. data/lib/delayed/master/job_finder.rb +31 -0
  16. data/lib/delayed/master/job_listener.rb +31 -0
  17. data/lib/delayed/master/monitoring.rb +37 -36
  18. data/lib/delayed/master/postgresql/job_listener.rb +73 -0
  19. data/lib/delayed/master/postgresql/job_notifier.rb +45 -0
  20. data/lib/delayed/master/safe_array.rb +30 -0
  21. data/lib/delayed/master/signaler.rb +17 -7
  22. data/lib/delayed/master/sleep.rb +18 -0
  23. data/lib/delayed/master/worker/backend/active_record.rb +41 -0
  24. data/lib/delayed/master/worker/extension.rb +14 -0
  25. data/lib/delayed/master/worker/lifecycle.rb +10 -0
  26. data/lib/delayed/master/worker/plugins/all.rb +17 -0
  27. data/lib/delayed/master/worker/plugins/executor_wrapper.rb +23 -0
  28. data/lib/delayed/master/worker/plugins/memory_checker.rb +28 -0
  29. data/lib/delayed/master/worker/plugins/signal_handler.rb +35 -0
  30. data/lib/delayed/master/worker/plugins/status_notifier.rb +21 -0
  31. data/lib/delayed/master/worker/thread_pool.rb +64 -0
  32. data/lib/delayed/master/worker/thread_worker.rb +65 -0
  33. data/lib/delayed/master/worker.rb +8 -7
  34. data/lib/delayed/master/worker_setting.rb +72 -0
  35. data/lib/delayed/master.rb +7 -100
  36. data/lib/delayed_job_master/railtie.rb +16 -0
  37. data/lib/delayed_job_master/version.rb +5 -0
  38. data/lib/delayed_job_master.rb +15 -1
  39. data/lib/generators/delayed_job_master/templates/config.rb +14 -14
  40. data/lib/generators/delayed_job_master/templates/script +1 -1
  41. metadata +71 -17
  42. data/gemfiles/rails50.gemfile +0 -7
  43. data/gemfiles/rails51.gemfile +0 -7
  44. data/gemfiles/rails52.gemfile +0 -7
  45. data/lib/delayed/master/database_detector.rb +0 -37
  46. data/lib/delayed/master/job_counter.rb +0 -26
  47. data/lib/delayed/master/plugins/memory_checker.rb +0 -24
  48. data/lib/delayed/master/plugins/signal_handler.rb +0 -31
  49. data/lib/delayed/master/plugins/status_notifier.rb +0 -17
  50. data/lib/delayed/master/util/file_reopener.rb +0 -18
  51. data/lib/delayed/master/version.rb +0 -5
  52. data/lib/delayed/master/worker_extension.rb +0 -19
@@ -0,0 +1,73 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ module Postgresql
6
+ class JobListener < Delayed::Master::JobListener
7
+ def initialize(master)
8
+ @master = master
9
+ @config = master.config
10
+ @databases = master.databases
11
+ @threads = []
12
+ end
13
+
14
+ def start
15
+ @threads = @databases.map do |database|
16
+ Thread.new(database) do |database|
17
+ loop do
18
+ if @master.stop?
19
+ break
20
+ else
21
+ listen(database)
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
27
+
28
+ def wait
29
+ @threads.each(&:join)
30
+ end
31
+
32
+ def shutdown
33
+ @threads.each(&:kill)
34
+ end
35
+
36
+ private
37
+
38
+ def listen(database)
39
+ database.with_connection do |connection|
40
+ listen_connection(database, connection) do
41
+ loop do
42
+ if @master.stop?
43
+ break
44
+ else
45
+ wait_for_notify(database, connection)
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+
52
+ def listen_connection(database, connection)
53
+ @master.logger.info { "listening @#{database.spec_name}..." }
54
+ connection.execute("LISTEN delayed_job_master")
55
+ yield
56
+ rescue => e
57
+ @master.logger.warn { "#{e.class}: #{e.message}" }
58
+ @master.logger.debug { e.backtrace.join("\n") }
59
+ ensure
60
+ @master.logger.info { "unlisten @#{database.spec_name}" }
61
+ connection.execute("UNLISTEN delayed_job_master")
62
+ end
63
+
64
+ def wait_for_notify(database, connection)
65
+ connection.raw_connection.wait_for_notify(1) do |_event, _pid, _payload|
66
+ @master.logger.info { "received notification @#{database.spec_name}" }
67
+ @master.job_checker.schedule(database)
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ module Postgresql
6
+ class << self
7
+ def notify(model)
8
+ model.connection.execute "NOTIFY delayed_job_master"
9
+ end
10
+ end
11
+
12
+ module JobNotifier
13
+ extend ActiveSupport::Concern
14
+
15
+ included do
16
+ after_create :notify_to_delayed_job_master
17
+ end
18
+
19
+ private
20
+
21
+ def notify_to_delayed_job_master
22
+ if run_at && run_at < Time.zone.now
23
+ Delayed::Master::Postgresql.notify(self.class)
24
+ end
25
+ end
26
+ end
27
+
28
+ module BulkJobNotifier
29
+ extend ActiveSupport::Concern
30
+
31
+ included do
32
+ after_enqueue :notify_to_delayed_job_master
33
+ end
34
+
35
+ private
36
+
37
+ def notify_to_delayed_job_master
38
+ if @jobs.any? { |job| job.run_at && job.run_at < Time.zone.now }
39
+ Delayed::Master::Postgresql.notify(@jobs.first.class)
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class SafeArray < Array
6
+ def initialize(*args)
7
+ @mon = Monitor.new
8
+ super
9
+ end
10
+
11
+ def <<(*args)
12
+ @mon.synchronize do
13
+ super
14
+ end
15
+ end
16
+
17
+ def delete(*args)
18
+ @mon.synchronize do
19
+ super
20
+ end
21
+ end
22
+
23
+ def clear
24
+ @mon.synchronize do
25
+ super
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -1,13 +1,23 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Delayed
2
- class Master
4
+ module Master
3
5
  class Signaler
6
+ SIGNAL_HANDLERS = [
7
+ [:TERM, :stop],
8
+ [:INT, :stop],
9
+ [:QUIT, :quit],
10
+ [:WINCH, :graceful_stop],
11
+ [:USR1, :reopen_files],
12
+ [:USR2, :restart]
13
+ ]
14
+
4
15
  def initialize(master)
5
16
  @master = master
6
17
  end
7
18
 
8
19
  def register
9
- signals = [[:TERM, :stop], [:INT, :stop], [:QUIT, :quit], [:USR1, :reopen_files], [:USR2, :restart]]
10
- signals.each do |signal, method|
20
+ SIGNAL_HANDLERS.each do |signal, method|
11
21
  register_signal(signal, method)
12
22
  end
13
23
  end
@@ -22,9 +32,9 @@ module Delayed
22
32
  private
23
33
 
24
34
  def register_signal(signal, method)
25
- trap(signal) do
35
+ Signal.trap(signal) do
26
36
  Thread.new do
27
- @master.logger.info "received #{signal} signal"
37
+ @master.logger.info { "received #{signal} signal" }
28
38
  @master.public_send(method)
29
39
  end
30
40
  end
@@ -32,9 +42,9 @@ module Delayed
32
42
 
33
43
  def dispatch_to(signal, pid)
34
44
  Process.kill(signal, pid)
35
- @master.logger.info "sent #{signal} signal to worker #{pid}"
45
+ @master.logger.info { "sent #{signal} signal to worker #{pid}" }
36
46
  rescue
37
- @master.logger.error "failed to send #{signal} signal to worker #{pid}"
47
+ @master.logger.error { "failed to send #{signal} signal to worker #{pid}" }
38
48
  end
39
49
  end
40
50
  end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ module Sleep
6
+ def loop_with_sleep(sec)
7
+ count = [sec.to_i, 1].max
8
+ div = sec.to_f / count
9
+ loop do
10
+ count.times do |i|
11
+ yield i
12
+ sleep div
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Backend
5
+ module ActiveRecord
6
+ class Job < ::ActiveRecord::Base
7
+ # Remove locked_by from query because all jobs reserved by current process have same locked_by.
8
+ def self.ready_to_run(worker_name, max_run_time)
9
+ where(
10
+ "(run_at <= ? AND (locked_at IS NULL OR locked_at < ?)) AND failed_at IS NULL",
11
+ db_time_now,
12
+ db_time_now - max_run_time
13
+ )
14
+ end
15
+
16
+ # Patch for postgresql query.
17
+ def self.reserve_with_scope_using_optimized_postgres(ready_scope, worker, now)
18
+ quoted_name = connection.quote_table_name(table_name)
19
+ subquery = ready_scope.limit(1).lock(true).select("id").to_sql
20
+ sql = <<~SQL.squish
21
+ WITH job AS (#{subquery} SKIP LOCKED)
22
+ UPDATE #{quoted_name} AS jobs SET locked_at = ?, locked_by = ? FROM job
23
+ WHERE jobs.id = job.id RETURNING *
24
+ SQL
25
+ reserved = find_by_sql([sql, now, worker.name])
26
+ reserved[0]
27
+ end
28
+
29
+ # Patch for mysql query.
30
+ def self.reserve_with_scope_using_optimized_mysql(ready_scope, worker, now)
31
+ transaction do
32
+ ready_scope.limit(worker.read_ahead).select(:id).lock.detect do |job|
33
+ count = where(id: job.id).update_all(locked_at: now, locked_by: worker.name)
34
+ count == 1 && job.reload
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'lifecycle'
4
+ require_relative 'thread_pool'
5
+ require_relative 'thread_worker'
6
+ require_relative 'plugins/all'
7
+ require_relative 'backend/active_record' if defined?(Delayed::Backend::ActiveRecord)
8
+
9
+ module Delayed
10
+ class Worker
11
+ attr_accessor :master_logger
12
+ attr_accessor :max_threads, :max_memory
13
+ end
14
+ end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ events = Delayed::Lifecycle::EVENTS.dup.merge(
4
+ thread: [:worker],
5
+ scheduler_thread: [:worker],
6
+ worker_thread: [:worker, :job]
7
+ )
8
+
9
+ Delayed::Lifecycle.send(:remove_const, :EVENTS)
10
+ Delayed::Lifecycle.const_set(:EVENTS, events)
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'executor_wrapper'
4
+ require_relative 'memory_checker'
5
+ require_relative 'signal_handler'
6
+ require_relative 'status_notifier'
7
+
8
+ [
9
+ Delayed::Master::Worker::Plugins::ExecutorWrapper,
10
+ Delayed::Master::Worker::Plugins::MemoryChecker,
11
+ Delayed::Master::Worker::Plugins::SignalHandler,
12
+ Delayed::Master::Worker::Plugins::StatusNotifier
13
+ ].each do |plugin|
14
+ unless Delayed::Worker.plugins.include?(plugin)
15
+ Delayed::Worker.plugins << plugin
16
+ end
17
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class Worker
6
+ module Plugins
7
+ class ExecutorWrapper < Delayed::Plugin
8
+ callbacks do |lifecycle|
9
+ lifecycle.around(:thread) do |worker, &block|
10
+ if defined?(Rails)
11
+ Rails.application.executor.wrap do
12
+ block.call
13
+ end
14
+ else
15
+ block.call
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'get_process_mem'
4
+
5
+ module Delayed
6
+ module Master
7
+ class Worker
8
+ module Plugins
9
+ class MemoryChecker < Delayed::Plugin
10
+ callbacks do |lifecycle|
11
+ lifecycle.before(:perform) do |worker, job|
12
+ mem = GetProcessMem.new
13
+ worker.master_logger.info { "performing #{job.name}, memory: #{mem.mb.to_i} MB" }
14
+ end
15
+ lifecycle.after(:perform) do |worker, job|
16
+ mem = GetProcessMem.new
17
+ worker.master_logger.info { "performed #{job.name}, memory: #{mem.mb.to_i} MB" }
18
+ if worker.max_memory && mem.mb > worker.max_memory
19
+ worker.master_logger.info { "shutting down worker #{Process.pid} because it consumes large memory..." }
20
+ worker.stop
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class Worker
6
+ module Plugins
7
+ class SignalHandler < Delayed::Plugin
8
+ callbacks do |lifecycle|
9
+ lifecycle.before(:execute) do |worker|
10
+ worker.instance_eval do
11
+ Signal.trap(:USR1) do
12
+ Thread.new do
13
+ master_logger.info { "reopening files..." }
14
+ Delayed::Master::FileReopener.reopen
15
+ master_logger.info { "reopened" }
16
+ end
17
+ end
18
+ Signal.trap(:USR2) do
19
+ Thread.new do
20
+ $0 = "#{$0} [OLD]"
21
+ master_logger.info { "shutting down worker #{Process.pid}..." }
22
+ stop
23
+ end
24
+ end
25
+ end
26
+ end
27
+ lifecycle.after(:execute) do |worker|
28
+ worker.master_logger.info { "shut down worker #{Process.pid}" }
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class Worker
6
+ module Plugins
7
+ class StatusNotifier < Delayed::Plugin
8
+ callbacks do |lifecycle|
9
+ lifecycle.around(:execute) do |worker, job, &block|
10
+ title = $0
11
+ $0 = "#{title} [BUSY]"
12
+ ret = block.call
13
+ $0 = title
14
+ ret
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class Worker
6
+ class ThreadPool
7
+ def initialize(worker, size)
8
+ @worker = worker
9
+ @size = size
10
+ @queue = SizedQueue.new(@size)
11
+ @queue_delay = 0.5
12
+ end
13
+
14
+ def schedule
15
+ @scheduler = Thread.new do
16
+ Delayed::Worker.lifecycle.run_callbacks(:thread, @worker) do
17
+ loop do
18
+ while @queue.num_waiting == 0
19
+ sleep @queue_delay
20
+ end
21
+
22
+ if item = yield
23
+ @queue.push(item)
24
+ Thread.pass
25
+ else
26
+ @size.times { @queue.push(:exit) }
27
+ break
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+
34
+ def work
35
+ @threads = @size.times.map do
36
+ Thread.new do
37
+ Delayed::Worker.lifecycle.run_callbacks(:thread, @worker) do
38
+ loop do
39
+ item = @queue.pop
40
+ if item == :exit
41
+ break
42
+ else
43
+ yield item
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+
51
+ def wait
52
+ @scheduler.join
53
+ @threads.each(&:join)
54
+ end
55
+
56
+ def shutdown
57
+ @scheduler.kill
58
+ @threads.each(&:kill)
59
+ @queue.close
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Overrides Delayed::Worker to support multithread.
4
+ # See original code at https://github.com/collectiveidea/delayed_job/blob/master/lib/delayed/worker.rb
5
+ module Delayed
6
+ module Master
7
+ class Worker
8
+ module ThreadWorker
9
+ def work_off(num = 100)
10
+ if multithread?
11
+ work_off_for_multithread
12
+ else
13
+ super
14
+ end
15
+ end
16
+
17
+ def multithread?
18
+ @max_threads.to_i > 1
19
+ end
20
+
21
+ def work_off_for_multithread
22
+ success = 0
23
+ failure = 0
24
+
25
+ monitor = Monitor.new
26
+ thread_pool = ThreadPool.new(self, @max_threads)
27
+
28
+ thread_pool.schedule do
29
+ self.class.lifecycle.run_callbacks(:scheduler_thread, self) do
30
+ if stop?
31
+ next nil
32
+ else
33
+ next reserve_job
34
+ end
35
+ end
36
+ end
37
+
38
+ thread_pool.work do |job|
39
+ @master_logger.debug { "start worker thread #{Thread.current.object_id}" }
40
+ self.class.lifecycle.run_callbacks(:worker_thread, self, job) do
41
+ case run_one_job(job)
42
+ when true
43
+ monitor.synchronize { success += 1 }
44
+ when false
45
+ monitor.synchronize { failure += 1 }
46
+ end
47
+ end
48
+ @master_logger.debug { "stop worker thread #{Thread.current.object_id}" }
49
+ end
50
+
51
+ thread_pool.wait
52
+ thread_pool.shutdown
53
+
54
+ [success, failure]
55
+ end
56
+
57
+ def run_one_job(job)
58
+ self.class.lifecycle.run_callbacks(:perform, self, job) { run(job) }
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+
65
+ Delayed::Worker.prepend Delayed::Master::Worker::ThreadWorker
@@ -1,7 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Delayed
2
- class Master
4
+ module Master
3
5
  class Worker
4
- attr_accessor :index, :setting, :database
6
+ attr_accessor :setting, :database
5
7
  attr_accessor :pid, :instance
6
8
 
7
9
  def initialize(attrs = {})
@@ -15,14 +17,13 @@ module Delayed
15
17
  end
16
18
 
17
19
  def info
18
- str = name
19
- str << " @#{@database}" if @database
20
- str << " (#{@setting.queues.join(', ')})" if @setting.queues.respond_to?(:join)
21
- str
20
+ strs = [@setting.worker_info]
21
+ strs << "@#{@database.spec_name}" if @database
22
+ strs.join(' ')
22
23
  end
23
24
 
24
25
  def process_title
25
- "delayed_job.#{@index}: #{info}"
26
+ "delayed_job: #{info}"
26
27
  end
27
28
  end
28
29
  end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Delayed
4
+ module Master
5
+ class WorkerSetting
6
+ SIMPLE_CONFIGS = [:id, :max_processes, :max_threads, :max_memory,
7
+ :min_priority, :max_priority, :sleep_delay, :read_ahead, :exit_on_complete,
8
+ :max_attempts, :max_run_time, :destroy_failed_jobs]
9
+ ARRAY_CONFIGS = [:queues]
10
+
11
+ attr_accessor *SIMPLE_CONFIGS
12
+ attr_accessor *ARRAY_CONFIGS
13
+
14
+ def initialize(attrs = {})
15
+ @queues = []
16
+ @max_processes = 1
17
+ @max_threads = 1
18
+ @exit_on_complete = true
19
+ self.attributes = attrs
20
+ end
21
+
22
+ def attributes=(attrs = {})
23
+ attrs.each do |key, value|
24
+ send("#{key}=", value)
25
+ end
26
+ end
27
+
28
+ def worker_name
29
+ "worker[#{id}]"
30
+ end
31
+
32
+ def worker_info
33
+ strs = [worker_name]
34
+ strs << "(#{@queues.join(', ')})" if @queues.present?
35
+ strs.join(' ')
36
+ end
37
+
38
+ SIMPLE_CONFIGS.each do |key|
39
+ define_method(key) do |*args|
40
+ if args.size > 0
41
+ instance_variable_set("@#{key}", args[0])
42
+ else
43
+ instance_variable_get("@#{key}")
44
+ end
45
+ end
46
+ end
47
+
48
+ ARRAY_CONFIGS.each do |key|
49
+ define_method(key) do |*args|
50
+ if args.size > 0
51
+ instance_variable_set("@#{key}", Array(args[0]))
52
+ else
53
+ instance_variable_get("@#{key}")
54
+ end
55
+ end
56
+ end
57
+
58
+ def control(value = nil)
59
+ ActiveSupport::Deprecation.warn <<-TEXT.squish
60
+ deprecated 'control' setting was used. Remove it from your config file.
61
+ TEXT
62
+ end
63
+
64
+ def count(value = nil)
65
+ ActiveSupport::Deprecation.warn <<-TEXT.squish
66
+ deprecated 'count' setting was used. Use 'max_processes' instead.
67
+ TEXT
68
+ max_processes value
69
+ end
70
+ end
71
+ end
72
+ end