exekutor 0.1.0 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -3
- data/exe/exekutor +2 -2
- data/lib/active_job/queue_adapters/exekutor_adapter.rb +2 -1
- data/lib/exekutor/asynchronous.rb +143 -75
- data/lib/exekutor/cleanup.rb +27 -28
- data/lib/exekutor/configuration.rb +102 -48
- data/lib/exekutor/hook.rb +15 -11
- data/lib/exekutor/info/worker.rb +3 -3
- data/lib/exekutor/internal/base_record.rb +2 -1
- data/lib/exekutor/internal/callbacks.rb +55 -35
- data/lib/exekutor/internal/cli/app.rb +33 -23
- data/lib/exekutor/internal/cli/application_loader.rb +17 -6
- data/lib/exekutor/internal/cli/cleanup.rb +54 -40
- data/lib/exekutor/internal/cli/daemon.rb +9 -11
- data/lib/exekutor/internal/cli/default_option_value.rb +3 -1
- data/lib/exekutor/internal/cli/info.rb +117 -84
- data/lib/exekutor/internal/cli/manager.rb +234 -123
- data/lib/exekutor/internal/configuration_builder.rb +49 -30
- data/lib/exekutor/internal/database_connection.rb +6 -0
- data/lib/exekutor/internal/executable.rb +12 -7
- data/lib/exekutor/internal/executor.rb +50 -21
- data/lib/exekutor/internal/hooks.rb +11 -8
- data/lib/exekutor/internal/listener.rb +85 -43
- data/lib/exekutor/internal/logger.rb +29 -10
- data/lib/exekutor/internal/provider.rb +96 -77
- data/lib/exekutor/internal/reserver.rb +66 -19
- data/lib/exekutor/internal/status_server.rb +87 -54
- data/lib/exekutor/job.rb +1 -1
- data/lib/exekutor/job_error.rb +1 -1
- data/lib/exekutor/job_options.rb +22 -13
- data/lib/exekutor/plugins/appsignal.rb +7 -5
- data/lib/exekutor/plugins.rb +8 -4
- data/lib/exekutor/queue.rb +69 -30
- data/lib/exekutor/version.rb +1 -1
- data/lib/exekutor/worker.rb +89 -48
- data/lib/exekutor.rb +2 -2
- data/lib/generators/exekutor/configuration_generator.rb +11 -6
- data/lib/generators/exekutor/install_generator.rb +24 -15
- data/lib/generators/exekutor/templates/install/functions/exekutor_broadcast_job_enqueued.sql +10 -0
- data/lib/generators/exekutor/templates/install/functions/exekutor_requeue_orphaned_jobs.sql +11 -0
- data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +23 -22
- data/lib/generators/exekutor/templates/install/triggers/exekutor_broadcast_job_enqueued.sql +7 -0
- data/lib/generators/exekutor/templates/install/triggers/exekutor_requeue_orphaned_jobs.sql +5 -0
- data.tar.gz.sig +0 -0
- metadata +67 -23
- metadata.gz.sig +0 -0
- data/lib/generators/exekutor/templates/install/functions/job_notifier.sql +0 -7
- data/lib/generators/exekutor/templates/install/functions/requeue_orphaned_jobs.sql +0 -7
- data/lib/generators/exekutor/templates/install/triggers/notify_workers.sql +0 -6
- data/lib/generators/exekutor/templates/install/triggers/requeue_orphaned_jobs.sql +0 -5
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative "application_loader"
|
2
4
|
require "terminal-table"
|
3
5
|
|
@@ -19,96 +21,128 @@ module Exekutor
|
|
19
21
|
load_application(options[:environment], print_message: !quiet?)
|
20
22
|
|
21
23
|
ActiveSupport.on_load(:active_record, yield: true) do
|
22
|
-
|
23
|
-
|
24
|
+
clear_application_loading_message
|
25
|
+
print_time_zone_warning if different_time_zone? && !quiet?
|
24
26
|
|
25
27
|
hosts = Exekutor::Info::Worker.distinct.pluck(:hostname)
|
26
|
-
job_info =
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
28
|
+
job_info = pending_jobs_per_queue
|
29
|
+
|
30
|
+
print_workers(hosts, job_info.present?, options)
|
31
|
+
puts
|
32
|
+
print_jobs(job_info)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def pending_jobs_per_queue
|
39
|
+
Exekutor::Job.pending.order(:queue).group(:queue)
|
40
|
+
.pluck(:queue, Arel.sql("COUNT(*)"), Arel.sql("MIN(scheduled_at)"))
|
41
|
+
end
|
42
|
+
|
43
|
+
def print_jobs(job_info)
|
44
|
+
puts Rainbow("Jobs").bright.blue
|
45
|
+
if job_info.present?
|
46
|
+
puts create_job_info_table(job_info)
|
47
|
+
else
|
48
|
+
puts Rainbow("No pending jobs").green
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def create_job_info_table(job_info)
|
53
|
+
Terminal::Table.new(headings: ["Queue", "Pending jobs", "Next job scheduled at"]).tap do |table|
|
54
|
+
total_count = 0
|
55
|
+
job_info.each do |queue, count, min_scheduled_at|
|
56
|
+
table << [queue, { value: count, alignment: :right }, format_scheduled_at(min_scheduled_at)]
|
57
|
+
total_count += count
|
58
|
+
end
|
59
|
+
if job_info.many?
|
60
|
+
table.add_separator
|
61
|
+
table << ["Total", { value: total_count, alignment: :right, colspan: 2 }]
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def format_scheduled_at(min_scheduled_at)
|
67
|
+
if min_scheduled_at.nil?
|
68
|
+
"N/A"
|
69
|
+
elsif min_scheduled_at < 30.minutes.ago
|
70
|
+
Rainbow(min_scheduled_at.strftime("%D %R")).red
|
71
|
+
elsif min_scheduled_at < 1.minute.ago
|
72
|
+
Rainbow(min_scheduled_at.strftime("%D %R")).yellow
|
73
|
+
else
|
74
|
+
min_scheduled_at.strftime("%D %R")
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def print_workers(hosts, has_pending_jobs, options)
|
79
|
+
puts Rainbow("Workers").bright.blue
|
80
|
+
if hosts.present?
|
81
|
+
total_workers = 0
|
82
|
+
hosts.each do |host|
|
83
|
+
total_workers += print_host_info(host, options.merge(many_hosts: hosts.many?))
|
77
84
|
end
|
78
85
|
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
total_count = 0
|
85
|
-
job_info.each do |queue, count, min_scheduled_at|
|
86
|
-
table << [
|
87
|
-
queue, count,
|
88
|
-
if min_scheduled_at.nil?
|
89
|
-
"N/A"
|
90
|
-
elsif min_scheduled_at < 30.minutes.ago
|
91
|
-
Rainbow(min_scheduled_at.strftime("%D %R")).red
|
92
|
-
elsif min_scheduled_at < 1.minute.ago
|
93
|
-
Rainbow(min_scheduled_at.strftime("%D %R")).yellow
|
94
|
-
else
|
95
|
-
min_scheduled_at.strftime("%D %R")
|
96
|
-
end
|
97
|
-
]
|
98
|
-
total_count += count
|
99
|
-
end
|
100
|
-
if job_info.many?
|
101
|
-
table.add_separator
|
102
|
-
table.add_row ["Total", { value: total_count, alignment: :right, colspan: 2 }]
|
103
|
-
end
|
104
|
-
puts table
|
105
|
-
else
|
106
|
-
puts Rainbow("No pending jobs").green
|
86
|
+
if hosts.many?
|
87
|
+
puts Terminal::Table.new rows: [
|
88
|
+
["Total hosts", hosts.size],
|
89
|
+
["Total workers", total_workers]
|
90
|
+
]
|
107
91
|
end
|
92
|
+
else
|
93
|
+
message = Rainbow("There are no active workers")
|
94
|
+
message = message.red if has_pending_jobs
|
95
|
+
puts message
|
108
96
|
end
|
109
97
|
end
|
110
98
|
|
111
|
-
|
99
|
+
def print_host_info(host, options)
|
100
|
+
many_hosts = options[:many_hosts]
|
101
|
+
table = Terminal::Table.new headings: ["id", "Status", "Last heartbeat"]
|
102
|
+
table.title = host if many_hosts
|
103
|
+
worker_count = 0
|
104
|
+
Exekutor::Info::Worker.where(hostname: host).find_each do |worker|
|
105
|
+
worker_count += 1
|
106
|
+
table << worker_info_row(worker)
|
107
|
+
end
|
108
|
+
table.add_separator
|
109
|
+
table.add_row [(many_hosts ? "Subtotal" : "Total"),
|
110
|
+
{ value: worker_count, alignment: :right, colspan: 2 }]
|
111
|
+
puts table
|
112
|
+
worker_count
|
113
|
+
end
|
114
|
+
|
115
|
+
def worker_info_row(worker)
|
116
|
+
[
|
117
|
+
worker.id.split("-").first << "…",
|
118
|
+
worker.status,
|
119
|
+
worker_heartbeat_column(worker)
|
120
|
+
]
|
121
|
+
end
|
122
|
+
|
123
|
+
def worker_heartbeat_column(worker)
|
124
|
+
last_heartbeat_at = worker.last_heartbeat_at
|
125
|
+
if last_heartbeat_at
|
126
|
+
colorize_heartbeat(last_heartbeat_at)
|
127
|
+
elsif !worker.running?
|
128
|
+
"N/A"
|
129
|
+
elsif worker.started_at < 10.minutes.ago
|
130
|
+
Rainbow("None").red
|
131
|
+
else
|
132
|
+
"None"
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
def colorize_heartbeat(timestamp)
|
137
|
+
case Time.now - timestamp
|
138
|
+
when (10.minutes)..nil
|
139
|
+
Rainbow(timestamp.strftime("%D %R")).red
|
140
|
+
when (2.minutes)..(10.minutes)
|
141
|
+
Rainbow(timestamp.strftime("%R")).yellow
|
142
|
+
else
|
143
|
+
timestamp.strftime "%R"
|
144
|
+
end
|
145
|
+
end
|
112
146
|
|
113
147
|
# @return [Boolean] Whether quiet mode is enabled. Overrides verbose mode.
|
114
148
|
def quiet?
|
@@ -119,8 +153,7 @@ module Exekutor
|
|
119
153
|
def verbose?
|
120
154
|
!quiet? && !!@global_options[:verbose]
|
121
155
|
end
|
122
|
-
|
123
156
|
end
|
124
157
|
end
|
125
158
|
end
|
126
|
-
end
|
159
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative "application_loader"
|
2
4
|
require_relative "default_option_value"
|
3
5
|
require_relative "daemon"
|
@@ -21,104 +23,27 @@ module Exekutor
|
|
21
23
|
# @option options [String] :environment The Rails environment to load
|
22
24
|
# @option options [String] :queue The queue(s) to watch
|
23
25
|
# @option options [String] :threads The number of threads to use for job execution
|
26
|
+
# @option options [String] :priority The priorities to execute
|
24
27
|
# @option options [Integer] :poll_interval The interval in seconds for job polling
|
25
28
|
# @return [Void]
|
26
29
|
def start(options)
|
30
|
+
Process.setproctitle "Exekutor worker (Initializing…) [#{$PROGRAM_NAME}]"
|
27
31
|
daemonize(restarting: options[:restart]) if options[:daemonize]
|
28
32
|
|
29
33
|
load_application(options[:environment])
|
30
34
|
|
31
|
-
config_files = if options[:configfile].is_a? DefaultConfigFileValue
|
32
|
-
options[:configfile].to_a(@global_options[:identifier])
|
33
|
-
else
|
34
|
-
options[:configfile]&.map { |path| File.expand_path(path, Rails.root) }
|
35
|
-
end
|
36
|
-
|
37
|
-
worker_options = DEFAULT_CONFIGURATION.dup
|
38
|
-
|
39
|
-
config_files&.each do |path|
|
40
|
-
puts "Loading config file: #{path}" if verbose?
|
41
|
-
config = begin
|
42
|
-
YAML.safe_load(File.read(path), symbolize_names: true)
|
43
|
-
rescue => e
|
44
|
-
raise Error, "Cannot read config file: #{path} (#{e.to_s})"
|
45
|
-
end
|
46
|
-
unless config.keys == [:exekutor]
|
47
|
-
raise Error, "Config should have an `exekutor` root node: #{path} (Found: #{config.keys.join(', ')})"
|
48
|
-
end
|
49
|
-
|
50
|
-
# Remove worker specific options before calling Exekutor.config.set
|
51
|
-
worker_options.merge! config[:exekutor].extract!(:queue, :status_server_port)
|
52
|
-
|
53
|
-
begin
|
54
|
-
Exekutor.config.set **config[:exekutor]
|
55
|
-
rescue => e
|
56
|
-
raise Error, "Cannot load config file: #{path} (#{e.to_s})"
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
worker_options.merge! Exekutor.config.worker_options
|
61
|
-
worker_options.merge! @global_options.slice(:identifier)
|
62
|
-
if verbose?
|
63
|
-
worker_options[:verbose] = true
|
64
|
-
elsif quiet?
|
65
|
-
worker_options[:quiet] = true
|
66
|
-
end
|
67
|
-
if options[:threads] && !options[:threads].is_a?(DefaultOptionValue)
|
68
|
-
min, max = if options[:threads].is_a?(Integer)
|
69
|
-
[options[:threads], options[:threads]]
|
70
|
-
else
|
71
|
-
options[:threads].to_s.split(":")
|
72
|
-
end
|
73
|
-
if max.nil?
|
74
|
-
options[:min_threads] = options[:max_threads] = Integer(min)
|
75
|
-
else
|
76
|
-
options[:min_threads] = Integer(min)
|
77
|
-
options[:max_threads] = Integer(max)
|
78
|
-
end
|
79
|
-
end
|
80
|
-
worker_options.merge!(
|
81
|
-
options.slice(:queue, :min_threads, :max_threads, :poll_interval)
|
82
|
-
.reject { |_, value| value.is_a? DefaultOptionValue }
|
83
|
-
.transform_keys(poll_interval: :polling_interval)
|
84
|
-
)
|
85
|
-
|
86
|
-
worker_options[:queue] = nil if worker_options[:queue] == ["*"]
|
87
|
-
|
88
|
-
# TODO health check server
|
89
|
-
|
90
35
|
# Specify `yield: true` to prevent running in the context of the loaded module
|
91
36
|
ActiveSupport.on_load(:exekutor, yield: true) do
|
92
|
-
|
93
|
-
worker = Worker.new(worker_options)
|
94
|
-
%w[INT TERM QUIT].each do |signal|
|
95
|
-
::Kernel.trap(signal) { ::Thread.new { worker.stop } }
|
96
|
-
end
|
97
|
-
|
98
|
-
Process.setproctitle "Exekutor worker #{worker.id} [#{Rails.root}]"
|
99
|
-
if worker_options[:set_db_connection_name]
|
100
|
-
Internal::BaseRecord.connection.class.set_callback(:checkout, :after) do
|
101
|
-
Internal::DatabaseConnection.set_application_name raw_connection, worker.id
|
102
|
-
end
|
103
|
-
Internal::BaseRecord.connection_pool.connections.each do |conn|
|
104
|
-
Internal::DatabaseConnection.set_application_name conn.raw_connection, worker.id
|
105
|
-
end
|
106
|
-
end
|
37
|
+
worker_options = worker_options(options[:configfile], cli_worker_overrides(options))
|
107
38
|
|
108
|
-
|
109
|
-
|
110
|
-
puts "#{worker_options.pretty_inspect}" if verbose?
|
111
|
-
begin
|
112
|
-
worker.start
|
113
|
-
worker.join
|
114
|
-
ensure
|
115
|
-
worker.stop if worker.running?
|
116
|
-
end
|
117
|
-
end
|
39
|
+
ActiveSupport.on_load(:active_record, yield: true) do
|
40
|
+
start_and_join_worker(worker_options, options[:daemonize])
|
118
41
|
end
|
119
42
|
end
|
120
43
|
end
|
121
44
|
|
45
|
+
# Stops a daemonized worker
|
46
|
+
# @return [Void]
|
122
47
|
def stop(options)
|
123
48
|
daemon = Daemon.new(pidfile: pidfile)
|
124
49
|
pid = daemon.pid
|
@@ -136,23 +61,12 @@ module Exekutor
|
|
136
61
|
end
|
137
62
|
|
138
63
|
Process.kill("INT", pid)
|
139
|
-
|
140
|
-
wait_until = if options[:shutdown_timeout].nil? || options[:shutdown_timeout] == DEFAULT_FOREVER
|
141
|
-
nil
|
142
|
-
else
|
143
|
-
Time.now + options[:shutdown_timeout]
|
144
|
-
end
|
145
|
-
while daemon.status?(:running, :not_owned)
|
146
|
-
puts "Waiting for worker to finish…" unless quiet?
|
147
|
-
if wait_until && wait_until > Time.now
|
148
|
-
Process.kill("TERM", pid)
|
149
|
-
break
|
150
|
-
end
|
151
|
-
sleep 0.1
|
152
|
-
end
|
64
|
+
wait_for_process_end(daemon, pid, shutdown_timeout(options))
|
153
65
|
puts "Worker (PID: #{pid}) stopped." unless quiet?
|
154
66
|
end
|
155
67
|
|
68
|
+
# Restarts a daemonized worker
|
69
|
+
# @return [Void]
|
156
70
|
def restart(stop_options, start_options)
|
157
71
|
stop stop_options.merge(restart: true)
|
158
72
|
start start_options.merge(restart: true, daemonize: true)
|
@@ -160,6 +74,110 @@ module Exekutor
|
|
160
74
|
|
161
75
|
private
|
162
76
|
|
77
|
+
def worker_options(config_file, cli_overrides)
|
78
|
+
worker_options = DEFAULT_CONFIGURATION.dup
|
79
|
+
|
80
|
+
ConfigLoader.new(config_file, @global_options).load_config(worker_options)
|
81
|
+
|
82
|
+
worker_options.merge! Exekutor.config.worker_options
|
83
|
+
worker_options.merge! @global_options.slice(:identifier)
|
84
|
+
worker_options.merge! cli_overrides
|
85
|
+
|
86
|
+
if quiet?
|
87
|
+
worker_options[:quiet] = true
|
88
|
+
elsif verbose?
|
89
|
+
worker_options[:verbose] = true
|
90
|
+
end
|
91
|
+
|
92
|
+
worker_options[:queue] = nil if Array.wrap(worker_options[:queue]) == ["*"]
|
93
|
+
worker_options
|
94
|
+
end
|
95
|
+
|
96
|
+
def cli_worker_overrides(cli_options)
|
97
|
+
worker_options = cli_options.slice(:queue, :poll_interval)
|
98
|
+
.reject { |_, value| value.is_a? DefaultOptionValue }
|
99
|
+
.transform_keys(poll_interval: :polling_interval)
|
100
|
+
|
101
|
+
min_threads, max_threads = parse_integer_range(cli_options[:threads])
|
102
|
+
if min_threads
|
103
|
+
worker_options[:min_threads] = min_threads
|
104
|
+
worker_options[:max_threads] = max_threads || min_threads
|
105
|
+
end
|
106
|
+
|
107
|
+
min_priority, max_priority = parse_integer_range(cli_options[:priority])
|
108
|
+
if min_threads
|
109
|
+
worker_options[:min_priority] = min_priority
|
110
|
+
worker_options[:max_priority] = max_priority if max_priority
|
111
|
+
end
|
112
|
+
|
113
|
+
worker_options
|
114
|
+
end
|
115
|
+
|
116
|
+
def parse_integer_range(threads)
|
117
|
+
return if threads.blank? || threads.is_a?(DefaultOptionValue)
|
118
|
+
|
119
|
+
if threads.is_a?(Integer)
|
120
|
+
[threads, threads]
|
121
|
+
else
|
122
|
+
threads.to_s.split(":").map { |s| Integer(s) }
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def start_and_join_worker(worker_options, is_daemonized)
|
127
|
+
worker = Worker.new(worker_options)
|
128
|
+
%w[INT TERM QUIT].each do |signal|
|
129
|
+
::Kernel.trap(signal) { ::Thread.new { worker.stop } }
|
130
|
+
end
|
131
|
+
|
132
|
+
Process.setproctitle "Exekutor worker #{worker.id} [#{Rails.root}]"
|
133
|
+
set_db_connection_name(worker.id) if worker_options[:set_db_connection_name]
|
134
|
+
|
135
|
+
ActiveSupport.on_load(:active_job, yield: true) do
|
136
|
+
worker.start
|
137
|
+
print_startup_message(worker, worker_options) unless quiet? || is_daemonized
|
138
|
+
worker.join
|
139
|
+
ensure
|
140
|
+
worker.stop if worker.running?
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def print_startup_message(worker, worker_options)
|
145
|
+
puts "Worker #{worker.id} started (Use `#{Rainbow("ctrl + c").magenta}` to stop)"
|
146
|
+
puts worker_options.pretty_inspect if verbose?
|
147
|
+
end
|
148
|
+
|
149
|
+
# rubocop:disable Naming/AccessorMethodName
|
150
|
+
def set_db_connection_name(worker_id)
|
151
|
+
# rubocop:enable Naming/AccessorMethodName
|
152
|
+
Internal::BaseRecord.connection.class.set_callback(:checkout, :after) do
|
153
|
+
Internal::DatabaseConnection.set_application_name raw_connection, worker_id
|
154
|
+
end
|
155
|
+
Internal::BaseRecord.connection_pool.connections.each do |conn|
|
156
|
+
Internal::DatabaseConnection.set_application_name conn.raw_connection, worker_id
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def wait_for_process_end(daemon, pid, shutdown_timeout)
|
161
|
+
wait_until = (Time.now.to_f + shutdown_timeout if shutdown_timeout)
|
162
|
+
sleep 0.1
|
163
|
+
while daemon.status?(:running, :not_owned)
|
164
|
+
if wait_until && wait_until > Time.now.to_f
|
165
|
+
puts "Sending TERM signal" unless quiet?
|
166
|
+
Process.kill("TERM", pid) if pid
|
167
|
+
break
|
168
|
+
end
|
169
|
+
sleep 0.1
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def shutdown_timeout(options)
|
174
|
+
if options[:shutdown_timeout].nil? || options[:shutdown_timeout] == DEFAULT_FOREVER
|
175
|
+
nil
|
176
|
+
else
|
177
|
+
options[:shutdown_timeout]
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
163
181
|
# @return [Boolean] Whether quiet mode is enabled. Overrides verbose mode.
|
164
182
|
def quiet?
|
165
183
|
!!@global_options[:quiet]
|
@@ -175,6 +193,8 @@ module Exekutor
|
|
175
193
|
@global_options[:identifier]
|
176
194
|
end
|
177
195
|
|
196
|
+
# rubocop:disable Style/FormatStringToken
|
197
|
+
|
178
198
|
# @return [String] The path to the pidfile
|
179
199
|
def pidfile
|
180
200
|
pidfile = @global_options[:pidfile] || DEFAULT_PIDFILE
|
@@ -187,37 +207,110 @@ module Exekutor
|
|
187
207
|
end
|
188
208
|
end
|
189
209
|
|
190
|
-
# Daemonizes the current process.
|
210
|
+
# Daemonizes the current process.
|
191
211
|
# @return [Void]
|
192
212
|
def daemonize(restarting: false)
|
193
213
|
daemonizer = Daemon.new(pidfile: pidfile)
|
194
214
|
daemonizer.validate!
|
195
|
-
unless quiet?
|
196
|
-
if restarting
|
197
|
-
puts "Restarting worker as a daemon…"
|
198
|
-
else
|
199
|
-
stop_options = if @global_options[:pidfile] && @global_options[:pidfile] != DEFAULT_PIDFILE
|
200
|
-
"--pid #{pidfile} "
|
201
|
-
elsif identifier
|
202
|
-
"--id #{identifier} "
|
203
|
-
end
|
215
|
+
print_daemonize_message(restarting) unless quiet?
|
204
216
|
|
205
|
-
puts "Running worker as a daemon… (Use `#{Rainbow("exekutor #{stop_options}stop").magenta}` to stop)"
|
206
|
-
end
|
207
|
-
end
|
208
217
|
daemonizer.daemonize
|
209
218
|
rescue Daemon::Error => e
|
210
219
|
puts Rainbow(e.message).red
|
211
220
|
raise GLI::CustomExit.new(nil, 1)
|
212
221
|
end
|
213
222
|
|
223
|
+
def print_daemonize_message(restarting)
|
224
|
+
if restarting
|
225
|
+
puts "Restarting worker as a daemon…"
|
226
|
+
else
|
227
|
+
stop_options = if @global_options[:pidfile] && @global_options[:pidfile] != DEFAULT_PIDFILE
|
228
|
+
"--pid #{pidfile} "
|
229
|
+
elsif identifier
|
230
|
+
"--id #{identifier} "
|
231
|
+
end
|
232
|
+
|
233
|
+
puts "Running worker as a daemon… (Use `#{Rainbow("exekutor #{stop_options}stop").magenta}` to stop)"
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
# Takes care of loading YAML configuration
|
238
|
+
class ConfigLoader
|
239
|
+
def initialize(files, options)
|
240
|
+
@config_files = files
|
241
|
+
@options = options
|
242
|
+
end
|
243
|
+
|
244
|
+
def load_config(worker_options)
|
245
|
+
each_file do |path|
|
246
|
+
config = load_config_file(path)
|
247
|
+
convert_duration_options! config
|
248
|
+
|
249
|
+
worker_options.merge! extract_worker_options!(config)
|
250
|
+
apply_config_file(config)
|
251
|
+
end
|
252
|
+
Exekutor.config
|
253
|
+
end
|
254
|
+
|
255
|
+
private
|
256
|
+
|
257
|
+
WORKER_OPTIONS = %i[queues min_priority max_priority min_threads max_threads max_thread_idletime
|
258
|
+
wait_for_termination].freeze
|
259
|
+
|
260
|
+
def each_file(&block)
|
261
|
+
if @config_files.is_a? DefaultConfigFileValue
|
262
|
+
@config_files.to_a(@options[:identifier]).each(&block)
|
263
|
+
elsif @config_files.is_a? String
|
264
|
+
yield File.expand_path(@config_files, Rails.root)
|
265
|
+
else
|
266
|
+
@config_files.map { |path| File.expand_path(path, Rails.root) }.each(&block)
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
def extract_worker_options!(config)
|
271
|
+
config.extract!(*WORKER_OPTIONS)
|
272
|
+
end
|
273
|
+
|
274
|
+
def load_config_file(path)
|
275
|
+
puts "Loading config file: #{path}" if @options[:verbose]
|
276
|
+
config = begin
|
277
|
+
YAML.safe_load(File.read(path), symbolize_names: true)
|
278
|
+
rescue StandardError => e
|
279
|
+
raise Error, "Cannot read config file: #{path} (#{e})"
|
280
|
+
end
|
281
|
+
unless config.keys == [:exekutor]
|
282
|
+
raise Error, "Config should have an `exekutor` root node: #{path} (Found: #{config.keys.join(", ")})"
|
283
|
+
end
|
284
|
+
|
285
|
+
config[:exekutor]
|
286
|
+
end
|
287
|
+
|
288
|
+
def apply_config_file(config)
|
289
|
+
Exekutor.config.set(**config)
|
290
|
+
rescue StandardError => e
|
291
|
+
raise Error, "Cannot load config file (#{e})"
|
292
|
+
end
|
293
|
+
|
294
|
+
def convert_duration_options!(config)
|
295
|
+
{ polling_interval: :seconds, max_execution_thread_idletime: :seconds, healthcheck_timeout: :minutes }
|
296
|
+
.each do |duration_option, duration_interval|
|
297
|
+
if config[duration_option].is_a? Numeric
|
298
|
+
config[duration_option] = config[duration_option].send(duration_interval)
|
299
|
+
end
|
300
|
+
end
|
301
|
+
end
|
302
|
+
end
|
303
|
+
|
304
|
+
# The default value for the pid file
|
214
305
|
class DefaultPidFileValue < DefaultOptionValue
|
215
306
|
def initialize
|
216
307
|
super("tmp/pids/exekutor[.%{identifier}].pid")
|
217
308
|
end
|
218
309
|
|
310
|
+
# @param identifier [nil,String] the worker identifier
|
311
|
+
# @return [String] the path to the default pidfile of the worker with the specified identifier
|
219
312
|
def for_identifier(identifier)
|
220
|
-
if identifier.nil? || identifier.
|
313
|
+
if identifier.nil? || identifier.empty? # rubocop:disable Rails/Blank – Rails is not loaded here
|
221
314
|
"tmp/pids/exekutor.pid"
|
222
315
|
else
|
223
316
|
"tmp/pids/exekutor.#{identifier}.pid"
|
@@ -225,36 +318,54 @@ module Exekutor
|
|
225
318
|
end
|
226
319
|
end
|
227
320
|
|
321
|
+
# The default value for the config file
|
228
322
|
class DefaultConfigFileValue < DefaultOptionValue
|
229
323
|
def initialize
|
230
|
-
super(
|
324
|
+
super(<<~DESC)
|
325
|
+
"config/exekutor.yml", overridden by "config/exekutor.%{identifier}.yml" if an identifier is specified
|
326
|
+
DESC
|
231
327
|
end
|
232
328
|
|
329
|
+
# @param identifier [nil,String] the worker identifier
|
330
|
+
# @return [Array<String>] the paths to the configfiles to load
|
233
331
|
def to_a(identifier = nil)
|
234
332
|
files = []
|
235
|
-
|
236
|
-
|
237
|
-
|
333
|
+
%w[config/exekutor.yml config/exekutor.yaml].each do |path|
|
334
|
+
path = Rails.root.join(path)
|
335
|
+
if File.exist? path
|
336
|
+
files.append path
|
337
|
+
break
|
338
|
+
end
|
339
|
+
end
|
238
340
|
if identifier.present?
|
239
|
-
|
240
|
-
|
241
|
-
|
341
|
+
%W[config/exekutor.#{identifier}.yml config/exekutor.#{identifier}.yaml].each do |path|
|
342
|
+
path = Rails.root.join(path)
|
343
|
+
if File.exist? path
|
344
|
+
files.append path
|
345
|
+
break
|
346
|
+
end
|
347
|
+
end
|
242
348
|
end
|
243
|
-
files
|
349
|
+
files
|
244
350
|
end
|
245
351
|
end
|
246
352
|
|
353
|
+
# rubocop:enable Style/FormatStringToken
|
354
|
+
|
247
355
|
DEFAULT_PIDFILE = DefaultPidFileValue.new.freeze
|
248
356
|
DEFAULT_CONFIG_FILES = DefaultConfigFileValue.new.freeze
|
249
357
|
|
250
|
-
DEFAULT_THREADS = DefaultOptionValue.new(
|
358
|
+
DEFAULT_THREADS = DefaultOptionValue.new(
|
359
|
+
"Minimum: 1, Maximum: Active record pool size minus 1, with a minimum of 1"
|
360
|
+
).freeze
|
251
361
|
DEFAULT_QUEUE = DefaultOptionValue.new("All queues").freeze
|
362
|
+
DEFAULT_PRIORITIES = DefaultOptionValue.new("All priorities").freeze
|
252
363
|
DEFAULT_FOREVER = DefaultOptionValue.new("Forever").freeze
|
253
364
|
|
254
|
-
DEFAULT_CONFIGURATION = { set_db_connection_name: true }
|
365
|
+
DEFAULT_CONFIGURATION = { set_db_connection_name: true }.freeze
|
255
366
|
|
256
367
|
class Error < StandardError; end
|
257
368
|
end
|
258
369
|
end
|
259
370
|
end
|
260
|
-
end
|
371
|
+
end
|