exekutor 0.1.0 → 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/exe/exekutor +2 -2
  4. data/lib/active_job/queue_adapters/exekutor_adapter.rb +2 -1
  5. data/lib/exekutor/asynchronous.rb +143 -75
  6. data/lib/exekutor/cleanup.rb +27 -28
  7. data/lib/exekutor/configuration.rb +48 -25
  8. data/lib/exekutor/hook.rb +15 -11
  9. data/lib/exekutor/info/worker.rb +3 -3
  10. data/lib/exekutor/internal/base_record.rb +2 -1
  11. data/lib/exekutor/internal/callbacks.rb +55 -35
  12. data/lib/exekutor/internal/cli/app.rb +31 -23
  13. data/lib/exekutor/internal/cli/application_loader.rb +17 -6
  14. data/lib/exekutor/internal/cli/cleanup.rb +54 -40
  15. data/lib/exekutor/internal/cli/daemon.rb +9 -11
  16. data/lib/exekutor/internal/cli/default_option_value.rb +3 -1
  17. data/lib/exekutor/internal/cli/info.rb +117 -84
  18. data/lib/exekutor/internal/cli/manager.rb +190 -123
  19. data/lib/exekutor/internal/configuration_builder.rb +40 -27
  20. data/lib/exekutor/internal/database_connection.rb +6 -0
  21. data/lib/exekutor/internal/executable.rb +12 -7
  22. data/lib/exekutor/internal/executor.rb +50 -21
  23. data/lib/exekutor/internal/hooks.rb +11 -8
  24. data/lib/exekutor/internal/listener.rb +66 -39
  25. data/lib/exekutor/internal/logger.rb +28 -10
  26. data/lib/exekutor/internal/provider.rb +93 -74
  27. data/lib/exekutor/internal/reserver.rb +27 -12
  28. data/lib/exekutor/internal/status_server.rb +81 -49
  29. data/lib/exekutor/job.rb +1 -1
  30. data/lib/exekutor/job_error.rb +1 -1
  31. data/lib/exekutor/job_options.rb +22 -13
  32. data/lib/exekutor/plugins/appsignal.rb +7 -5
  33. data/lib/exekutor/plugins.rb +8 -4
  34. data/lib/exekutor/queue.rb +40 -22
  35. data/lib/exekutor/version.rb +1 -1
  36. data/lib/exekutor/worker.rb +88 -47
  37. data/lib/exekutor.rb +2 -2
  38. data/lib/generators/exekutor/configuration_generator.rb +9 -5
  39. data/lib/generators/exekutor/install_generator.rb +26 -15
  40. data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +11 -10
  41. data.tar.gz.sig +0 -0
  42. metadata +63 -19
  43. metadata.gz.sig +0 -0
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Exekutor
2
4
  module Internal
3
5
  # Mixin to define callbacks on a class
@@ -49,41 +51,21 @@ module Exekutor
49
51
  # @param type [:on, :before, :around, :after] the type of the callback
50
52
  # @param action [Symbol] the name of the callback
51
53
  # @param args [Any] the callback args
52
- def run_callbacks(type, action, *args)
54
+ def run_callbacks(type, action, *args, &block)
53
55
  callbacks = __callbacks && __callbacks[:"#{type}_#{action}"]
54
56
  unless callbacks
55
- yield(*args) if block_given?
57
+ yield(*args) if block
56
58
  return
57
59
  end
58
60
  if type == :around
59
- # Chain all callbacks together, ending with the original given block
60
- callbacks.inject(-> { yield(*args) }) do |next_callback, (callback, extra_args)|
61
- callback_args = if callback.arity.zero?
62
- []
63
- else
64
- args + extra_args
65
- end
66
- lambda do
67
- has_yielded = false
68
- callback.call(*callback_args) { has_yielded = true; next_callback.call }
69
- raise MissingYield, "Callback did not yield!" unless has_yielded
70
- rescue StandardError => err
71
- raise if err.is_a? MissingYield
72
- Exekutor.on_fatal_error err, "[Executor] Callback error!"
73
- next_callback.call
74
- end
75
- end.call
76
- return
77
- end
78
- iterator = type == :after ? :each : :reverse_each
79
- callbacks.send(iterator) do |(callback, extra_args)|
80
- if callback.arity.zero?
81
- callback.call
82
- else
83
- callback.call(*(args + extra_args))
61
+ chain_callbacks(callbacks, args, &block).call
62
+ else
63
+ # Invoke :before in reverse order (last registered first),
64
+ # invoke :after in original order (last registered last)
65
+ iterator = type == :after ? :each : :reverse_each
66
+ callbacks.send(iterator) do |(callback, extra_args)|
67
+ invoke_callback(callback, args, extra_args)
84
68
  end
85
- rescue StandardError => err
86
- Exekutor.on_fatal_error err, "[Executor] Callback error!"
87
69
  end
88
70
  nil
89
71
  end
@@ -98,6 +80,41 @@ module Exekutor
98
80
 
99
81
  private
100
82
 
83
+ # Chain all callbacks together, ending with the original given block
84
+ def chain_callbacks(callbacks, args)
85
+ callbacks.inject(-> { yield(*args) }) do |next_callback, (callback, extra_args)|
86
+ # collect args outside of the lambda
87
+ callback_args = if callback.arity.zero?
88
+ []
89
+ else
90
+ args + extra_args
91
+ end
92
+ lambda do
93
+ has_yielded = false
94
+ callback.call(*callback_args) do
95
+ has_yielded = true
96
+ next_callback.call
97
+ end
98
+ raise MissingYield, "Callback did not yield!" unless has_yielded
99
+ rescue StandardError => e
100
+ raise if e.is_a? MissingYield
101
+
102
+ Exekutor.on_fatal_error e, "[Executor] Callback error!"
103
+ next_callback.call
104
+ end
105
+ end
106
+ end
107
+
108
+ def invoke_callback(callback, args, extra_args)
109
+ if callback.arity.zero?
110
+ callback.call
111
+ else
112
+ callback.call(*(args + extra_args))
113
+ end
114
+ rescue StandardError => e
115
+ Exekutor.on_fatal_error e, "[Executor] Callback error!"
116
+ end
117
+
101
118
  def add_callback!(type, args, callback)
102
119
  @__callbacks ||= Concurrent::Hash.new
103
120
  __callbacks[type] ||= Concurrent::Array.new
@@ -105,12 +122,15 @@ module Exekutor
105
122
  end
106
123
 
107
124
  class_methods do
108
- # Defines the specified callbacks on this class. Also defines a method with the given name to register the callback.
109
- # @param callbacks [Symbol] the callback names to define. Must start with +on_+, +before_+, +after_+, or +around_+.
125
+ # Defines the specified callbacks on this class. Also defines a method with the given name to register the
126
+ # callback.
127
+ # @param callbacks [Symbol] the callback names to define. Must start with +on_+, +before_+, +after_+, or
128
+ # +around_+.
110
129
  # @param freeze [Boolean] if true, freezes the callbacks so that no other callbacks can be defined
111
130
  # @raise [Error] if a callback name is invalid or if the callbacks are frozen
112
131
  def define_callbacks(*callbacks, freeze: true)
113
132
  raise Error, "Callbacks are frozen, no other callbacks may be defined" if __callback_names.frozen?
133
+
114
134
  callbacks.each do |name|
115
135
  unless /^(on_)|(before_)|(after_)|(around_)[a-z]+/.match? name.to_s
116
136
  raise Error, "Callback name must start with `on_`, `before_`, `after_`, or `around_`"
@@ -118,9 +138,9 @@ module Exekutor
118
138
 
119
139
  __callback_names << name
120
140
  module_eval <<-RUBY, __FILE__, __LINE__ + 1
121
- def #{name}(*args, &callback)
122
- add_callback! :#{name}, args, callback
123
- end
141
+ def #{name}(*args, &callback) # def callback_method(*args, &callback
142
+ add_callback! :#{name}, args, callback # add_callback! :callback_method, args, callback
143
+ end # end
124
144
  RUBY
125
145
  end
126
146
 
@@ -135,4 +155,4 @@ module Exekutor
135
155
  class MissingYield < Exekutor::Error; end
136
156
  end
137
157
  end
138
- end
158
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "gli"
2
4
  require "rainbow"
3
5
  require_relative "cleanup"
@@ -29,24 +31,28 @@ module Exekutor
29
31
  switch %i[quiet], negatable: false, desc: "Enable less output"
30
32
 
31
33
  # Defines start command flags
32
- def self.define_start_options(c)
33
- c.flag %i[env environment], desc: "The Rails environment"
34
- c.flag %i[q queue], default_value: Manager::DEFAULT_QUEUE, multiple: true,
35
- desc: "Queue to work from"
36
- c.flag %i[t threads], type: String, default_value: Manager::DEFAULT_THREADS,
37
- desc: "The number of threads for executing jobs, specified as `min:max`"
38
- c.flag %i[p poll_interval], type: Integer, default_value: DefaultOptionValue.new( value: 60),
39
- desc: "Interval between polls for available jobs (in seconds)"
40
- c.flag %i[cfg configfile], type: String, default_value: Manager::DEFAULT_CONFIG_FILES, multiple: true,
41
- desc: "The YAML configuration file to load. If specifying multiple files, the last file takes precedence"
34
+ def self.define_start_options(cmd)
35
+ cmd.flag %i[env environment], desc: "The Rails environment"
36
+ cmd.flag %i[q queue], default_value: Manager::DEFAULT_QUEUE, multiple: true,
37
+ desc: "Queue to work from"
38
+ cmd.flag %i[t threads], type: String, default_value: Manager::DEFAULT_THREADS,
39
+ desc: "The number of threads for executing jobs, specified as `min:max`"
40
+ cmd.flag %i[p poll_interval], type: Integer, default_value: DefaultOptionValue.new(value: 60),
41
+ desc: "Interval between polls for available jobs (in seconds)"
42
+ cmd.flag %i[cfg configfile], type: String, default_value: Manager::DEFAULT_CONFIG_FILES, multiple: true,
43
+ desc: "The YAML configuration file to load. If specifying multiple files, the last file takes " \
44
+ "precedence"
42
45
  end
46
+
43
47
  private_class_method :define_start_options
44
48
 
45
49
  # Defines stop command flags
46
- def self.define_stop_options(c)
47
- c.flag %i[timeout shutdown_timeout], default_value: Manager::DEFAULT_FOREVER,
48
- desc: "Number of seconds to wait for jobs to finish when shutting down before killing the worker. (in seconds)"
50
+ def self.define_stop_options(cmd)
51
+ cmd.flag %i[timeout shutdown_timeout], default_value: Manager::DEFAULT_FOREVER,
52
+ desc: "Number of seconds to wait for jobs to finish when shutting down before killing the worker. " \
53
+ "(in seconds)"
49
54
  end
55
+
50
56
  private_class_method :define_stop_options
51
57
 
52
58
  desc "Starts a worker"
@@ -67,8 +73,8 @@ module Exekutor
67
73
 
68
74
  desc "Stops a daemonized worker"
69
75
  long_desc <<~TEXT
70
- Stops a daemonized worker. This uses the PID file to send a shutdown command to a running worker. If the worker
71
- does not exit within the shutdown timeout it will kill the process.
76
+ Stops a daemonized worker. This uses the PID file to send a shutdown command to a running worker. If the
77
+ worker does not exit within the shutdown timeout it will kill the process.
72
78
  TEXT
73
79
  command :stop do |c|
74
80
  c.switch :all, negatable: false, desc: "Stops all workers with default pid files."
@@ -76,7 +82,9 @@ module Exekutor
76
82
 
77
83
  c.action do |global_options, options|
78
84
  if options[:all]
79
- puts "The identifier option is ignored for --all" unless global_options[:identifier].nil? || global_options[:quiet]
85
+ unless global_options[:identifier].nil? || global_options[:quiet]
86
+ puts "The identifier option is ignored for --all"
87
+ end
80
88
  pidfile_pattern = if options[:pidfile].nil? || options[:pidfile] == Manager::DEFAULT_PIDFILE
81
89
  "tmp/pids/exekutor*.pid"
82
90
  else
@@ -98,8 +106,8 @@ module Exekutor
98
106
 
99
107
  desc "Restarts a daemonized worker"
100
108
  long_desc <<~TEXT
101
- Restarts a daemonized worker. Will issue the stop command if a worker is running and wait for the active worker
102
- to exit before starting a new worker. If no worker is currently running, a new worker will be started.
109
+ Restarts a daemonized worker. Will issue the stop command if a worker is running and wait for the active
110
+ worker to exit before starting a new worker. If no worker is currently running, a new worker will be started.
103
111
  TEXT
104
112
  command :restart do |c|
105
113
  define_stop_options c
@@ -107,7 +115,7 @@ module Exekutor
107
115
 
108
116
  c.action do |global_options, options|
109
117
  Manager.new(global_options).restart(options.slice(:shutdown_timeout),
110
- options.reject { |k, _| k == :shutdown_timeout })
118
+ options.except(:shutdown_timeout))
111
119
  end
112
120
  end
113
121
 
@@ -127,14 +135,15 @@ module Exekutor
127
135
  long_desc <<~TEXT
128
136
  Cleans up the finished jobs and stale workers
129
137
  TEXT
130
- command :cleanup do |c|
138
+ command :cleanup do |c| # rubocop:disable Metrics/BlockLength
131
139
  c.flag %i[env environment], desc: "The Rails environment."
132
140
 
133
141
  c.flag %i[t timeout],
134
142
  desc: "The global timeout in hours. Workers and jobs before the timeout will be purged"
135
143
  c.flag %i[worker_timeout],
136
144
  default_value: 4,
137
- desc: "The worker timeout in hours. Workers where the last heartbeat is before the timeout will be deleted."
145
+ desc: "The worker timeout in hours. Workers where the last heartbeat is before the timeout will be " \
146
+ "deleted."
138
147
  c.flag %i[job_timeout],
139
148
  default_value: 48,
140
149
  desc: "The job timeout in hours. Jobs where scheduled at is before the timeout will be purged."
@@ -167,7 +176,6 @@ module Exekutor
167
176
  end
168
177
  end
169
178
  end
170
-
171
179
  end
172
180
  end
173
- end
181
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Exekutor
2
4
  module Internal
3
5
  module CLI
@@ -12,6 +14,7 @@ module Exekutor
12
14
  # @param print_message [Boolean] whether to print a loading message to STDOUT
13
15
  def load_application(environment, path = "config/environment.rb", print_message: false)
14
16
  return if @application_loaded
17
+
15
18
  if print_message
16
19
  printf LOADING_MESSAGE
17
20
  @loading_message_printed = true
@@ -23,14 +26,22 @@ module Exekutor
23
26
 
24
27
  # Clears the loading message if it was printed
25
28
  def clear_application_loading_message
26
- if @loading_message_printed
27
- printf "\r#{" " * LOADING_MESSAGE.length}\r"
28
- @loading_message_printed = false
29
- end
29
+ return unless @loading_message_printed
30
+
31
+ printf "\r#{" " * LOADING_MESSAGE.length}\r"
32
+ @loading_message_printed = false
30
33
  end
31
34
 
35
+ # @return Whether the system time zone differs from the app time zone
36
+ def different_time_zone?
37
+ Time.zone.name != Time.new.zone
38
+ end
39
+
40
+ # Prints a message to STDOUT indicating in which time zone times are printed
41
+ def print_time_zone_warning
42
+ puts "(times are printed in the #{Time.zone.name} time zone)\n\n"
43
+ end
32
44
  end
33
45
  end
34
46
  end
35
-
36
- end
47
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative "application_loader"
2
4
  require_relative "default_option_value"
3
5
  require "terminal-table"
@@ -21,26 +23,13 @@ module Exekutor
21
23
  load_application options[:environment], print_message: !quiet?
22
24
 
23
25
  ActiveSupport.on_load(:active_record, yield: true) do
24
- # Use system time zone
25
- Time.zone = Time.new.zone
26
+ clear_application_loading_message
27
+ print_time_zone_warning if different_time_zone? && !quiet?
26
28
 
27
- clear_application_loading_message unless quiet?
28
- timeout = options[:timeout] || options[:worker_timeout] || 4
29
+ timeout = worker_cleanup_timeout(options)
29
30
  workers = cleaner.cleanup_workers timeout: timeout.hours
30
- return if quiet?
31
-
32
- puts Rainbow("Workers").bright.blue if options[:print_header]
33
- if workers.present?
34
- puts "Purged #{workers.size} worker#{"s" if workers.many?}"
35
- if verbose?
36
- table = Terminal::Table.new
37
- table.headings = ["id", "Last heartbeat"]
38
- workers.each { |w| table << [w.id.split("-").first << "…", w.last_heartbeat_at] }
39
- puts table
40
- end
41
- else
42
- puts "Nothing purged"
43
- end
31
+
32
+ print_worker_cleanup_result(options, workers) unless quiet?
44
33
  end
45
34
  end
46
35
 
@@ -50,30 +39,56 @@ module Exekutor
50
39
  load_application options[:environment], print_message: !quiet?
51
40
 
52
41
  ActiveSupport.on_load(:active_record, yield: true) do
53
- # Use system time zone
54
- Time.zone = Time.new.zone
55
-
56
- clear_application_loading_message unless quiet?
57
- timeout = options[:timeout] || options[:job_timeout] || 48
58
- status = if options[:job_status].is_a? Array
59
- options[:job_status]
60
- elsif options[:job_status] && options[:job_status] != DEFAULT_STATUSES
61
- options[:job_status]
62
- end
63
- purged_count = cleaner.cleanup_jobs before: timeout.hours.ago, status: status
64
- return if quiet?
65
-
66
- puts Rainbow("Jobs").bright.blue if options[:print_header]
67
- if purged_count.zero?
68
- puts "Nothing purged"
69
- else
70
- puts "Purged #{purged_count} job#{"s" if purged_count > 1}"
71
- end
42
+ clear_application_loading_message
43
+ print_time_zone_warning if different_time_zone? && !quiet?
44
+
45
+ timeout = job_cleanup_timeout(options)
46
+ purged_count = cleaner.cleanup_jobs before: timeout.hours.ago, status: job_cleanup_statuses(options)
47
+
48
+ print_job_cleanup_result(options, purged_count) unless quiet?
72
49
  end
73
50
  end
74
51
 
75
52
  private
76
53
 
54
+ def job_cleanup_statuses(options)
55
+ options[:job_status] if options[:job_status] && options[:job_status] != DEFAULT_STATUSES
56
+ end
57
+
58
+ def job_cleanup_timeout(options)
59
+ options[:timeout] || options[:job_timeout] || 48
60
+ end
61
+
62
+ def print_job_cleanup_result(options, purged_count)
63
+ puts Rainbow("Jobs").bright.blue if options[:print_header]
64
+ if purged_count.zero?
65
+ puts "Nothing purged"
66
+ else
67
+ puts "Purged #{purged_count} job#{"s" if purged_count > 1}"
68
+ end
69
+ end
70
+
71
+ def worker_cleanup_timeout(options)
72
+ options[:timeout] || options[:worker_timeout] || 4
73
+ end
74
+
75
+ def print_worker_cleanup_result(options, workers)
76
+ puts Rainbow("Workers").bright.blue if options[:print_header]
77
+ if workers.present?
78
+ puts "Purged #{workers.size} worker#{"s" if workers.many?}"
79
+ print_worker_info(workers) if verbose?
80
+ else
81
+ puts "Nothing purged"
82
+ end
83
+ end
84
+
85
+ def print_worker_info(workers)
86
+ table = Terminal::Table.new
87
+ table.headings = ["id", "Last heartbeat"]
88
+ workers.each { |w| table << [w.id.split("-").first << "…", w.last_heartbeat_at] }
89
+ puts table
90
+ end
91
+
77
92
  # @return [Boolean] Whether quiet mode is enabled. Overrides verbose mode.
78
93
  def quiet?
79
94
  !!@global_options[:quiet]
@@ -85,12 +100,11 @@ module Exekutor
85
100
  end
86
101
 
87
102
  def cleaner
88
- @delegate ||= ::Exekutor::Cleanup.new
103
+ @cleaner ||= ::Exekutor::Cleanup.new
89
104
  end
90
105
 
91
106
  DEFAULT_STATUSES = DefaultOptionValue.new("All except :pending").freeze
92
107
  end
93
108
  end
94
-
95
109
  end
96
- end
110
+ end
@@ -31,11 +31,9 @@ module Exekutor
31
31
  return nil unless ::File.exist? pidfile
32
32
 
33
33
  pid = ::File.read(pidfile)
34
- if pid.to_i.positive?
35
- pid.to_i
36
- else
37
- raise Error, "Corrupt PID-file. Check #{pidfile}"
38
- end
34
+ raise Error, "Corrupt PID-file. Check #{pidfile}" unless pid.to_i.positive?
35
+
36
+ pid.to_i
39
37
  end
40
38
 
41
39
  # The process status for this daemon. Possible states are:
@@ -48,8 +46,8 @@ module Exekutor
48
46
  pid = self.pid
49
47
  return :not_running if pid.nil?
50
48
 
51
- # If sig is 0, then no signal is sent, but error checking is still performed; this can be used to check for the
52
- # existence of a process ID or process group ID.
49
+ # If sig is 0, then no signal is sent, but error checking is still performed; this can be used to check for
50
+ # the existence of a process ID or process group ID.
53
51
  ::Process.kill(0, pid)
54
52
  :running
55
53
  rescue Errno::ESRCH
@@ -63,14 +61,14 @@ module Exekutor
63
61
  # @return [Boolean] whether the status matches
64
62
  # @see #status
65
63
  def status?(*statuses)
66
- statuses.include? self.status
64
+ statuses.include? status
67
65
  end
68
66
 
69
67
  # Raises an {Error} if a daemon is already running. Deletes the pidfile is the process is dead.
70
68
  # @return [void]
71
69
  # @raise [Error] when the daemon is running
72
70
  def validate!
73
- case self.status
71
+ case status
74
72
  when :running, :not_owned
75
73
  raise Error, "A worker is already running. Check #{pidfile}"
76
74
  else
@@ -97,7 +95,7 @@ module Exekutor
97
95
  # @return [void]
98
96
  # @see #pidfile
99
97
  def delete_pid
100
- File.delete(pidfile) if File.exist?(pidfile)
98
+ FileUtils.rm_f(pidfile)
101
99
  end
102
100
 
103
101
  # Raised when spawning a daemon process fails
@@ -105,4 +103,4 @@ module Exekutor
105
103
  end
106
104
  end
107
105
  end
108
- end
106
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Exekutor
2
4
  # @private
3
5
  module Internal
@@ -26,4 +28,4 @@ module Exekutor
26
28
  end
27
29
  end
28
30
  end
29
- end
31
+ end