exekutor 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/exe/exekutor +2 -2
  4. data/lib/active_job/queue_adapters/exekutor_adapter.rb +2 -1
  5. data/lib/exekutor/asynchronous.rb +143 -75
  6. data/lib/exekutor/cleanup.rb +27 -28
  7. data/lib/exekutor/configuration.rb +48 -25
  8. data/lib/exekutor/hook.rb +15 -11
  9. data/lib/exekutor/info/worker.rb +3 -3
  10. data/lib/exekutor/internal/base_record.rb +2 -1
  11. data/lib/exekutor/internal/callbacks.rb +55 -35
  12. data/lib/exekutor/internal/cli/app.rb +31 -23
  13. data/lib/exekutor/internal/cli/application_loader.rb +17 -6
  14. data/lib/exekutor/internal/cli/cleanup.rb +54 -40
  15. data/lib/exekutor/internal/cli/daemon.rb +9 -11
  16. data/lib/exekutor/internal/cli/default_option_value.rb +3 -1
  17. data/lib/exekutor/internal/cli/info.rb +117 -84
  18. data/lib/exekutor/internal/cli/manager.rb +190 -123
  19. data/lib/exekutor/internal/configuration_builder.rb +40 -27
  20. data/lib/exekutor/internal/database_connection.rb +6 -0
  21. data/lib/exekutor/internal/executable.rb +12 -7
  22. data/lib/exekutor/internal/executor.rb +50 -21
  23. data/lib/exekutor/internal/hooks.rb +11 -8
  24. data/lib/exekutor/internal/listener.rb +66 -39
  25. data/lib/exekutor/internal/logger.rb +28 -10
  26. data/lib/exekutor/internal/provider.rb +93 -74
  27. data/lib/exekutor/internal/reserver.rb +27 -12
  28. data/lib/exekutor/internal/status_server.rb +81 -49
  29. data/lib/exekutor/job.rb +1 -1
  30. data/lib/exekutor/job_error.rb +1 -1
  31. data/lib/exekutor/job_options.rb +22 -13
  32. data/lib/exekutor/plugins/appsignal.rb +7 -5
  33. data/lib/exekutor/plugins.rb +8 -4
  34. data/lib/exekutor/queue.rb +40 -22
  35. data/lib/exekutor/version.rb +1 -1
  36. data/lib/exekutor/worker.rb +88 -47
  37. data/lib/exekutor.rb +2 -2
  38. data/lib/generators/exekutor/configuration_generator.rb +9 -5
  39. data/lib/generators/exekutor/install_generator.rb +26 -15
  40. data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +11 -10
  41. data.tar.gz.sig +0 -0
  42. metadata +63 -19
  43. metadata.gz.sig +0 -0
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Exekutor
2
4
  module Internal
3
5
  # Mixin to define callbacks on a class
@@ -49,41 +51,21 @@ module Exekutor
49
51
  # @param type [:on, :before, :around, :after] the type of the callback
50
52
  # @param action [Symbol] the name of the callback
51
53
  # @param args [Any] the callback args
52
- def run_callbacks(type, action, *args)
54
+ def run_callbacks(type, action, *args, &block)
53
55
  callbacks = __callbacks && __callbacks[:"#{type}_#{action}"]
54
56
  unless callbacks
55
- yield(*args) if block_given?
57
+ yield(*args) if block
56
58
  return
57
59
  end
58
60
  if type == :around
59
- # Chain all callbacks together, ending with the original given block
60
- callbacks.inject(-> { yield(*args) }) do |next_callback, (callback, extra_args)|
61
- callback_args = if callback.arity.zero?
62
- []
63
- else
64
- args + extra_args
65
- end
66
- lambda do
67
- has_yielded = false
68
- callback.call(*callback_args) { has_yielded = true; next_callback.call }
69
- raise MissingYield, "Callback did not yield!" unless has_yielded
70
- rescue StandardError => err
71
- raise if err.is_a? MissingYield
72
- Exekutor.on_fatal_error err, "[Executor] Callback error!"
73
- next_callback.call
74
- end
75
- end.call
76
- return
77
- end
78
- iterator = type == :after ? :each : :reverse_each
79
- callbacks.send(iterator) do |(callback, extra_args)|
80
- if callback.arity.zero?
81
- callback.call
82
- else
83
- callback.call(*(args + extra_args))
61
+ chain_callbacks(callbacks, args, &block).call
62
+ else
63
+ # Invoke :before in reverse order (last registered first),
64
+ # invoke :after in original order (last registered last)
65
+ iterator = type == :after ? :each : :reverse_each
66
+ callbacks.send(iterator) do |(callback, extra_args)|
67
+ invoke_callback(callback, args, extra_args)
84
68
  end
85
- rescue StandardError => err
86
- Exekutor.on_fatal_error err, "[Executor] Callback error!"
87
69
  end
88
70
  nil
89
71
  end
@@ -98,6 +80,41 @@ module Exekutor
98
80
 
99
81
  private
100
82
 
83
+ # Chain all callbacks together, ending with the original given block
84
+ def chain_callbacks(callbacks, args)
85
+ callbacks.inject(-> { yield(*args) }) do |next_callback, (callback, extra_args)|
86
+ # collect args outside of the lambda
87
+ callback_args = if callback.arity.zero?
88
+ []
89
+ else
90
+ args + extra_args
91
+ end
92
+ lambda do
93
+ has_yielded = false
94
+ callback.call(*callback_args) do
95
+ has_yielded = true
96
+ next_callback.call
97
+ end
98
+ raise MissingYield, "Callback did not yield!" unless has_yielded
99
+ rescue StandardError => e
100
+ raise if e.is_a? MissingYield
101
+
102
+ Exekutor.on_fatal_error e, "[Executor] Callback error!"
103
+ next_callback.call
104
+ end
105
+ end
106
+ end
107
+
108
+ def invoke_callback(callback, args, extra_args)
109
+ if callback.arity.zero?
110
+ callback.call
111
+ else
112
+ callback.call(*(args + extra_args))
113
+ end
114
+ rescue StandardError => e
115
+ Exekutor.on_fatal_error e, "[Executor] Callback error!"
116
+ end
117
+
101
118
  def add_callback!(type, args, callback)
102
119
  @__callbacks ||= Concurrent::Hash.new
103
120
  __callbacks[type] ||= Concurrent::Array.new
@@ -105,12 +122,15 @@ module Exekutor
105
122
  end
106
123
 
107
124
  class_methods do
108
- # Defines the specified callbacks on this class. Also defines a method with the given name to register the callback.
109
- # @param callbacks [Symbol] the callback names to define. Must start with +on_+, +before_+, +after_+, or +around_+.
125
+ # Defines the specified callbacks on this class. Also defines a method with the given name to register the
126
+ # callback.
127
+ # @param callbacks [Symbol] the callback names to define. Must start with +on_+, +before_+, +after_+, or
128
+ # +around_+.
110
129
  # @param freeze [Boolean] if true, freezes the callbacks so that no other callbacks can be defined
111
130
  # @raise [Error] if a callback name is invalid or if the callbacks are frozen
112
131
  def define_callbacks(*callbacks, freeze: true)
113
132
  raise Error, "Callbacks are frozen, no other callbacks may be defined" if __callback_names.frozen?
133
+
114
134
  callbacks.each do |name|
115
135
  unless /^(on_)|(before_)|(after_)|(around_)[a-z]+/.match? name.to_s
116
136
  raise Error, "Callback name must start with `on_`, `before_`, `after_`, or `around_`"
@@ -118,9 +138,9 @@ module Exekutor
118
138
 
119
139
  __callback_names << name
120
140
  module_eval <<-RUBY, __FILE__, __LINE__ + 1
121
- def #{name}(*args, &callback)
122
- add_callback! :#{name}, args, callback
123
- end
141
+ def #{name}(*args, &callback) # def callback_method(*args, &callback
142
+ add_callback! :#{name}, args, callback # add_callback! :callback_method, args, callback
143
+ end # end
124
144
  RUBY
125
145
  end
126
146
 
@@ -135,4 +155,4 @@ module Exekutor
135
155
  class MissingYield < Exekutor::Error; end
136
156
  end
137
157
  end
138
- end
158
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "gli"
2
4
  require "rainbow"
3
5
  require_relative "cleanup"
@@ -29,24 +31,28 @@ module Exekutor
29
31
  switch %i[quiet], negatable: false, desc: "Enable less output"
30
32
 
31
33
  # Defines start command flags
32
- def self.define_start_options(c)
33
- c.flag %i[env environment], desc: "The Rails environment"
34
- c.flag %i[q queue], default_value: Manager::DEFAULT_QUEUE, multiple: true,
35
- desc: "Queue to work from"
36
- c.flag %i[t threads], type: String, default_value: Manager::DEFAULT_THREADS,
37
- desc: "The number of threads for executing jobs, specified as `min:max`"
38
- c.flag %i[p poll_interval], type: Integer, default_value: DefaultOptionValue.new( value: 60),
39
- desc: "Interval between polls for available jobs (in seconds)"
40
- c.flag %i[cfg configfile], type: String, default_value: Manager::DEFAULT_CONFIG_FILES, multiple: true,
41
- desc: "The YAML configuration file to load. If specifying multiple files, the last file takes precedence"
34
+ def self.define_start_options(cmd)
35
+ cmd.flag %i[env environment], desc: "The Rails environment"
36
+ cmd.flag %i[q queue], default_value: Manager::DEFAULT_QUEUE, multiple: true,
37
+ desc: "Queue to work from"
38
+ cmd.flag %i[t threads], type: String, default_value: Manager::DEFAULT_THREADS,
39
+ desc: "The number of threads for executing jobs, specified as `min:max`"
40
+ cmd.flag %i[p poll_interval], type: Integer, default_value: DefaultOptionValue.new(value: 60),
41
+ desc: "Interval between polls for available jobs (in seconds)"
42
+ cmd.flag %i[cfg configfile], type: String, default_value: Manager::DEFAULT_CONFIG_FILES, multiple: true,
43
+ desc: "The YAML configuration file to load. If specifying multiple files, the last file takes " \
44
+ "precedence"
42
45
  end
46
+
43
47
  private_class_method :define_start_options
44
48
 
45
49
  # Defines stop command flags
46
- def self.define_stop_options(c)
47
- c.flag %i[timeout shutdown_timeout], default_value: Manager::DEFAULT_FOREVER,
48
- desc: "Number of seconds to wait for jobs to finish when shutting down before killing the worker. (in seconds)"
50
+ def self.define_stop_options(cmd)
51
+ cmd.flag %i[timeout shutdown_timeout], default_value: Manager::DEFAULT_FOREVER,
52
+ desc: "Number of seconds to wait for jobs to finish when shutting down before killing the worker. " \
53
+ "(in seconds)"
49
54
  end
55
+
50
56
  private_class_method :define_stop_options
51
57
 
52
58
  desc "Starts a worker"
@@ -67,8 +73,8 @@ module Exekutor
67
73
 
68
74
  desc "Stops a daemonized worker"
69
75
  long_desc <<~TEXT
70
- Stops a daemonized worker. This uses the PID file to send a shutdown command to a running worker. If the worker
71
- does not exit within the shutdown timeout it will kill the process.
76
+ Stops a daemonized worker. This uses the PID file to send a shutdown command to a running worker. If the
77
+ worker does not exit within the shutdown timeout it will kill the process.
72
78
  TEXT
73
79
  command :stop do |c|
74
80
  c.switch :all, negatable: false, desc: "Stops all workers with default pid files."
@@ -76,7 +82,9 @@ module Exekutor
76
82
 
77
83
  c.action do |global_options, options|
78
84
  if options[:all]
79
- puts "The identifier option is ignored for --all" unless global_options[:identifier].nil? || global_options[:quiet]
85
+ unless global_options[:identifier].nil? || global_options[:quiet]
86
+ puts "The identifier option is ignored for --all"
87
+ end
80
88
  pidfile_pattern = if options[:pidfile].nil? || options[:pidfile] == Manager::DEFAULT_PIDFILE
81
89
  "tmp/pids/exekutor*.pid"
82
90
  else
@@ -98,8 +106,8 @@ module Exekutor
98
106
 
99
107
  desc "Restarts a daemonized worker"
100
108
  long_desc <<~TEXT
101
- Restarts a daemonized worker. Will issue the stop command if a worker is running and wait for the active worker
102
- to exit before starting a new worker. If no worker is currently running, a new worker will be started.
109
+ Restarts a daemonized worker. Will issue the stop command if a worker is running and wait for the active
110
+ worker to exit before starting a new worker. If no worker is currently running, a new worker will be started.
103
111
  TEXT
104
112
  command :restart do |c|
105
113
  define_stop_options c
@@ -107,7 +115,7 @@ module Exekutor
107
115
 
108
116
  c.action do |global_options, options|
109
117
  Manager.new(global_options).restart(options.slice(:shutdown_timeout),
110
- options.reject { |k, _| k == :shutdown_timeout })
118
+ options.except(:shutdown_timeout))
111
119
  end
112
120
  end
113
121
 
@@ -127,14 +135,15 @@ module Exekutor
127
135
  long_desc <<~TEXT
128
136
  Cleans up the finished jobs and stale workers
129
137
  TEXT
130
- command :cleanup do |c|
138
+ command :cleanup do |c| # rubocop:disable Metrics/BlockLength
131
139
  c.flag %i[env environment], desc: "The Rails environment."
132
140
 
133
141
  c.flag %i[t timeout],
134
142
  desc: "The global timeout in hours. Workers and jobs before the timeout will be purged"
135
143
  c.flag %i[worker_timeout],
136
144
  default_value: 4,
137
- desc: "The worker timeout in hours. Workers where the last heartbeat is before the timeout will be deleted."
145
+ desc: "The worker timeout in hours. Workers where the last heartbeat is before the timeout will be " \
146
+ "deleted."
138
147
  c.flag %i[job_timeout],
139
148
  default_value: 48,
140
149
  desc: "The job timeout in hours. Jobs where scheduled at is before the timeout will be purged."
@@ -167,7 +176,6 @@ module Exekutor
167
176
  end
168
177
  end
169
178
  end
170
-
171
179
  end
172
180
  end
173
- end
181
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Exekutor
2
4
  module Internal
3
5
  module CLI
@@ -12,6 +14,7 @@ module Exekutor
12
14
  # @param print_message [Boolean] whether to print a loading message to STDOUT
13
15
  def load_application(environment, path = "config/environment.rb", print_message: false)
14
16
  return if @application_loaded
17
+
15
18
  if print_message
16
19
  printf LOADING_MESSAGE
17
20
  @loading_message_printed = true
@@ -23,14 +26,22 @@ module Exekutor
23
26
 
24
27
  # Clears the loading message if it was printed
25
28
  def clear_application_loading_message
26
- if @loading_message_printed
27
- printf "\r#{" " * LOADING_MESSAGE.length}\r"
28
- @loading_message_printed = false
29
- end
29
+ return unless @loading_message_printed
30
+
31
+ printf "\r#{" " * LOADING_MESSAGE.length}\r"
32
+ @loading_message_printed = false
30
33
  end
31
34
 
35
+ # @return Whether the system time zone differs from the app time zone
36
+ def different_time_zone?
37
+ Time.zone.name != Time.new.zone
38
+ end
39
+
40
+ # Prints a message to STDOUT indicating in which time zone times are printed
41
+ def print_time_zone_warning
42
+ puts "(times are printed in the #{Time.zone.name} time zone)\n\n"
43
+ end
32
44
  end
33
45
  end
34
46
  end
35
-
36
- end
47
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative "application_loader"
2
4
  require_relative "default_option_value"
3
5
  require "terminal-table"
@@ -21,26 +23,13 @@ module Exekutor
21
23
  load_application options[:environment], print_message: !quiet?
22
24
 
23
25
  ActiveSupport.on_load(:active_record, yield: true) do
24
- # Use system time zone
25
- Time.zone = Time.new.zone
26
+ clear_application_loading_message
27
+ print_time_zone_warning if different_time_zone? && !quiet?
26
28
 
27
- clear_application_loading_message unless quiet?
28
- timeout = options[:timeout] || options[:worker_timeout] || 4
29
+ timeout = worker_cleanup_timeout(options)
29
30
  workers = cleaner.cleanup_workers timeout: timeout.hours
30
- return if quiet?
31
-
32
- puts Rainbow("Workers").bright.blue if options[:print_header]
33
- if workers.present?
34
- puts "Purged #{workers.size} worker#{"s" if workers.many?}"
35
- if verbose?
36
- table = Terminal::Table.new
37
- table.headings = ["id", "Last heartbeat"]
38
- workers.each { |w| table << [w.id.split("-").first << "…", w.last_heartbeat_at] }
39
- puts table
40
- end
41
- else
42
- puts "Nothing purged"
43
- end
31
+
32
+ print_worker_cleanup_result(options, workers) unless quiet?
44
33
  end
45
34
  end
46
35
 
@@ -50,30 +39,56 @@ module Exekutor
50
39
  load_application options[:environment], print_message: !quiet?
51
40
 
52
41
  ActiveSupport.on_load(:active_record, yield: true) do
53
- # Use system time zone
54
- Time.zone = Time.new.zone
55
-
56
- clear_application_loading_message unless quiet?
57
- timeout = options[:timeout] || options[:job_timeout] || 48
58
- status = if options[:job_status].is_a? Array
59
- options[:job_status]
60
- elsif options[:job_status] && options[:job_status] != DEFAULT_STATUSES
61
- options[:job_status]
62
- end
63
- purged_count = cleaner.cleanup_jobs before: timeout.hours.ago, status: status
64
- return if quiet?
65
-
66
- puts Rainbow("Jobs").bright.blue if options[:print_header]
67
- if purged_count.zero?
68
- puts "Nothing purged"
69
- else
70
- puts "Purged #{purged_count} job#{"s" if purged_count > 1}"
71
- end
42
+ clear_application_loading_message
43
+ print_time_zone_warning if different_time_zone? && !quiet?
44
+
45
+ timeout = job_cleanup_timeout(options)
46
+ purged_count = cleaner.cleanup_jobs before: timeout.hours.ago, status: job_cleanup_statuses(options)
47
+
48
+ print_job_cleanup_result(options, purged_count) unless quiet?
72
49
  end
73
50
  end
74
51
 
75
52
  private
76
53
 
54
+ def job_cleanup_statuses(options)
55
+ options[:job_status] if options[:job_status] && options[:job_status] != DEFAULT_STATUSES
56
+ end
57
+
58
+ def job_cleanup_timeout(options)
59
+ options[:timeout] || options[:job_timeout] || 48
60
+ end
61
+
62
+ def print_job_cleanup_result(options, purged_count)
63
+ puts Rainbow("Jobs").bright.blue if options[:print_header]
64
+ if purged_count.zero?
65
+ puts "Nothing purged"
66
+ else
67
+ puts "Purged #{purged_count} job#{"s" if purged_count > 1}"
68
+ end
69
+ end
70
+
71
+ def worker_cleanup_timeout(options)
72
+ options[:timeout] || options[:worker_timeout] || 4
73
+ end
74
+
75
+ def print_worker_cleanup_result(options, workers)
76
+ puts Rainbow("Workers").bright.blue if options[:print_header]
77
+ if workers.present?
78
+ puts "Purged #{workers.size} worker#{"s" if workers.many?}"
79
+ print_worker_info(workers) if verbose?
80
+ else
81
+ puts "Nothing purged"
82
+ end
83
+ end
84
+
85
+ def print_worker_info(workers)
86
+ table = Terminal::Table.new
87
+ table.headings = ["id", "Last heartbeat"]
88
+ workers.each { |w| table << [w.id.split("-").first << "…", w.last_heartbeat_at] }
89
+ puts table
90
+ end
91
+
77
92
  # @return [Boolean] Whether quiet mode is enabled. Overrides verbose mode.
78
93
  def quiet?
79
94
  !!@global_options[:quiet]
@@ -85,12 +100,11 @@ module Exekutor
85
100
  end
86
101
 
87
102
  def cleaner
88
- @delegate ||= ::Exekutor::Cleanup.new
103
+ @cleaner ||= ::Exekutor::Cleanup.new
89
104
  end
90
105
 
91
106
  DEFAULT_STATUSES = DefaultOptionValue.new("All except :pending").freeze
92
107
  end
93
108
  end
94
-
95
109
  end
96
- end
110
+ end
@@ -31,11 +31,9 @@ module Exekutor
31
31
  return nil unless ::File.exist? pidfile
32
32
 
33
33
  pid = ::File.read(pidfile)
34
- if pid.to_i.positive?
35
- pid.to_i
36
- else
37
- raise Error, "Corrupt PID-file. Check #{pidfile}"
38
- end
34
+ raise Error, "Corrupt PID-file. Check #{pidfile}" unless pid.to_i.positive?
35
+
36
+ pid.to_i
39
37
  end
40
38
 
41
39
  # The process status for this daemon. Possible states are:
@@ -48,8 +46,8 @@ module Exekutor
48
46
  pid = self.pid
49
47
  return :not_running if pid.nil?
50
48
 
51
- # If sig is 0, then no signal is sent, but error checking is still performed; this can be used to check for the
52
- # existence of a process ID or process group ID.
49
+ # If sig is 0, then no signal is sent, but error checking is still performed; this can be used to check for
50
+ # the existence of a process ID or process group ID.
53
51
  ::Process.kill(0, pid)
54
52
  :running
55
53
  rescue Errno::ESRCH
@@ -63,14 +61,14 @@ module Exekutor
63
61
  # @return [Boolean] whether the status matches
64
62
  # @see #status
65
63
  def status?(*statuses)
66
- statuses.include? self.status
64
+ statuses.include? status
67
65
  end
68
66
 
69
67
  # Raises an {Error} if a daemon is already running. Deletes the pidfile is the process is dead.
70
68
  # @return [void]
71
69
  # @raise [Error] when the daemon is running
72
70
  def validate!
73
- case self.status
71
+ case status
74
72
  when :running, :not_owned
75
73
  raise Error, "A worker is already running. Check #{pidfile}"
76
74
  else
@@ -97,7 +95,7 @@ module Exekutor
97
95
  # @return [void]
98
96
  # @see #pidfile
99
97
  def delete_pid
100
- File.delete(pidfile) if File.exist?(pidfile)
98
+ FileUtils.rm_f(pidfile)
101
99
  end
102
100
 
103
101
  # Raised when spawning a daemon process fails
@@ -105,4 +103,4 @@ module Exekutor
105
103
  end
106
104
  end
107
105
  end
108
- end
106
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Exekutor
2
4
  # @private
3
5
  module Internal
@@ -26,4 +28,4 @@ module Exekutor
26
28
  end
27
29
  end
28
30
  end
29
- end
31
+ end