backburner-allq 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.travis.yml +29 -0
  4. data/CHANGELOG.md +133 -0
  5. data/CONTRIBUTING.md +37 -0
  6. data/Gemfile +4 -0
  7. data/HOOKS.md +99 -0
  8. data/LICENSE +22 -0
  9. data/README.md +658 -0
  10. data/Rakefile +17 -0
  11. data/TODO +4 -0
  12. data/backburner-allq.gemspec +26 -0
  13. data/bin/backburner +7 -0
  14. data/circle.yml +3 -0
  15. data/deploy.sh +3 -0
  16. data/examples/custom.rb +25 -0
  17. data/examples/demo.rb +60 -0
  18. data/examples/god.rb +46 -0
  19. data/examples/hooked.rb +87 -0
  20. data/examples/retried.rb +31 -0
  21. data/examples/simple.rb +43 -0
  22. data/examples/stress.rb +31 -0
  23. data/lib/backburner.rb +75 -0
  24. data/lib/backburner/allq_wrapper.rb +317 -0
  25. data/lib/backburner/async_proxy.rb +25 -0
  26. data/lib/backburner/cli.rb +53 -0
  27. data/lib/backburner/configuration.rb +48 -0
  28. data/lib/backburner/connection.rb +157 -0
  29. data/lib/backburner/helpers.rb +193 -0
  30. data/lib/backburner/hooks.rb +53 -0
  31. data/lib/backburner/job.rb +118 -0
  32. data/lib/backburner/logger.rb +53 -0
  33. data/lib/backburner/performable.rb +95 -0
  34. data/lib/backburner/queue.rb +145 -0
  35. data/lib/backburner/tasks.rb +54 -0
  36. data/lib/backburner/version.rb +3 -0
  37. data/lib/backburner/worker.rb +221 -0
  38. data/lib/backburner/workers/forking.rb +52 -0
  39. data/lib/backburner/workers/simple.rb +29 -0
  40. data/lib/backburner/workers/threading.rb +163 -0
  41. data/lib/backburner/workers/threads_on_fork.rb +263 -0
  42. data/test/async_proxy_test.rb +36 -0
  43. data/test/back_burner_test.rb +88 -0
  44. data/test/connection_test.rb +179 -0
  45. data/test/fixtures/hooked.rb +122 -0
  46. data/test/fixtures/test_fork_jobs.rb +72 -0
  47. data/test/fixtures/test_forking_jobs.rb +56 -0
  48. data/test/fixtures/test_jobs.rb +87 -0
  49. data/test/fixtures/test_queue_settings.rb +14 -0
  50. data/test/helpers/templogger.rb +22 -0
  51. data/test/helpers_test.rb +278 -0
  52. data/test/hooks_test.rb +112 -0
  53. data/test/job_test.rb +185 -0
  54. data/test/logger_test.rb +44 -0
  55. data/test/performable_test.rb +88 -0
  56. data/test/queue_test.rb +69 -0
  57. data/test/test_helper.rb +128 -0
  58. data/test/worker_test.rb +157 -0
  59. data/test/workers/forking_worker_test.rb +181 -0
  60. data/test/workers/simple_worker_test.rb +350 -0
  61. data/test/workers/threading_worker_test.rb +104 -0
  62. data/test/workers/threads_on_fork_worker_test.rb +484 -0
  63. metadata +217 -0
@@ -0,0 +1,29 @@
1
+ module Backburner
2
+ module Workers
3
+ class Simple < Worker
4
+ # Used to prepare job queues before processing jobs.
5
+ # Setup beanstalk tube_names and watch all specified tubes for jobs.
6
+ #
7
+ # @raise [Beaneater::NotConnected] If beanstalk fails to connect.
8
+ # @example
9
+ # @worker.prepare
10
+ #
11
+ def prepare
12
+ self.tube_names.map! { |name| expand_tube_name(name) }.uniq!
13
+ log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
14
+ self.connection.tubes.watch!(*self.tube_names)
15
+ end
16
+
17
+ # Starts processing new jobs indefinitely.
18
+ # Primary way to consume and process jobs in specified tubes.
19
+ #
20
+ # @example
21
+ # @worker.start
22
+ #
23
+ def start
24
+ prepare
25
+ loop { work_one_job }
26
+ end
27
+ end # Basic
28
+ end # Workers
29
+ end # Backburner
@@ -0,0 +1,163 @@
1
+ require 'concurrent'
2
+
3
+ module Backburner
4
+ module Workers
5
+ class Threading < Worker
6
+ attr_accessor :self_read, :self_write, :exit_on_shutdown
7
+
8
+ @shutdown_timeout = 10
9
+
10
+ class << self
11
+ attr_accessor :threads_number
12
+ attr_accessor :shutdown_timeout
13
+ end
14
+
15
+ # Custom initializer just to set @tubes_data
16
+ def initialize(*args)
17
+ @tubes_data = {}
18
+ super
19
+ self.process_tube_options
20
+ @exit_on_shutdown = true
21
+ end
22
+
23
+ # Used to prepare job queues before processing jobs.
24
+ # Setup beanstalk tube_names and watch all specified tubes for jobs.
25
+ #
26
+ # @raise [Beaneater::NotConnected] If beanstalk fails to connect.
27
+ # @example
28
+ # @worker.prepare
29
+ #
30
+ def prepare
31
+ self.tube_names.map! { |name| expand_tube_name(name) }.uniq!
32
+ log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
33
+ @thread_pools = {}
34
+ @tubes_data.each do |name, config|
35
+ max_threads = (config[:threads] || self.class.threads_number || ::Concurrent.processor_count).to_i
36
+ @thread_pools[name] = (::Concurrent::ThreadPoolExecutor.new(min_threads: 1, max_threads: max_threads))
37
+ end
38
+ end
39
+
40
+ # Starts processing new jobs indefinitely.
41
+ # Primary way to consume and process jobs in specified tubes.
42
+ #
43
+ # @example
44
+ # @worker.start
45
+ #
46
+ def start(wait=true)
47
+ prepare
48
+
49
+ @thread_pools.each do |tube_name, pool|
50
+ pool.max_length.times do
51
+ # Create a new connection and set it up to listen on this tube name
52
+ connection = new_connection.tap{ |conn| conn.tubes.watch!(tube_name) }
53
+ connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
54
+
55
+ # Make it work jobs using its own connection per thread
56
+ pool.post(connection) do |memo_connection|
57
+ # TODO: use read-write lock?
58
+ loop do
59
+ begin
60
+ break if @in_shutdown
61
+ work_one_job(memo_connection)
62
+ rescue => e
63
+ log_error("Exception caught in thread pool loop. Continuing. -> #{e.message}\nBacktrace: #{e.backtrace}")
64
+ end
65
+ end
66
+
67
+ connection.close
68
+ end
69
+ end
70
+ end
71
+
72
+ wait_for_shutdown! if wait
73
+ end
74
+
75
+ # FIXME: We can't use this on_reconnect method since we don't know which thread
76
+ # pool the connection belongs to (and therefore we can't re-watch the right tubes).
77
+ # However, we set the individual connections' on_reconnect method in #start
78
+ # def on_reconnect(conn)
79
+ # watch_tube(@watching_tube, conn) if @watching_tube
80
+ # end
81
+
82
+ # Process the special tube_names of Threading worker:
83
+ # The format is tube_name:custom_threads_limit
84
+ #
85
+ # @example
86
+ # process_tube_names(['foo:10', 'lol'])
87
+ # => ['foo', lol']
88
+ def process_tube_names(tube_names)
89
+ names = compact_tube_names(tube_names)
90
+ if names.nil?
91
+ nil
92
+ else
93
+ names.map do |name|
94
+ data = name.split(":")
95
+ tube_name = data.first
96
+ threads_number = data[1].empty? ? nil : data[1].to_i rescue nil
97
+ @tubes_data[expand_tube_name(tube_name)] = {
98
+ :threads => threads_number
99
+ }
100
+ tube_name
101
+ end
102
+ end
103
+ end
104
+
105
+ # Process the tube settings
106
+ # This overrides @tubes_data set by process_tube_names method. So a tube has name 'super_job:5'
107
+ # and the tube class has setting queue_jobs_limit 10, the result limit will be 10
108
+ # If the tube is known by existing beanstalkd queue, but not by class - skip it
109
+ #
110
+ def process_tube_options
111
+ Backburner::Worker.known_queue_classes.each do |queue|
112
+ next if @tubes_data[expand_tube_name(queue)].nil?
113
+ queue_settings = {
114
+ :threads => queue.queue_jobs_limit
115
+ }
116
+ @tubes_data[expand_tube_name(queue)].merge!(queue_settings){|k, v1, v2| v2.nil? ? v1 : v2 }
117
+ end
118
+ end
119
+
120
+ # Wait for the shutdown signel
121
+ def wait_for_shutdown!
122
+ raise Interrupt while IO.select([self_read])
123
+ rescue Interrupt
124
+ shutdown
125
+ end
126
+
127
+ def shutdown_threadpools
128
+ @thread_pools.each { |_name, pool| pool.shutdown }
129
+ shutdown_time = Time.now
130
+ @in_shutdown = true
131
+ all_shutdown = @thread_pools.all? do |_name, pool|
132
+ time_to_wait = self.class.shutdown_timeout - (Time.now - shutdown_time).to_i
133
+ pool.wait_for_termination(time_to_wait) if time_to_wait > 0
134
+ end
135
+ rescue Interrupt
136
+ log_info "graceful shutdown aborted, shutting down immediately"
137
+ ensure
138
+ kill unless all_shutdown
139
+ end
140
+
141
+ def kill
142
+ @thread_pools.each { |_name, pool| pool.kill unless pool.shutdown? }
143
+ end
144
+
145
+ def shutdown
146
+ log_info "beginning graceful worker shutdown"
147
+ shutdown_threadpools
148
+ super if @exit_on_shutdown
149
+ end
150
+
151
+ # Registers signal handlers TERM and INT to trigger
152
+ def register_signal_handlers!
153
+ @self_read, @self_write = IO.pipe
154
+ %w[TERM INT].each do |sig|
155
+ trap(sig) do
156
+ raise Interrupt if @in_shutdown
157
+ self_write.puts(sig)
158
+ end
159
+ end
160
+ end
161
+ end # Threading
162
+ end # Workers
163
+ end # Backburner
@@ -0,0 +1,263 @@
1
+ module Backburner
2
+ module Workers
3
+ class ThreadsOnFork < Worker
4
+ class << self
5
+ attr_accessor :shutdown
6
+ attr_accessor :threads_number
7
+ attr_accessor :garbage_after
8
+ attr_accessor :is_child
9
+
10
+ # return the pids of all alive children/forks
11
+ def child_pids
12
+ return [] if is_child
13
+ @child_pids ||= []
14
+ tmp_ids = []
15
+ for id in @child_pids
16
+ next if id.to_i == Process.pid
17
+ begin
18
+ Process.kill(0, id)
19
+ tmp_ids << id
20
+ rescue Errno::ESRCH
21
+ end
22
+ end
23
+ @child_pids = tmp_ids if @child_pids != tmp_ids
24
+ @child_pids
25
+ end
26
+
27
+ # Send a SIGTERM signal to all children
28
+ # This is the same of a normal exit
29
+ # We are simply asking the children to exit
30
+ def stop_forks
31
+ for id in child_pids
32
+ begin
33
+ Process.kill("SIGTERM", id)
34
+ rescue Errno::ESRCH
35
+ end
36
+ end
37
+ end
38
+
39
+ # Send a SIGKILL signal to all children
40
+ # This is the same of assassinate
41
+ # We are KILLING those folks that don't obey us
42
+ def kill_forks
43
+ for id in child_pids
44
+ begin
45
+ Process.kill("SIGKILL", id)
46
+ rescue Errno::ESRCH
47
+ end
48
+ end
49
+ end
50
+
51
+ def finish_forks
52
+ return if is_child
53
+
54
+ ids = child_pids
55
+ if ids.length > 0
56
+ puts "[ThreadsOnFork workers] Stopping forks: #{ids.join(", ")}"
57
+ stop_forks
58
+ Kernel.sleep 1
59
+ ids = child_pids
60
+ if ids.length > 0
61
+ puts "[ThreadsOnFork workers] Killing remaining forks: #{ids.join(", ")}"
62
+ kill_forks
63
+ Process.waitall
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ # Custom initializer just to set @tubes_data
70
+ def initialize(*args)
71
+ @tubes_data = {}
72
+ super
73
+ self.process_tube_options
74
+ end
75
+
76
+ # Process the special tube_names of ThreadsOnFork worker
77
+ # The idea is tube_name:custom_threads_limit:custom_garbage_limit:custom_retries
78
+ # Any custom can be ignore. So if you want to set just the custom_retries
79
+ # you will need to write this 'tube_name:::10'
80
+ #
81
+ # @example
82
+ # process_tube_names(['foo:10:5:1', 'bar:2::3', 'lol'])
83
+ # => ['foo', 'bar', 'lol']
84
+ def process_tube_names(tube_names)
85
+ names = compact_tube_names(tube_names)
86
+ if names.nil?
87
+ nil
88
+ else
89
+ names.map do |name|
90
+ data = name.split(":")
91
+ tube_name = data.first
92
+ threads_number = data[1].empty? ? nil : data[1].to_i rescue nil
93
+ garbage_number = data[2].empty? ? nil : data[2].to_i rescue nil
94
+ retries_number = data[3].empty? ? nil : data[3].to_i rescue nil
95
+ @tubes_data[expand_tube_name(tube_name)] = {
96
+ :threads => threads_number,
97
+ :garbage => garbage_number,
98
+ :retries => retries_number
99
+ }
100
+ tube_name
101
+ end
102
+ end
103
+ end
104
+
105
+ # Process the tube settings
106
+ # This overrides @tubes_data set by process_tube_names method. So a tube has name 'super_job:5:20:10'
107
+ # and the tube class has setting queue_jobs_limit 10, the result limit will be 10
108
+ # If the tube is known by existing beanstalkd queue, but not by class - skip it
109
+ #
110
+ def process_tube_options
111
+ Backburner::Worker.known_queue_classes.each do |queue|
112
+ next if @tubes_data[expand_tube_name(queue)].nil?
113
+ queue_settings = {
114
+ :threads => queue.queue_jobs_limit,
115
+ :garbage => queue.queue_garbage_limit,
116
+ :retries => queue.queue_retry_limit
117
+ }
118
+ @tubes_data[expand_tube_name(queue)].merge!(queue_settings){|k, v1, v2| v2.nil? ? v1 : v2 }
119
+ end
120
+ end
121
+
122
+ def prepare
123
+ self.tube_names ||= Backburner.default_queues.any? ? Backburner.default_queues : all_existing_queues
124
+ self.tube_names = Array(self.tube_names)
125
+ tube_names.map! { |name| expand_tube_name(name) }.uniq!
126
+ tube_display_names = tube_names.map{|name| "#{name}:#{@tubes_data[name].values}"}
127
+ log_info "Working #{tube_names.size} queues: [ #{tube_display_names.join(', ')} ]"
128
+ end
129
+
130
+ # For each tube we will call fork_and_watch to create the fork
131
+ # The lock argument define if this method should block or no
132
+ def start(lock=true)
133
+ prepare
134
+ tube_names.each do |name|
135
+ fork_and_watch(name)
136
+ end
137
+
138
+ if lock
139
+ sleep 0.1 while true
140
+ end
141
+ end
142
+
143
+ # Make the fork and create a thread to watch the child process
144
+ # The exit code '99' means that the fork exited because of the garbage limit
145
+ # Any other code is an error
146
+ def fork_and_watch(name)
147
+ create_thread(name) do |tube_name|
148
+ until self.class.shutdown
149
+ pid = fork_tube(tube_name)
150
+ _, status = wait_for_process(pid)
151
+
152
+ # 99 = garbaged
153
+ if status.exitstatus != 99
154
+ log_error("Catastrophic failure: tube #{tube_name} exited with code #{status.exitstatus}.")
155
+ end
156
+ end
157
+ end
158
+ end
159
+
160
+ # This makes easy to test
161
+ def fork_tube(name)
162
+ fork_it do
163
+ fork_inner(name)
164
+ end
165
+ end
166
+
167
+ # Here we are already on the forked child
168
+ # We will watch just the selected tube and change the configuration of
169
+ # queue_config.max_job_retries if needed
170
+ #
171
+ # If we limit the number of threads to 1 it will just run in a loop without
172
+ # creating any extra thread.
173
+ def fork_inner(name)
174
+ if @tubes_data[name]
175
+ queue_config.max_job_retries = @tubes_data[name][:retries] if @tubes_data[name][:retries]
176
+ else
177
+ @tubes_data[name] = {}
178
+ end
179
+ @garbage_after = @tubes_data[name][:garbage] || self.class.garbage_after
180
+ @threads_number = (@tubes_data[name][:threads] || self.class.threads_number || 1).to_i
181
+
182
+ @runs = 0
183
+
184
+ if @threads_number == 1
185
+ watch_tube(name)
186
+ run_while_can
187
+ else
188
+ threads_count = Thread.list.count
189
+ @threads_number.times do
190
+ create_thread do
191
+ begin
192
+ conn = new_connection
193
+ watch_tube(name, conn)
194
+ run_while_can(conn)
195
+ ensure
196
+ conn.close if conn
197
+ end
198
+ end
199
+ end
200
+ sleep 0.1 while Thread.list.count > threads_count
201
+ end
202
+
203
+ coolest_exit
204
+ end
205
+
206
+ # Run work_one_job while we can
207
+ def run_while_can(conn = connection)
208
+ while @garbage_after.nil? or @garbage_after > @runs
209
+ @runs += 1 # FIXME: Likely race condition
210
+ work_one_job(conn)
211
+ end
212
+ end
213
+
214
+ # Shortcut for watching a tube on our beanstalk connection
215
+ def watch_tube(name, conn = connection)
216
+ # No op for allq
217
+ end
218
+
219
+ def on_reconnect(conn)
220
+ watch_tube(@watching_tube, conn) if @watching_tube
221
+ end
222
+
223
+ # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
224
+ # parent process
225
+ # We will use exitcode 99 that means the fork reached the garbage number
226
+ def coolest_exit
227
+ Kernel.exit! 99
228
+ end
229
+
230
+ # Create a thread. Easy to test
231
+ def create_thread(*args, &block)
232
+ Thread.new(*args, &block)
233
+ end
234
+
235
+ # Wait for a specific process. Easy to test
236
+ def wait_for_process(pid)
237
+ out = Process.wait2(pid)
238
+ self.class.child_pids.delete(pid)
239
+ out
240
+ end
241
+
242
+ # Forks the specified block and adds the process to the child process pool
243
+ # FIXME: If blk.call breaks then the pid isn't added to child_pids and is
244
+ # never shutdown
245
+ def fork_it(&blk)
246
+ pid = Kernel.fork do
247
+ self.class.is_child = true
248
+ $0 = "[ThreadsOnFork worker] parent: #{Process.ppid}"
249
+ blk.call
250
+ end
251
+ self.class.child_pids << pid
252
+ pid
253
+ end
254
+ end
255
+ end
256
+ end
257
+
258
+ at_exit do
259
+ unless Backburner::Workers::ThreadsOnFork.is_child
260
+ Backburner::Workers::ThreadsOnFork.shutdown = true
261
+ end
262
+ Backburner::Workers::ThreadsOnFork.finish_forks
263
+ end