coney_island 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/coney_island +1 -1
- data/lib/coney_island.rb +33 -248
- data/lib/coney_island/submitter.rb +92 -0
- data/lib/coney_island/version.rb +1 -1
- data/lib/coney_island/worker.rb +195 -0
- data/test/coney_island_test.rb +19 -3
- data/test/dummy/db/test.sqlite3 +0 -0
- data/test/dummy/log/test.log +0 -0
- data/test/submitter_test.rb +33 -0
- data/test/test_helper.rb +40 -1
- data/test/worker_test.rb +51 -0
- metadata +29 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fdf7b8e2dfbaa750deb86831a4c1a98b42e620b0
|
4
|
+
data.tar.gz: 05701ccb0b9c4c3cdf4f1d293270bc55062c8c26
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be8931b31581fa30a558947a0625b672733b3b8dba793ef7887fed06ae659aa9f538452d30ae20d2f8bc36b97330b1844851258ee8992a31a45f051adc350001
|
7
|
+
data.tar.gz: 367b56c8b52f12e96f2451d80860e256767f013c3d3c19430822695da2923a08ef64d283771d150605be8ee3bdfd100b9b9baff8853b33c75f937373abb85b51
|
data/bin/coney_island
CHANGED
data/lib/coney_island.rb
CHANGED
@@ -1,13 +1,8 @@
|
|
1
1
|
module ConeyIsland
|
2
|
-
BG_TIMEOUT_SECONDS = 30
|
3
2
|
|
4
|
-
|
5
|
-
@run_inline = true
|
6
|
-
end
|
3
|
+
### BEGIN configuration
|
7
4
|
|
8
|
-
|
9
|
-
@notifier = "ConeyIsland::Notifiers::#{service_name}Notifier".constantize
|
10
|
-
end
|
5
|
+
BG_TIMEOUT_SECONDS = 30
|
11
6
|
|
12
7
|
def self.amqp_connection
|
13
8
|
@connection
|
@@ -18,13 +13,17 @@ module ConeyIsland
|
|
18
13
|
end
|
19
14
|
|
20
15
|
def self.amqp_parameters
|
21
|
-
@amqp_parameters ||= (self.config.present? ? self.config[
|
16
|
+
@amqp_parameters ||= (self.config.present? ? self.config[:amqp_connection] : nil)
|
22
17
|
end
|
23
18
|
|
24
19
|
def self.handle_connection
|
25
20
|
@connection ||= AMQP.connect(self.amqp_parameters)
|
26
21
|
@channel ||= AMQP::Channel.new(@connection)
|
27
|
-
|
22
|
+
self.exchange = @channel.topic('coney_island')
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.exchange=(amqp_exchange)
|
26
|
+
@exchange ||= amqp_exchange
|
28
27
|
end
|
29
28
|
|
30
29
|
def self.exchange
|
@@ -35,266 +34,52 @@ module ConeyIsland
|
|
35
34
|
@channel
|
36
35
|
end
|
37
36
|
|
38
|
-
def self.config
|
39
|
-
|
40
|
-
@config = Psych.load(File.read(File.join(Rails.root,"config","coney_island.yml")))
|
41
|
-
@config = @config[Rails.env]
|
42
|
-
end
|
37
|
+
def self.config=(config_hash)
|
38
|
+
ConeyIsland::Worker.config=(config_hash)
|
43
39
|
end
|
44
40
|
|
45
|
-
|
46
|
-
|
47
|
-
def self.submit(*args)
|
48
|
-
if RequestStore.store[:cache_jobs]
|
49
|
-
RequestStore.store[:jobs].push args
|
50
|
-
else
|
51
|
-
self.submit!(args)
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
def self.submit!(args)
|
56
|
-
if @run_inline
|
57
|
-
self.handle_publish(args)
|
58
|
-
else
|
59
|
-
EventMachine.next_tick do
|
60
|
-
self.handle_publish(args)
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
def self.handle_publish(args)
|
66
|
-
self.handle_connection unless @run_inline
|
67
|
-
jobs = (args.first.is_a? Array) ? args : [args]
|
68
|
-
jobs.each do |args|
|
69
|
-
if (args.first.is_a? Class or args.first.is_a? Module) and (args[1].is_a? String or args[1].is_a? Symbol) and args.last.is_a? Hash and 3 == args.length
|
70
|
-
klass = args.shift
|
71
|
-
klass = klass.name unless @run_inline
|
72
|
-
method_name = args.shift
|
73
|
-
job_args = args.shift
|
74
|
-
job_args ||= {}
|
75
|
-
job_args['klass'] = klass
|
76
|
-
job_args['method_name'] = method_name
|
77
|
-
if @run_inline
|
78
|
-
job_args.stringify_keys!
|
79
|
-
method_args = job_args['args']
|
80
|
-
if job_args.has_key? 'instance_id'
|
81
|
-
instance_id = job_args.delete 'instance_id'
|
82
|
-
object = klass.find(instance_id)
|
83
|
-
else
|
84
|
-
object = klass
|
85
|
-
end
|
86
|
-
if method_args && (method_args.length > 0)
|
87
|
-
object.send method_name, *method_args
|
88
|
-
else
|
89
|
-
object.send method_name
|
90
|
-
end
|
91
|
-
else
|
92
|
-
work_queue = job_args.delete :work_queue
|
93
|
-
work_queue ||= 'default'
|
94
|
-
self.exchange.publish((job_args.to_json), routing_key: "carousels.#{work_queue}")
|
95
|
-
end
|
96
|
-
end
|
97
|
-
RequestStore.store[:completed_jobs] ||= 0
|
98
|
-
RequestStore.store[:completed_jobs] += 1
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
def self.cache_jobs
|
103
|
-
RequestStore.store[:cache_jobs] = true
|
104
|
-
RequestStore.store[:jobs] = []
|
41
|
+
def self.config
|
42
|
+
ConeyIsland::Worker.config
|
105
43
|
end
|
106
44
|
|
107
|
-
def self.
|
108
|
-
|
109
|
-
self.submit!(jobs) if jobs.any?
|
110
|
-
RequestStore.store[:jobs] = []
|
45
|
+
def self.initialize_background
|
46
|
+
ConeyIsland::Worker.initialize_background
|
111
47
|
end
|
112
48
|
|
113
|
-
def self.
|
114
|
-
|
115
|
-
ConeyIsland.cache_jobs
|
116
|
-
ConeyIsland.submit(klass, method, *args)
|
117
|
-
ConeyIsland.flush_jobs
|
118
|
-
ConeyIsland.publisher_shutdown
|
119
|
-
end
|
49
|
+
def self.start_worker
|
50
|
+
ConeyIsland::Worker.start
|
120
51
|
end
|
121
52
|
|
122
|
-
def self.
|
123
|
-
|
124
|
-
if RequestStore.store[:jobs] && (RequestStore.store[:jobs].length > RequestStore.store[:completed_jobs])
|
125
|
-
Rails.logger.info("Waiting for #{RequestStore.store[:jobs].length - RequestStore.store[:completed_jobs]} publishes to finish")
|
126
|
-
else
|
127
|
-
Rails.logger.info("Shutting down coney island publisher")
|
128
|
-
EventMachine.stop
|
129
|
-
end
|
130
|
-
end
|
53
|
+
def self.run_inline
|
54
|
+
ConeyIsland::Submitter.run_inline
|
131
55
|
end
|
132
56
|
|
133
|
-
|
134
|
-
|
135
|
-
def self.initialize_background
|
136
|
-
ENV['NEW_RELIC_AGENT_ENABLED'] = 'false'
|
137
|
-
ENV['NEWRELIC_ENABLE'] = 'false'
|
138
|
-
@ticket = ARGV[0]
|
139
|
-
|
140
|
-
#TODO: set an env variable or constant that can be checked by pubnub to decide sync or not
|
141
|
-
@log_io = self.config['log'].constantize rescue nil
|
142
|
-
@log_io ||= self.config['log']
|
143
|
-
@log = Logger.new(@log_io)
|
144
|
-
|
145
|
-
@ticket ||= 'default'
|
146
|
-
|
147
|
-
@instance_config = self.config['carousels'][@ticket]
|
148
|
-
|
149
|
-
@prefetch_count = @instance_config['prefetch_count'] if @instance_config
|
150
|
-
@prefetch_count ||= 20
|
151
|
-
|
152
|
-
@worker_count = @instance_config['worker_count'] if @instance_config
|
153
|
-
@worker_count ||= 1
|
154
|
-
@child_count = @worker_count - 1
|
155
|
-
@child_pids = []
|
156
|
-
|
157
|
-
@full_instance_name = @ticket
|
158
|
-
@job_attempts = {}
|
159
|
-
|
160
|
-
@log.level = @config['log_level']
|
161
|
-
@log.info("config: #{self.config}")
|
162
|
-
|
57
|
+
def self.stop_running_inline
|
58
|
+
ConeyIsland::Submitter.stop_running_inline
|
163
59
|
end
|
164
60
|
|
165
|
-
def self.
|
166
|
-
|
167
|
-
@child_pids.each do |child_pid|
|
168
|
-
Process.kill(signal, child_pid)
|
169
|
-
end
|
170
|
-
@queue.unsubscribe
|
171
|
-
EventMachine.add_periodic_timer(1) do
|
172
|
-
if @job_attempts.any?
|
173
|
-
@log.info("Waiting for #{@job_attempts.length} requests to finish")
|
174
|
-
else
|
175
|
-
@log.info("Shutting down coney island #{@ticket}")
|
176
|
-
EventMachine.stop
|
177
|
-
end
|
178
|
-
end
|
61
|
+
def self.cache_jobs
|
62
|
+
ConeyIsland::Submitter.cache_jobs
|
179
63
|
end
|
180
64
|
|
181
|
-
def self.
|
182
|
-
|
183
|
-
method_name = args['method_name']
|
184
|
-
klass = class_name.constantize
|
185
|
-
method_args = args['args']
|
186
|
-
timeout = args['timeout']
|
187
|
-
timeout ||= BG_TIMEOUT_SECONDS
|
188
|
-
begin
|
189
|
-
Timeout::timeout(timeout) do
|
190
|
-
if args.has_key? 'instance_id'
|
191
|
-
instance_id = args['instance_id']
|
192
|
-
object = klass.find(instance_id)
|
193
|
-
else
|
194
|
-
object = klass
|
195
|
-
end
|
196
|
-
if method_args and method_args.length > 0
|
197
|
-
object.send method_name, *method_args
|
198
|
-
else
|
199
|
-
object.send method_name
|
200
|
-
end
|
201
|
-
end
|
202
|
-
rescue Timeout::Error => e
|
203
|
-
if @job_attempts.has_key? job_id
|
204
|
-
if @job_attempts[job_id] >= 3
|
205
|
-
@log.error("Request #{job_id} timed out after #{timeout} seconds, bailing out after 3 attempts")
|
206
|
-
self.finalize_job(metadata,job_id)
|
207
|
-
self.poke_the_badger(e, {work_queue: @ticket, job_payload: args, reason: 'Bailed out after 3 attempts'})
|
208
|
-
else
|
209
|
-
@log.error("Request #{job_id} timed out after #{timeout} seconds on attempt number #{@job_attempts[job_id]}, retrying...")
|
210
|
-
@job_attempts[job_id] += 1
|
211
|
-
self.handle_job(metadata,args,job_id)
|
212
|
-
end
|
213
|
-
end
|
214
|
-
rescue Exception => e
|
215
|
-
self.poke_the_badger(e, {work_queue: @ticket, job_payload: args})
|
216
|
-
@log.error("Error executing #{class_name}##{method_name} #{job_id} for id #{args['instance_id']} with args #{args}:")
|
217
|
-
@log.error(e.message)
|
218
|
-
@log.error(e.backtrace.join("\n"))
|
219
|
-
self.finalize_job(metadata,job_id)
|
220
|
-
else
|
221
|
-
self.finalize_job(metadata,job_id)
|
222
|
-
end
|
65
|
+
def self.stop_caching_jobs
|
66
|
+
ConeyIsland::Submitter.stop_caching_jobs
|
223
67
|
end
|
224
68
|
|
225
|
-
def self.
|
226
|
-
|
227
|
-
@log.info("finished job #{job_id}")
|
228
|
-
@job_attempts.delete job_id
|
69
|
+
def self.flush_jobs
|
70
|
+
ConeyIsland::Submitter.flush_jobs
|
229
71
|
end
|
230
72
|
|
231
|
-
def self.
|
232
|
-
|
233
|
-
child_pid = Process.fork
|
234
|
-
unless child_pid
|
235
|
-
@log.info("started child for ticket #{@ticket} with pid #{Process.pid}")
|
236
|
-
break
|
237
|
-
end
|
238
|
-
@child_pids.push child_pid
|
239
|
-
end
|
240
|
-
defined?(ActiveRecord::Base) and
|
241
|
-
ActiveRecord::Base.establish_connection
|
242
|
-
EventMachine.run do
|
243
|
-
|
244
|
-
Signal.trap('INT') do
|
245
|
-
self.shutdown('INT')
|
246
|
-
end
|
247
|
-
Signal.trap('TERM') do
|
248
|
-
self.shutdown('TERM')
|
249
|
-
end
|
250
|
-
|
251
|
-
self.handle_connection
|
252
|
-
@log.info("Connecting to AMQP broker. Running #{AMQP::VERSION}")
|
253
|
-
|
254
|
-
#send a heartbeat every 15 seconds to avoid aggresive network configurations that close quiet connections
|
255
|
-
heartbeat_exchange = self.channel.fanout('coney_island_heartbeat')
|
256
|
-
EventMachine.add_periodic_timer(15) do
|
257
|
-
heartbeat_exchange.publish({:instance_name => @ticket})
|
258
|
-
end
|
259
|
-
|
260
|
-
self.channel.prefetch @prefetch_count
|
261
|
-
@queue = self.channel.queue(@full_instance_name, auto_delete: false, durable: true)
|
262
|
-
@queue.bind(self.exchange, routing_key: 'carousels.' + @ticket)
|
263
|
-
@queue.subscribe(:ack => true) do |metadata,payload|
|
264
|
-
begin
|
265
|
-
job_id = SecureRandom.uuid
|
266
|
-
@job_attempts[job_id] = 1
|
267
|
-
args = JSON.parse(payload)
|
268
|
-
@log.info ("Starting job #{job_id}: #{args}")
|
269
|
-
if args.has_key? 'delay'
|
270
|
-
EventMachine.add_timer(args['delay'].to_i) do
|
271
|
-
self.handle_job(metadata,args,job_id)
|
272
|
-
end
|
273
|
-
else
|
274
|
-
self.handle_job(metadata,args,job_id)
|
275
|
-
end
|
276
|
-
rescue Timeout::Error => e
|
277
|
-
self.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args, reason: 'timeout in subscribe code before calling job method'})
|
278
|
-
rescue Exception => e
|
279
|
-
self.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args})
|
280
|
-
@log.error("ConeyIsland code error, not application code:\n#{e.inspect}\nARGS: #{args}")
|
281
|
-
end
|
282
|
-
end
|
283
|
-
end
|
73
|
+
def self.run_with_em(klass, method, *args)
|
74
|
+
ConeyIsland::Submitter.run_with_em(klass, method, *args)
|
284
75
|
end
|
285
76
|
|
286
|
-
def self.
|
287
|
-
|
288
|
-
Timeout::timeout(3) do
|
289
|
-
@notifier.notify(message, context)
|
290
|
-
end
|
291
|
-
rescue
|
292
|
-
if attempts <= 3
|
293
|
-
attempts += 1
|
294
|
-
self.poke_the_badger(message, context, attempts)
|
295
|
-
end
|
296
|
-
end
|
77
|
+
def self.submit(*args)
|
78
|
+
ConeyIsland::Submitter.submit(*args)
|
297
79
|
end
|
298
80
|
end
|
81
|
+
|
299
82
|
require 'coney_island/notifiers/honeybadger_notifier'
|
83
|
+
require 'coney_island/worker'
|
84
|
+
require 'coney_island/submitter'
|
300
85
|
|
@@ -0,0 +1,92 @@
|
|
1
|
+
module ConeyIsland
|
2
|
+
class Submitter
|
3
|
+
|
4
|
+
def self.run_inline
|
5
|
+
@run_inline = true
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.stop_running_inline
|
9
|
+
@run_inline = false
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.submit(*args)
|
13
|
+
if RequestStore.store[:cache_jobs]
|
14
|
+
RequestStore.store[:jobs].push args
|
15
|
+
else
|
16
|
+
self.submit!(args)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.submit!(args)
|
21
|
+
if @run_inline
|
22
|
+
self.handle_publish(args)
|
23
|
+
else
|
24
|
+
EventMachine.next_tick do
|
25
|
+
self.handle_publish(args)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.handle_publish(args)
|
31
|
+
ConeyIsland.handle_connection unless @run_inline
|
32
|
+
jobs = (args.first.is_a? Array) ? args : [args]
|
33
|
+
jobs.each do |args|
|
34
|
+
if (args.first.is_a? Class or args.first.is_a? Module) and (args[1].is_a? String or args[1].is_a? Symbol) and args.last.is_a? Hash and 3 == args.length
|
35
|
+
klass = args.shift
|
36
|
+
klass = klass.name
|
37
|
+
method_name = args.shift
|
38
|
+
job_args = args.shift
|
39
|
+
job_args ||= {}
|
40
|
+
job_args['klass'] = klass
|
41
|
+
job_args['method_name'] = method_name
|
42
|
+
if @run_inline
|
43
|
+
job_args.stringify_keys!
|
44
|
+
ConeyIsland::Worker.execute_job_method(job_args)
|
45
|
+
else
|
46
|
+
work_queue = job_args.delete :work_queue
|
47
|
+
work_queue ||= 'default'
|
48
|
+
ConeyIsland.exchange.publish((job_args.to_json), routing_key: "carousels.#{work_queue}")
|
49
|
+
end
|
50
|
+
end
|
51
|
+
RequestStore.store[:completed_jobs] ||= 0
|
52
|
+
RequestStore.store[:completed_jobs] += 1
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.cache_jobs
|
57
|
+
RequestStore.store[:cache_jobs] = true
|
58
|
+
RequestStore.store[:jobs] = []
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.flush_jobs
|
62
|
+
jobs = RequestStore.store[:jobs].dup
|
63
|
+
self.submit!(jobs) if jobs.any?
|
64
|
+
RequestStore.store[:jobs] = []
|
65
|
+
end
|
66
|
+
|
67
|
+
def self.stop_caching_jobs
|
68
|
+
RequestStore.store[:cache_jobs] = false
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.run_with_em(klass, method, *args)
|
72
|
+
EventMachine.run do
|
73
|
+
ConeyIsland.cache_jobs
|
74
|
+
klass.send(method, *args)
|
75
|
+
ConeyIsland.flush_jobs
|
76
|
+
ConeyIsland.publisher_shutdown
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def self.publisher_shutdown
|
81
|
+
EventMachine.add_periodic_timer(1) do
|
82
|
+
if RequestStore.store[:jobs] && (RequestStore.store[:jobs].length > RequestStore.store[:completed_jobs])
|
83
|
+
Rails.logger.info("Waiting for #{RequestStore.store[:jobs].length - RequestStore.store[:completed_jobs]} publishes to finish")
|
84
|
+
else
|
85
|
+
Rails.logger.info("Shutting down coney island publisher")
|
86
|
+
EventMachine.stop
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
data/lib/coney_island/version.rb
CHANGED
@@ -0,0 +1,195 @@
|
|
1
|
+
module ConeyIsland
|
2
|
+
class Worker
|
3
|
+
|
4
|
+
def self.config=(config_hash)
|
5
|
+
@config = config_hash.symbolize_keys!
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.config
|
9
|
+
@config
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.log
|
13
|
+
@log ||= Logger.new(File.open(File::NULL, "w"))
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.log=(log_thing)
|
17
|
+
@log = log_thing
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.job_attempts
|
21
|
+
@job_attempts ||= {}
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.initialize_background
|
25
|
+
ENV['NEW_RELIC_AGENT_ENABLED'] = 'false'
|
26
|
+
ENV['NEWRELIC_ENABLE'] = 'false'
|
27
|
+
@ticket = ARGV[0]
|
28
|
+
@ticket ||= 'default'
|
29
|
+
|
30
|
+
@log_io = self.config[:log]
|
31
|
+
self.log = Logger.new(@log_io)
|
32
|
+
|
33
|
+
@instance_config = self.config[:carousels][@ticket]
|
34
|
+
|
35
|
+
@prefetch_count = @instance_config[:prefetch_count] if @instance_config
|
36
|
+
@prefetch_count ||= 20
|
37
|
+
|
38
|
+
@worker_count = @instance_config[:worker_count] if @instance_config
|
39
|
+
@worker_count ||= 1
|
40
|
+
@child_count = @worker_count - 1
|
41
|
+
@child_pids = []
|
42
|
+
|
43
|
+
@full_instance_name = @ticket
|
44
|
+
|
45
|
+
self.log.level = self.config[:log_level]
|
46
|
+
self.log.info("config: #{self.config}")
|
47
|
+
|
48
|
+
@notifier = "ConeyIsland::Notifiers::#{self.config[:notifier_service]}Notifier".constantize
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.start
|
52
|
+
@child_count.times do
|
53
|
+
child_pid = Process.fork
|
54
|
+
unless child_pid
|
55
|
+
self.log.info("started child for ticket #{@ticket} with pid #{Process.pid}")
|
56
|
+
break
|
57
|
+
end
|
58
|
+
@child_pids.push child_pid
|
59
|
+
end
|
60
|
+
defined?(ActiveRecord::Base) and
|
61
|
+
ActiveRecord::Base.establish_connection
|
62
|
+
EventMachine.run do
|
63
|
+
|
64
|
+
Signal.trap('INT') do
|
65
|
+
self.shutdown('INT')
|
66
|
+
end
|
67
|
+
Signal.trap('TERM') do
|
68
|
+
self.shutdown('TERM')
|
69
|
+
end
|
70
|
+
|
71
|
+
ConeyIsland.handle_connection
|
72
|
+
|
73
|
+
self.log.info("Connecting to AMQP broker. Running #{AMQP::VERSION}")
|
74
|
+
|
75
|
+
#send a heartbeat every 15 seconds to avoid aggresive network configurations that close quiet connections
|
76
|
+
heartbeat_exchange = ConeyIsland.channel.fanout('coney_island_heartbeat')
|
77
|
+
EventMachine.add_periodic_timer(15) do
|
78
|
+
heartbeat_exchange.publish({:instance_name => @ticket})
|
79
|
+
end
|
80
|
+
|
81
|
+
ConeyIsland.channel.prefetch @prefetch_count
|
82
|
+
@queue = ConeyIsland.channel.queue(@full_instance_name, auto_delete: false, durable: true)
|
83
|
+
@queue.bind(ConeyIsland.exchange, routing_key: 'carousels.' + @ticket)
|
84
|
+
@queue.subscribe(:ack => true) do |metadata,payload|
|
85
|
+
self.handle_incoming_message(metadata,payload)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.handle_incoming_message(metadata,payload)
|
91
|
+
begin
|
92
|
+
job_id = SecureRandom.uuid
|
93
|
+
self.job_attempts[job_id] = 1
|
94
|
+
args = JSON.parse(payload)
|
95
|
+
self.log.info ("Starting job #{job_id}: #{args}")
|
96
|
+
if args.has_key? 'delay'
|
97
|
+
EventMachine.add_timer(args['delay'].to_i) do
|
98
|
+
self.handle_job(metadata,args,job_id)
|
99
|
+
end
|
100
|
+
else
|
101
|
+
self.handle_job(metadata,args,job_id)
|
102
|
+
end
|
103
|
+
rescue Timeout::Error => e
|
104
|
+
self.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args, reason: 'timeout in subscribe code before calling job method'})
|
105
|
+
rescue Exception => e
|
106
|
+
self.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args})
|
107
|
+
self.log.error("ConeyIsland code error, not application code:\n#{e.inspect}\nARGS: #{args}")
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.handle_job(metadata,args,job_id)
|
112
|
+
timeout = args['timeout']
|
113
|
+
timeout ||= BG_TIMEOUT_SECONDS
|
114
|
+
begin
|
115
|
+
Timeout::timeout(timeout) do
|
116
|
+
self.execute_job_method(args)
|
117
|
+
end
|
118
|
+
rescue Timeout::Error => e
|
119
|
+
if self.job_attempts.has_key? job_id
|
120
|
+
if self.job_attempts[job_id] >= 3
|
121
|
+
self.log.error("Request #{job_id} timed out after #{timeout} seconds, bailing out after 3 attempts")
|
122
|
+
self.finalize_job(metadata,job_id)
|
123
|
+
self.poke_the_badger(e, {work_queue: @ticket, job_payload: args, reason: 'Bailed out after 3 attempts'})
|
124
|
+
else
|
125
|
+
self.log.error("Request #{job_id} timed out after #{timeout} seconds on attempt number #{self.job_attempts[job_id]}, retrying...")
|
126
|
+
self.job_attempts[job_id] += 1
|
127
|
+
self.handle_job(metadata,args,job_id)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
rescue Exception => e
|
131
|
+
self.poke_the_badger(e, {work_queue: @ticket, job_payload: args})
|
132
|
+
self.log.error("Error executing #{args['klass']}##{args['method_name']} #{job_id} for id #{args['instance_id']} with args #{args}:")
|
133
|
+
self.log.error(e.message)
|
134
|
+
self.log.error(e.backtrace.join("\n"))
|
135
|
+
self.finalize_job(metadata,job_id)
|
136
|
+
else
|
137
|
+
self.finalize_job(metadata,job_id)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def self.execute_job_method(args)
|
142
|
+
class_name = args['klass']
|
143
|
+
method_name = args['method_name']
|
144
|
+
klass = class_name.constantize
|
145
|
+
method_args = args['args']
|
146
|
+
if args.has_key? 'instance_id'
|
147
|
+
instance_id = args['instance_id']
|
148
|
+
object = klass.find(instance_id)
|
149
|
+
else
|
150
|
+
object = klass
|
151
|
+
end
|
152
|
+
if method_args and method_args.length > 0
|
153
|
+
object.send method_name, *method_args
|
154
|
+
else
|
155
|
+
object.send method_name
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def self.finalize_job(metadata,job_id)
|
160
|
+
metadata.ack
|
161
|
+
self.log.info("finished job #{job_id}")
|
162
|
+
self.job_attempts.delete job_id
|
163
|
+
end
|
164
|
+
|
165
|
+
def self.poke_the_badger(message, context, attempts = 1)
|
166
|
+
begin
|
167
|
+
Timeout::timeout(3) do
|
168
|
+
@notifier.notify(message, context)
|
169
|
+
end
|
170
|
+
rescue
|
171
|
+
if attempts <= 3
|
172
|
+
attempts += 1
|
173
|
+
self.poke_the_badger(message, context, attempts)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def self.shutdown(signal)
|
179
|
+
shutdown_time = Time.now
|
180
|
+
@child_pids.each do |child_pid|
|
181
|
+
Process.kill(signal, child_pid)
|
182
|
+
end
|
183
|
+
@queue.unsubscribe
|
184
|
+
EventMachine.add_periodic_timer(1) do
|
185
|
+
if self.job_attempts.any?
|
186
|
+
self.log.info("Waiting for #{self.job_attempts.length} requests to finish")
|
187
|
+
else
|
188
|
+
self.log.info("Shutting down coney island #{@ticket}")
|
189
|
+
EventMachine.stop
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
end
|
data/test/coney_island_test.rb
CHANGED
@@ -1,7 +1,23 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
|
3
|
-
class ConeyIslandTest <
|
4
|
-
|
5
|
-
|
3
|
+
class ConeyIslandTest < MiniTest::Test
|
4
|
+
describe "running jobs" do
|
5
|
+
it "runs inline" do
|
6
|
+
ConeyIsland.run_inline
|
7
|
+
my_array = []
|
8
|
+
ConeyIsland.submit(TestModel, :add_to_list, args: [my_array])
|
9
|
+
my_array.first.must_equal 'Added one!'
|
10
|
+
end
|
11
|
+
it "caches jobs" do
|
12
|
+
ConeyIsland.run_inline
|
13
|
+
my_array = []
|
14
|
+
ConeyIsland.cache_jobs
|
15
|
+
ConeyIsland.submit(TestModel, :add_to_list, args: [my_array])
|
16
|
+
RequestStore.store[:jobs].length.must_equal 1
|
17
|
+
my_array.length.must_equal 0
|
18
|
+
ConeyIsland.flush_jobs
|
19
|
+
my_array.first.must_equal 'Added one!'
|
20
|
+
ConeyIsland.stop_caching_jobs
|
21
|
+
end
|
6
22
|
end
|
7
23
|
end
|
File without changes
|
File without changes
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class SubmitterTest < MiniTest::Test
|
4
|
+
describe "running jobs inline" do
|
5
|
+
it "calls the worker directly" do
|
6
|
+
@execute_job_method = Minitest::Mock.new
|
7
|
+
@execute_job_method.expect :call, nil, [Hash]
|
8
|
+
ConeyIsland::Worker.stub(:execute_job_method,@execute_job_method) do
|
9
|
+
ConeyIsland::Submitter.run_inline
|
10
|
+
ConeyIsland::Submitter.submit(TestModel, :add_to_list, args: [[]])
|
11
|
+
end
|
12
|
+
@execute_job_method.verify
|
13
|
+
end
|
14
|
+
end
|
15
|
+
describe "running jobs in the background" do
|
16
|
+
it "publishes the job to the message bus" do
|
17
|
+
@exchange = Minitest::Mock.new
|
18
|
+
@exchange.expect :publish, nil, [String,Hash]
|
19
|
+
@next_tick = Minitest::Mock.new
|
20
|
+
@next_tick.expect :call, nil, []
|
21
|
+
EventMachine.stub(:next_tick, @next_tick) do
|
22
|
+
ConeyIsland.stub(:handle_connection, nil) do
|
23
|
+
ConeyIsland.stub(:exchange, @exchange) do
|
24
|
+
ConeyIsland::Submitter.stop_running_inline
|
25
|
+
ConeyIsland::Submitter.submit(TestModel, :add_to_list, args: [[]])
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
@next_tick.verify
|
30
|
+
@exchange.verify
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/test/test_helper.rb
CHANGED
@@ -3,9 +3,13 @@ ENV["RAILS_ENV"] = "test"
|
|
3
3
|
|
4
4
|
require File.expand_path("../dummy/config/environment.rb", __FILE__)
|
5
5
|
require "rails/test_help"
|
6
|
+
require 'minitest/unit'
|
7
|
+
require 'minitest/autorun'
|
8
|
+
require 'minitest/pride'
|
9
|
+
require 'request_store'
|
10
|
+
require 'amqp'
|
6
11
|
|
7
12
|
Rails.backtrace_cleaner.remove_silencers!
|
8
|
-
|
9
13
|
# Load support files
|
10
14
|
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
|
11
15
|
|
@@ -13,3 +17,38 @@ Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
|
|
13
17
|
if ActiveSupport::TestCase.method_defined?(:fixture_path=)
|
14
18
|
ActiveSupport::TestCase.fixture_path = File.expand_path("../fixtures", __FILE__)
|
15
19
|
end
|
20
|
+
|
21
|
+
class TestModel
|
22
|
+
|
23
|
+
def self.add_to_list(array)
|
24
|
+
array.push "Added one!"
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.take_too_long
|
28
|
+
sleep(10)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.throw_an_error
|
32
|
+
raise 'It broke!'
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.instances
|
36
|
+
@instances ||= {}
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.find(id)
|
40
|
+
self.instances[id]
|
41
|
+
end
|
42
|
+
|
43
|
+
def initialize(params)
|
44
|
+
TestModel.instances[params['id']] = self
|
45
|
+
end
|
46
|
+
|
47
|
+
def set_color(color)
|
48
|
+
@color = (color)
|
49
|
+
end
|
50
|
+
|
51
|
+
def color
|
52
|
+
@color
|
53
|
+
end
|
54
|
+
end
|
data/test/worker_test.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class WorkerTest < MiniTest::Test
|
4
|
+
describe "handling jobs" do
|
5
|
+
before do
|
6
|
+
@metadata = MiniTest::Mock.new
|
7
|
+
@metadata.expect :ack, nil
|
8
|
+
end
|
9
|
+
it "handles timeouts with 3 retries before bailing out" do
|
10
|
+
ConeyIsland::Worker.job_attempts['my_job_id'] = 1
|
11
|
+
ConeyIsland::Worker.job_attempts.stub(:delete,nil) do
|
12
|
+
ConeyIsland::Worker.handle_job(@metadata,{
|
13
|
+
'klass' => 'TestModel',
|
14
|
+
'method_name' => :take_too_long,
|
15
|
+
'timeout' => 0.01
|
16
|
+
},'my_job_id')
|
17
|
+
end
|
18
|
+
ConeyIsland::Worker.job_attempts['my_job_id'].must_equal 3
|
19
|
+
end
|
20
|
+
it "sends other exeptions to a notification service" do
|
21
|
+
@poke_the_badger = MiniTest::Mock.new
|
22
|
+
@poke_the_badger.expect :call, nil, [Exception,Hash]
|
23
|
+
ConeyIsland::Worker.stub(:poke_the_badger,@poke_the_badger) do
|
24
|
+
ConeyIsland::Worker.handle_job(@metadata,{
|
25
|
+
'klass' => 'TestModel',
|
26
|
+
'method_name' => :throw_an_error
|
27
|
+
},'my_job_id')
|
28
|
+
end
|
29
|
+
@poke_the_badger.verify
|
30
|
+
end
|
31
|
+
it "calls find on the submitted class if an instance_id is present" do
|
32
|
+
TestModel.new('id' => 'my_id')
|
33
|
+
ConeyIsland::Worker.handle_job(@metadata,{
|
34
|
+
'klass' => 'TestModel',
|
35
|
+
'method_name' => :set_color,
|
36
|
+
'instance_id' => 'my_id',
|
37
|
+
'args' => ['green']
|
38
|
+
},'my_job_id')
|
39
|
+
my_thing = TestModel.find('my_id')
|
40
|
+
my_thing.color.must_equal 'green'
|
41
|
+
end
|
42
|
+
it "acknowledges job completion to the message bus" do
|
43
|
+
ConeyIsland::Worker.handle_job(@metadata,
|
44
|
+
{ 'klass' => 'TestModel',
|
45
|
+
'method_name' => :add_to_list,
|
46
|
+
'args' => [[]]},
|
47
|
+
'my_job_id')
|
48
|
+
@metadata.verify
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: coney_island
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Eric Draut
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-09-
|
12
|
+
date: 2014-09-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -81,6 +81,20 @@ dependencies:
|
|
81
81
|
- - ">="
|
82
82
|
- !ruby/object:Gem::Version
|
83
83
|
version: '0'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: minitest
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '0'
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ">="
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0'
|
84
98
|
description: An industrial-strength background worker system for rails using RabbitMQ.
|
85
99
|
email:
|
86
100
|
- edraut@gmail.com
|
@@ -96,7 +110,9 @@ files:
|
|
96
110
|
- lib/coney_island.rb
|
97
111
|
- lib/coney_island/notifiers/airbrake_notifier.rb
|
98
112
|
- lib/coney_island/notifiers/honeybadger_notifier.rb
|
113
|
+
- lib/coney_island/submitter.rb
|
99
114
|
- lib/coney_island/version.rb
|
115
|
+
- lib/coney_island/worker.rb
|
100
116
|
- lib/tasks/coney_island_tasks.rake
|
101
117
|
- test/coney_island_test.rb
|
102
118
|
- test/dummy/README.rdoc
|
@@ -128,11 +144,15 @@ files:
|
|
128
144
|
- test/dummy/config/locales/en.yml
|
129
145
|
- test/dummy/config/routes.rb
|
130
146
|
- test/dummy/config/secrets.yml
|
147
|
+
- test/dummy/db/test.sqlite3
|
148
|
+
- test/dummy/log/test.log
|
131
149
|
- test/dummy/public/404.html
|
132
150
|
- test/dummy/public/422.html
|
133
151
|
- test/dummy/public/500.html
|
134
152
|
- test/dummy/public/favicon.ico
|
153
|
+
- test/submitter_test.rb
|
135
154
|
- test/test_helper.rb
|
155
|
+
- test/worker_test.rb
|
136
156
|
homepage: https://github.com/edraut/coney_island
|
137
157
|
licenses:
|
138
158
|
- MIT
|
@@ -157,8 +177,9 @@ rubygems_version: 2.2.2
|
|
157
177
|
signing_key:
|
158
178
|
specification_version: 4
|
159
179
|
summary: Want guaranteed delivery between your queue and your workers using ACKs?
|
160
|
-
How about load-balancing?
|
161
|
-
offer and you must have a ticket to ride
|
180
|
+
How about load-balancing? Would job-specific timeouts be nice? Throw in all the
|
181
|
+
features other background worker systems offer and you must have a ticket to ride
|
182
|
+
at Coney Island.
|
162
183
|
test_files:
|
163
184
|
- test/coney_island_test.rb
|
164
185
|
- test/dummy/app/assets/javascripts/application.js
|
@@ -188,10 +209,14 @@ test_files:
|
|
188
209
|
- test/dummy/config/routes.rb
|
189
210
|
- test/dummy/config/secrets.yml
|
190
211
|
- test/dummy/config.ru
|
212
|
+
- test/dummy/db/test.sqlite3
|
213
|
+
- test/dummy/log/test.log
|
191
214
|
- test/dummy/public/404.html
|
192
215
|
- test/dummy/public/422.html
|
193
216
|
- test/dummy/public/500.html
|
194
217
|
- test/dummy/public/favicon.ico
|
195
218
|
- test/dummy/Rakefile
|
196
219
|
- test/dummy/README.rdoc
|
220
|
+
- test/submitter_test.rb
|
197
221
|
- test/test_helper.rb
|
222
|
+
- test/worker_test.rb
|