howler 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. data/.gemrc +1 -0
  2. data/.gitignore +5 -0
  3. data/.rspec +2 -0
  4. data/.rvmrc +1 -0
  5. data/Gemfile +2 -0
  6. data/Gemfile.lock +68 -0
  7. data/LICENSE +19 -0
  8. data/README.md +104 -0
  9. data/bin/howler +3 -0
  10. data/howler.gemspec +24 -0
  11. data/lib/howler.rb +34 -0
  12. data/lib/howler/async.rb +15 -0
  13. data/lib/howler/config.ru +4 -0
  14. data/lib/howler/exceptions.rb +5 -0
  15. data/lib/howler/exceptions/error.rb +25 -0
  16. data/lib/howler/exceptions/failed.rb +9 -0
  17. data/lib/howler/exceptions/notify.rb +15 -0
  18. data/lib/howler/exceptions/retry.rb +12 -0
  19. data/lib/howler/manager.rb +123 -0
  20. data/lib/howler/message.rb +15 -0
  21. data/lib/howler/queue.rb +138 -0
  22. data/lib/howler/runner.rb +34 -0
  23. data/lib/howler/support/config.rb +33 -0
  24. data/lib/howler/support/logger.rb +57 -0
  25. data/lib/howler/support/util.rb +23 -0
  26. data/lib/howler/support/version.rb +3 -0
  27. data/lib/howler/web.rb +47 -0
  28. data/lib/howler/web/public/application.css +24 -0
  29. data/lib/howler/web/public/bootstrap.css +3990 -0
  30. data/lib/howler/web/public/bootstrap.min.css +689 -0
  31. data/lib/howler/web/public/queues.css +19 -0
  32. data/lib/howler/web/views/failed_messages.erb +27 -0
  33. data/lib/howler/web/views/html.erb +10 -0
  34. data/lib/howler/web/views/index.erb +11 -0
  35. data/lib/howler/web/views/navigation.erb +25 -0
  36. data/lib/howler/web/views/notification_messages.erb +24 -0
  37. data/lib/howler/web/views/notifications.erb +15 -0
  38. data/lib/howler/web/views/pending_messages.erb +24 -0
  39. data/lib/howler/web/views/processed_messages.erb +28 -0
  40. data/lib/howler/web/views/queue.erb +36 -0
  41. data/lib/howler/web/views/queue_table.erb +27 -0
  42. data/lib/howler/web/views/queues.erb +15 -0
  43. data/lib/howler/worker.rb +17 -0
  44. data/spec/models/async_spec.rb +76 -0
  45. data/spec/models/exceptions/failed_spec.rb +15 -0
  46. data/spec/models/exceptions/message_spec.rb +53 -0
  47. data/spec/models/exceptions/notify_spec.rb +26 -0
  48. data/spec/models/exceptions/retry_spec.rb +49 -0
  49. data/spec/models/howler_spec.rb +69 -0
  50. data/spec/models/manager_spec.rb +397 -0
  51. data/spec/models/message_spec.rb +78 -0
  52. data/spec/models/queue_spec.rb +539 -0
  53. data/spec/models/runner_spec.rb +109 -0
  54. data/spec/models/support/config_spec.rb +56 -0
  55. data/spec/models/support/logger_spec.rb +147 -0
  56. data/spec/models/support/util_spec.rb +44 -0
  57. data/spec/models/worker_spec.rb +54 -0
  58. data/spec/requests/web_spec.rb +220 -0
  59. data/spec/spec_helper.rb +93 -0
  60. metadata +265 -0
@@ -0,0 +1,78 @@
1
+ require "spec_helper"
2
+
3
+ describe Howler::Message do
4
+ subject do
5
+ Howler::Message.new(
6
+ "id" => 123,
7
+ "class" => "Array",
8
+ "method" => "length",
9
+ "args" => [1234]
10
+ )
11
+ end
12
+
13
+ describe ".new" do
14
+ describe "requirements" do
15
+ it "should require a class" do
16
+ expect {
17
+ Howler::Message.new("method" => 'hey')
18
+ }.to raise_error(NoMethodError)
19
+ end
20
+
21
+ it "should require a valid class" do
22
+ expect {
23
+ Howler::Message.new("class" => 'a', "method" => 'hey')
24
+ }.to raise_error(NameError)
25
+ end
26
+
27
+ it "should require a method" do
28
+ expect {
29
+ Howler::Message.new("class" => 'Array')
30
+ }.to raise_error(ArgumentError, "A message requires a method")
31
+ end
32
+ end
33
+ end
34
+
35
+ describe "#klass" do
36
+ it "should return the class literal" do
37
+ subject.id.should == 123
38
+ end
39
+ end
40
+
41
+ describe "#klass" do
42
+ it "should return the class literal" do
43
+ subject.klass.should == Array
44
+ end
45
+ end
46
+
47
+ describe "#method" do
48
+ it "should return the synbolized method" do
49
+ subject.method.should == :length
50
+ end
51
+ end
52
+
53
+ describe "#args" do
54
+ it "should return the arguments" do
55
+ subject.args.should == [1234]
56
+ end
57
+ end
58
+
59
+ describe "#created_at" do
60
+ describe "when initialized" do
61
+ it "should be the initialization time" do
62
+ Timecop.freeze(DateTime.now) do
63
+ subject.created_at.should == Time.now.to_f
64
+ end
65
+ end
66
+ end
67
+
68
+ describe "when given the created time" do
69
+ subject { Howler::Message.new('created_at' => Time.now - 5.minutes, 'class' => 'Howler', 'method' => '') }
70
+
71
+ it "should be the given time" do
72
+ Timecop.freeze(DateTime.now) do
73
+ subject.created_at.should == Time.now - 5.minutes
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,539 @@
1
+ require "spec_helper"
2
+
3
+ describe Howler::Queue do
4
+ it "should identify the list of queues" do
5
+ Howler::Queue::INDEX.should == "queues"
6
+ end
7
+
8
+ it "should default to 'default'" do
9
+ Howler::Queue::DEFAULT.should == "default"
10
+ end
11
+
12
+ describe ".new" do
13
+ it "should be the default queue" do
14
+ Howler::Queue.new.name.should == "queues:default"
15
+ end
16
+
17
+ it "should add itself to the list of queues" do
18
+ Howler::Queue.new("queue_name")
19
+
20
+ Howler.redis.with {|redis| redis.smembers(Howler::Queue::INDEX).include?("queue_name").should be_true }
21
+ end
22
+
23
+ it "should have a logger" do
24
+ Howler::Logger.should_receive(:new)
25
+
26
+ Howler::Queue.new
27
+ end
28
+ end
29
+
30
+ describe "#id" do
31
+ it "should default to `default`" do
32
+ Howler::Queue.new.id.should == "default"
33
+ end
34
+
35
+ it "should be the given queue identifier" do
36
+ Howler::Queue.new("queue_name").id.should == "queue_name"
37
+ end
38
+ end
39
+
40
+ describe "#name" do
41
+ it "should be the namespaced queue name that is passed in" do
42
+ Howler::Queue.new("queue_name").name.should == "queues:queue_name"
43
+ end
44
+ end
45
+
46
+ describe "#created_at" do
47
+ it "should return the created time" do
48
+ Timecop.freeze(DateTime.now) do
49
+ subject.created_at.should == Time.now
50
+ end
51
+ end
52
+ end
53
+
54
+ describe "#push" do
55
+ let!(:message) { mock(Howler::Message) }
56
+ let!(:encoded_message) { mock("JSON:Howler::Message") }
57
+
58
+ before do
59
+ MultiJson.stub(:encode).and_return(encoded_message)
60
+ end
61
+
62
+ it "should JSON encode the message" do
63
+ MultiJson.should_receive(:encode).with(message)
64
+
65
+ subject.push(message)
66
+ end
67
+
68
+ it "should push the message into redis" do
69
+ Timecop.freeze(DateTime.now) do
70
+ Howler.send(:_redis).should_receive(:zadd).with(Howler::Manager::DEFAULT, Time.now.to_f, encoded_message)
71
+
72
+ subject.push(message)
73
+ end
74
+ end
75
+
76
+ it "should return true" do
77
+ Howler.send(:_redis).stub(:zadd).and_return(1)
78
+
79
+ subject.push(message).should == true
80
+ end
81
+
82
+ describe "when given a time" do
83
+ it "should add it with the specified time" do
84
+ Timecop.freeze(DateTime.now) do
85
+ Howler.send(:_redis).should_receive(:zadd).with(Howler::Manager::DEFAULT, (Time.now + 5.minutes).to_f, encoded_message)
86
+
87
+ subject.push(message, (Time.now + 5.minutes).to_f)
88
+ end
89
+ end
90
+ end
91
+
92
+ describe "when the message cannot be pushed" do
93
+ it "should return false" do
94
+ Howler.send(:_redis).stub(:zadd).and_return(0)
95
+
96
+ subject.push(message).should == false
97
+ end
98
+ end
99
+ end
100
+
101
+ describe "#immediate" do
102
+ let!(:worker) { mock(Howler::Worker) }
103
+ let(:message) { mock(Howler::Message, :klass => 1, :method => 2, :args => 3, :created_at => 4) }
104
+
105
+ it "should perform the method now" do
106
+ Howler::Worker.should_receive(:new).and_return(worker)
107
+ worker.should_receive(:perform).with(message, subject)
108
+
109
+ subject.immediate(message)
110
+ end
111
+ end
112
+
113
+ describe "#success" do
114
+ before do
115
+ 2.times { subject.statistics { lambda {} } }
116
+ end
117
+
118
+ describe "when the given block is successful" do
119
+ it "should return the number of messages processed" do
120
+ subject.success.should == 2
121
+ end
122
+ end
123
+ end
124
+
125
+ describe "#error" do
126
+ describe "when the given block raises an exception" do
127
+ before do
128
+ 3.times { subject.statistics { raise "failed" } }
129
+ end
130
+
131
+ it "should return the number of errors encountered" do
132
+ subject.error.should == 3
133
+ end
134
+ end
135
+ end
136
+
137
+ describe "statistics" do
138
+ let(:block) { lambda {} }
139
+ let!(:benchmark) { "0.1 0.3 0.5 ( 1.1)" }
140
+
141
+ it "should call the given block" do
142
+ block.should_receive(:call)
143
+
144
+ subject.statistics(&block)
145
+ end
146
+
147
+ it "should store metadata about each message" do
148
+ now = Time.now
149
+ Time.stub(:now).and_return(now)
150
+
151
+ Benchmark.stub(:measure).and_return(benchmark)
152
+
153
+ metadata = MultiJson.encode(
154
+ :class => "Array",
155
+ :method => "length",
156
+ :args => 1234,
157
+ :time => { :system => 0.5, :user => 1.1 },
158
+ :created_at => nil,
159
+ :status => 'success'
160
+ )
161
+
162
+ Howler.send(:_redis).should_receive(:zadd).with("#{subject.name}:messages", now.to_f, metadata)
163
+
164
+ subject.statistics(Array, :length, 1234, &block)
165
+ end
166
+
167
+ describe "when given the class" do
168
+ it "should increment the class' total in redis" do
169
+ Howler.send(:_redis).should_receive(:hincrby).with(subject.name, "Array", 1)
170
+
171
+ subject.statistics(Array, &block)
172
+ end
173
+ end
174
+
175
+ describe "when given the method name" do
176
+ before do
177
+ Howler.send(:_redis).stub(:hincrby)
178
+ end
179
+
180
+ it "should store the method in redis" do
181
+ Howler.send(:_redis).should_receive(:hincrby).with(subject.name, "Array:length", 1)
182
+
183
+ subject.statistics(Array, :length, &block)
184
+ end
185
+ end
186
+
187
+ describe "when given arguments" do
188
+ before do
189
+ Howler.send(:_redis).stub(:hincrby)
190
+ end
191
+
192
+ it "should store the method in redis" do
193
+ Howler.send(:_redis).should_receive(:hincrby).with(subject.name, "Array:length", 1)
194
+
195
+ subject.statistics(Array, :length, 1234, &block)
196
+ end
197
+ end
198
+
199
+ describe "when the block is successful" do
200
+ it "should update the success count" do
201
+ expect {
202
+ subject.statistics(&block)
203
+ }.to change(subject, :success).by(1)
204
+ end
205
+
206
+ it "should not update the error count" do
207
+ expect {
208
+ subject.statistics(&block)
209
+ }.not_to change(subject, :error)
210
+ end
211
+ end
212
+
213
+ describe "when the block encounters an error" do
214
+ before do
215
+ block.stub(:call).and_raise(Exception)
216
+ end
217
+
218
+ it "should not update the success count" do
219
+ expect {
220
+ subject.statistics(&block)
221
+ }.not_to change(subject, :success)
222
+ end
223
+
224
+ it "should update the error count" do
225
+ expect {
226
+ subject.statistics(&block)
227
+ }.to change(subject, :error).by(1)
228
+ end
229
+ end
230
+
231
+ describe "when the block sends a notification" do
232
+ let(:block) { lambda { raise Howler::Message::Notify.new(generate_exception) }}
233
+
234
+ it "should not error" do
235
+ expect {
236
+ subject.statistics(&block)
237
+ }.not_to raise_error
238
+ end
239
+
240
+ it "should add the message to notifications" do
241
+ should_change("notifications").length_by(1) do
242
+ subject.statistics(&block)
243
+ end
244
+ end
245
+
246
+ it "should have a status of notified" do
247
+ subject.statistics(&block)
248
+
249
+ Howler::Queue.notifications.first['status'].should == 'notified'
250
+ end
251
+ end
252
+
253
+ describe "when there are messages to be processed" do
254
+ let!(:benchmark) { "0.1 0.2 0.3 ( 1.1)" }
255
+ let(:manager) { Howler::Manager.current }
256
+ before do
257
+ manager.push(Array, :length, nil)
258
+ manager.push(Hash, :keys, nil)
259
+ end
260
+
261
+ it "should return a list of pending messages" do
262
+ subject.pending_messages.collect {|p| p['class']}.should == %w(Array Hash)
263
+ subject.pending_messages.collect {|p| p['method']}.should == %w(length keys)
264
+ end
265
+
266
+ it "should update when more messages are pushed" do
267
+ manager.push(Thread, :current, nil)
268
+
269
+ subject.pending_messages.collect {|p| p['class']}.should == %w(Array Hash Thread)
270
+ subject.pending_messages.collect {|p| p['method']}.should == %w(length keys current)
271
+ end
272
+ end
273
+
274
+ describe "when messages have been processed" do
275
+ let!(:benchmark) { "0.1 0.2 0.3 ( 1.1)" }
276
+
277
+ before do
278
+ Benchmark.stub(:measure).and_return(benchmark)
279
+ end
280
+
281
+ it "should store metadata" do
282
+ Howler.send(:_redis).should_receive(:zadd).with(subject.name + ":messages", anything, anything)
283
+
284
+ subject.statistics(&block)
285
+ end
286
+
287
+ it "should benchmark the runtime" do
288
+ Benchmark.should_receive(:measure)
289
+
290
+ subject.statistics(&block)
291
+ end
292
+
293
+ it "should have the message" do
294
+ subject.statistics(&block)
295
+
296
+ subject.should have(1).processed_messages
297
+ end
298
+
299
+ it "should be a message" do
300
+ subject.statistics(Array, :length, 1234, '10-10', &block)
301
+
302
+ subject.processed_messages.first.should == {
303
+ 'class' => 'Array',
304
+ 'method' => 'length',
305
+ 'args' => 1234,
306
+ 'time' => {'system' => 0.3, 'user' => 1.1},
307
+ 'status' => 'success',
308
+ 'created_at' => '10-10'
309
+ }
310
+ end
311
+
312
+ it "should include system time" do
313
+ subject.statistics(&block)
314
+
315
+ subject.processed_messages.first['time']['system'].should == 0.3
316
+ end
317
+
318
+ it "should include user time" do
319
+ subject.statistics(&block)
320
+
321
+ subject.processed_messages.first['time']['user'].should == 1.1
322
+ end
323
+
324
+ describe "when a message fails" do
325
+ before do
326
+ Benchmark.unstub(:measure)
327
+
328
+ subject.statistics { raise Howler::Message::Failed }
329
+ end
330
+
331
+ it "should add the messages to the :failed list" do
332
+ subject.should have(1).failed_messages
333
+ end
334
+
335
+ it "should not add it to the processed messages list" do
336
+ subject.should have(0).processed_messages
337
+ end
338
+
339
+ it "should log error" do
340
+ subject.failed_messages.first['status'].should == 'failed'
341
+ end
342
+
343
+ it "should include the failure cause" do
344
+ subject.failed_messages.first['cause'].should == 'Howler::Message::Failed'
345
+ end
346
+
347
+ it "should include the failed at time" do
348
+ Timecop.freeze(DateTime.now) do
349
+ subject.statistics { raise Howler::Message::Failed }
350
+
351
+ subject.failed_messages.first['failed_at'].should == Time.now.utc.to_f
352
+ end
353
+ end
354
+ end
355
+
356
+ describe "status" do
357
+ describe "when the block is successful" do
358
+ it "should log success" do
359
+ subject.statistics(&block)
360
+
361
+ subject.processed_messages.first['status'].should == 'success'
362
+ end
363
+ end
364
+
365
+ describe "when the block fails" do
366
+ before do
367
+ Benchmark.stub(:measure).and_raise
368
+ end
369
+
370
+ it "should log error" do
371
+ subject.statistics(&block)
372
+
373
+ subject.processed_messages.first['status'].should == 'error'
374
+ end
375
+ end
376
+
377
+ describe "when the message should retry" do
378
+ before do
379
+ Benchmark.unstub(:measure)
380
+ end
381
+
382
+ it "should not add the message to the queue.name:messages list" do
383
+ Howler.send(:_redis).should_not_receive(:zadd).with(subject.name + ":messages", anything, anything)
384
+
385
+ subject.statistics { raise Howler::Message::Retry }
386
+ end
387
+ end
388
+ end
389
+ end
390
+
391
+ describe "logging" do
392
+ let!(:logger) { mock(Howler::Logger) }
393
+ let!(:log) { mock(Howler::Logger, :info => nil, :debug => nil) }
394
+ let!(:block) { lambda {} }
395
+
396
+ before do
397
+ subject.instance_variable_set(:@logger, logger)
398
+ logger.stub(:log).and_yield(log)
399
+ end
400
+
401
+ describe "information" do
402
+ before do
403
+ Howler::Config[:log] = 'info'
404
+ end
405
+
406
+ it "should log the number of messages to be processed" do
407
+ log.should_not_receive(:info)
408
+
409
+ subject.statistics(&block)
410
+ end
411
+ end
412
+
413
+ describe "debug" do
414
+ before do
415
+ Howler::Config[:log] = 'debug'
416
+ end
417
+
418
+ describe "when the block Fails (explicitly)" do
419
+ before do
420
+ block.stub(:call).and_raise(Howler::Message::Failed)
421
+ end
422
+
423
+ it "should log to debugging" do
424
+ log.should_receive(:debug).with("Howler::Message::Failed - Array.new.send(2)")
425
+
426
+ subject.statistics(Array, :send, [2], &block)
427
+ end
428
+
429
+ it "should not log information" do
430
+ log.should_not_receive(:info)
431
+
432
+ subject.statistics(Array, :send, [2], &block)
433
+ end
434
+ end
435
+
436
+ describe "when the block Retries" do
437
+ before do
438
+ block.stub(:call).and_raise(Howler::Message::Retry)
439
+ end
440
+
441
+ it "should log to debugging" do
442
+ log.should_receive(:debug).with("Howler::Message::Retry - Array.new.send(2)")
443
+
444
+ subject.statistics(Array, :send, [2], &block)
445
+ end
446
+
447
+ it "should not log information" do
448
+ log.should_not_receive(:info)
449
+
450
+ subject.statistics(Array, :send, [2], &block)
451
+ end
452
+ end
453
+
454
+ describe "when the block Notifies" do
455
+ let(:block) { lambda { raise Howler::Message::Notify.new(:after => 1.minute)} }
456
+
457
+ it "should log to debugging" do
458
+ log.should_receive(:debug).with("Howler::Message::Notify - Array.new.send(2)")
459
+
460
+ subject.statistics(Array, :send, [2], &block)
461
+ end
462
+
463
+ it "should not log information" do
464
+ log.should_not_receive(:info)
465
+
466
+ subject.statistics(Array, :send, [2], &block)
467
+ end
468
+ end
469
+
470
+ describe "when the block fails" do
471
+ before do
472
+ block.stub(:call).and_raise(Exception)
473
+ end
474
+
475
+ it "should log to debugging" do
476
+ log.should_receive(:debug).with("Exception - Array.new.send(2)")
477
+
478
+ subject.statistics(Array, :send, [2], &block)
479
+ end
480
+
481
+ it "should not log information" do
482
+ log.should_not_receive(:info)
483
+
484
+ subject.statistics(Array, :send, [2], &block)
485
+ end
486
+ end
487
+ end
488
+ end
489
+ end
490
+
491
+ describe "#requeue" do
492
+ let(:block) { lambda {} }
493
+
494
+ describe "the default behavior" do
495
+ before do
496
+ block.stub(:call).and_raise(Howler::Message::Retry)
497
+ end
498
+
499
+ it "should retry the message five minutes later" do
500
+ Timecop.freeze(DateTime.now) do
501
+ Howler.send(:_redis).should_receive(:zadd).with("pending:default", (Time.now + 5.minutes).to_f, anything)
502
+
503
+ subject.statistics(&block)
504
+ end
505
+ end
506
+ end
507
+
508
+ describe "when the worker encounters an retry-able error" do
509
+ before do
510
+ block.stub(:call).and_raise(Howler::Message::Retry)
511
+ Howler::Message::Retry.any_instance.stub(:at).and_return(60)
512
+ end
513
+
514
+ it "should retry the message at the specified time" do
515
+ Timecop.freeze(DateTime.now) do
516
+ Howler.send(:_redis).should_receive(:zadd).with("pending:default", 60.0, anything)
517
+
518
+ subject.statistics(&block)
519
+ end
520
+ end
521
+
522
+ describe "when the exception specifies a time to live" do
523
+ let!(:message_retry) { Howler::Message::Retry.new(:ttl => -1.minutes) }
524
+
525
+ before do
526
+ block.stub(:call).and_raise(message_retry)
527
+ end
528
+
529
+ it "should retry the message until it reaches the the ttl" do
530
+ Timecop.freeze(DateTime.now) do
531
+ Howler.send(:_redis).should_not_receive(:zadd).with("pending:default", anything, anything)
532
+
533
+ subject.statistics(&block)
534
+ end
535
+ end
536
+ end
537
+ end
538
+ end
539
+ end