qs 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. data/bench/config.qs +4 -27
  2. data/bench/dispatcher.qs +24 -0
  3. data/bench/report.rb +80 -10
  4. data/bench/report.txt +10 -3
  5. data/bench/setup.rb +55 -0
  6. data/lib/qs.rb +75 -15
  7. data/lib/qs/client.rb +73 -22
  8. data/lib/qs/daemon.rb +21 -21
  9. data/lib/qs/daemon_data.rb +4 -4
  10. data/lib/qs/dispatch_job.rb +36 -0
  11. data/lib/qs/dispatch_job_handler.rb +79 -0
  12. data/lib/qs/dispatcher_queue.rb +19 -0
  13. data/lib/qs/error_handler.rb +12 -12
  14. data/lib/qs/event.rb +82 -0
  15. data/lib/qs/event_handler.rb +34 -0
  16. data/lib/qs/event_handler_test_helpers.rb +17 -0
  17. data/lib/qs/job.rb +19 -31
  18. data/lib/qs/job_handler.rb +6 -63
  19. data/lib/qs/{test_helpers.rb → job_handler_test_helpers.rb} +2 -2
  20. data/lib/qs/message.rb +29 -0
  21. data/lib/qs/message_handler.rb +84 -0
  22. data/lib/qs/payload.rb +98 -0
  23. data/lib/qs/payload_handler.rb +106 -54
  24. data/lib/qs/queue.rb +39 -6
  25. data/lib/qs/queue_item.rb +33 -0
  26. data/lib/qs/route.rb +7 -7
  27. data/lib/qs/runner.rb +6 -5
  28. data/lib/qs/test_runner.rb +41 -13
  29. data/lib/qs/version.rb +1 -1
  30. data/qs.gemspec +1 -1
  31. data/test/helper.rb +1 -1
  32. data/test/support/app_daemon.rb +77 -11
  33. data/test/support/factory.rb +34 -0
  34. data/test/system/daemon_tests.rb +146 -77
  35. data/test/system/queue_tests.rb +87 -0
  36. data/test/unit/client_tests.rb +184 -45
  37. data/test/unit/daemon_data_tests.rb +4 -4
  38. data/test/unit/daemon_tests.rb +32 -32
  39. data/test/unit/dispatch_job_handler_tests.rb +163 -0
  40. data/test/unit/dispatch_job_tests.rb +75 -0
  41. data/test/unit/dispatcher_queue_tests.rb +42 -0
  42. data/test/unit/error_handler_tests.rb +9 -9
  43. data/test/unit/event_handler_test_helpers_tests.rb +55 -0
  44. data/test/unit/event_handler_tests.rb +63 -0
  45. data/test/unit/event_tests.rb +162 -0
  46. data/test/unit/{test_helper_tests.rb → job_handler_test_helper_tests.rb} +13 -19
  47. data/test/unit/job_handler_tests.rb +17 -210
  48. data/test/unit/job_tests.rb +49 -79
  49. data/test/unit/message_handler_tests.rb +235 -0
  50. data/test/unit/message_tests.rb +64 -0
  51. data/test/unit/payload_handler_tests.rb +285 -86
  52. data/test/unit/payload_tests.rb +139 -0
  53. data/test/unit/qs_runner_tests.rb +6 -6
  54. data/test/unit/qs_tests.rb +167 -28
  55. data/test/unit/queue_item_tests.rb +51 -0
  56. data/test/unit/queue_tests.rb +126 -18
  57. data/test/unit/route_tests.rb +12 -13
  58. data/test/unit/runner_tests.rb +10 -10
  59. data/test/unit/test_runner_tests.rb +117 -24
  60. metadata +51 -21
  61. data/bench/queue.rb +0 -8
  62. data/lib/qs/redis_item.rb +0 -33
  63. data/test/unit/redis_item_tests.rb +0 -49
@@ -59,14 +59,14 @@ class Qs::DaemonData
59
59
  end
60
60
 
61
61
  should "build a routes lookup hash" do
62
- expected = @routes.inject({}){ |h, r| h.merge(r.name => r) }
62
+ expected = @routes.inject({}){ |h, r| h.merge(r.id => r) }
63
63
  assert_equal expected, subject.routes
64
64
  end
65
65
 
66
66
  should "allow looking up a route using `route_for`" do
67
- expected = @routes.choice
68
- route = subject.route_for(expected.name)
69
- assert_equal expected, route
67
+ exp_route = @routes.choice
68
+ route = subject.route_for(exp_route.id)
69
+ assert_equal exp_route, route
70
70
  end
71
71
 
72
72
  should "raise a not found error using `route_for` with an invalid name" do
@@ -6,7 +6,7 @@ require 'ns-options/assert_macros'
6
6
  require 'thread'
7
7
  require 'qs/client'
8
8
  require 'qs/queue'
9
- require 'qs/redis_item'
9
+ require 'qs/queue_item'
10
10
 
11
11
  module Qs::Daemon
12
12
 
@@ -275,8 +275,8 @@ module Qs::Daemon
275
275
  @daemon = @daemon_class.new
276
276
  @thread = @daemon.start
277
277
 
278
- @serialized_payload = Factory.string
279
- @client_spy.append(@queue.redis_key, @serialized_payload)
278
+ @encoded_payload = Factory.string
279
+ @client_spy.append(@queue.redis_key, @encoded_payload)
280
280
  end
281
281
  subject{ @daemon }
282
282
 
@@ -285,7 +285,7 @@ module Qs::Daemon
285
285
  assert_equal :block_dequeue, call.command
286
286
  exp = [subject.signals_redis_key, subject.queue_redis_keys, 0].flatten
287
287
  assert_equal exp, call.args
288
- exp = Qs::RedisItem.new(@queue.redis_key, @serialized_payload)
288
+ exp = Qs::QueueItem.new(@queue.redis_key, @encoded_payload)
289
289
  assert_equal exp, @worker_pool_spy.work_items.first
290
290
  end
291
291
 
@@ -353,15 +353,15 @@ module Qs::Daemon
353
353
  @daemon = @daemon_class.new
354
354
  @thread = @daemon.start
355
355
 
356
- @redis_item = Qs::RedisItem.new(Factory.string, Factory.string)
357
- @worker_pool_spy.work_proc.call(@redis_item)
356
+ @queue_item = Qs::QueueItem.new(Factory.string, Factory.string)
357
+ @worker_pool_spy.work_proc.call(@queue_item)
358
358
  end
359
359
  subject{ @daemon }
360
360
 
361
361
  should "build and run a payload handler" do
362
362
  assert_not_nil @ph_spy
363
363
  assert_equal subject.daemon_data, @ph_spy.daemon_data
364
- assert_equal @redis_item, @ph_spy.redis_item
364
+ assert_equal @queue_item, @ph_spy.queue_item
365
365
  end
366
366
 
367
367
  end
@@ -373,27 +373,27 @@ module Qs::Daemon
373
373
  @thread = @daemon.start
374
374
 
375
375
  @exception = Factory.exception
376
- @redis_item = Qs::RedisItem.new(Factory.string, Factory.string)
376
+ @queue_item = Qs::QueueItem.new(Factory.string, Factory.string)
377
377
  @callback = @worker_pool_spy.on_worker_error_callbacks.first
378
378
  end
379
379
  subject{ @daemon }
380
380
 
381
- should "requeue the redis item if it wasn't started" do
382
- @redis_item.started = false
383
- @callback.call('worker', @exception, @redis_item)
381
+ should "requeue the queue item if it wasn't started" do
382
+ @queue_item.started = false
383
+ @callback.call('worker', @exception, @queue_item)
384
384
  call = @client_spy.calls.detect{ |c| c.command == :prepend }
385
385
  assert_not_nil call
386
- assert_equal @redis_item.queue_redis_key, call.args.first
387
- assert_equal @redis_item.serialized_payload, call.args.last
386
+ assert_equal @queue_item.queue_redis_key, call.args.first
387
+ assert_equal @queue_item.encoded_payload, call.args.last
388
388
  end
389
389
 
390
- should "not requeue the redis item if it was started" do
391
- @redis_item.started = true
392
- @callback.call('worker', @exception, @redis_item)
390
+ should "not requeue the queue item if it was started" do
391
+ @queue_item.started = true
392
+ @callback.call('worker', @exception, @queue_item)
393
393
  assert_nil @client_spy.calls.detect{ |c| c.command == :prepend }
394
394
  end
395
395
 
396
- should "do nothing if not passed a redis item" do
396
+ should "do nothing if not passed a queue item" do
397
397
  assert_nothing_raised{ @callback.call(@exception, nil) }
398
398
  end
399
399
 
@@ -402,8 +402,8 @@ module Qs::Daemon
402
402
  class StopTests < StartTests
403
403
  desc "and then stopped"
404
404
  setup do
405
- @redis_item = Qs::RedisItem.new(@queue.redis_key, Factory.string)
406
- @worker_pool_spy.add_work(@redis_item)
405
+ @queue_item = Qs::QueueItem.new(@queue.redis_key, Factory.string)
406
+ @worker_pool_spy.add_work(@queue_item)
407
407
 
408
408
  @daemon.stop true
409
409
  end
@@ -416,8 +416,8 @@ module Qs::Daemon
416
416
  should "requeue any work left on the pool" do
417
417
  call = @client_spy.calls.last
418
418
  assert_equal :prepend, call.command
419
- assert_equal @redis_item.queue_redis_key, call.args.first
420
- assert_equal @redis_item.serialized_payload, call.args.last
419
+ assert_equal @queue_item.queue_redis_key, call.args.first
420
+ assert_equal @queue_item.encoded_payload, call.args.last
421
421
  end
422
422
 
423
423
  should "stop the work loop thread" do
@@ -449,8 +449,8 @@ module Qs::Daemon
449
449
  class HaltTests < StartTests
450
450
  desc "and then halted"
451
451
  setup do
452
- @redis_item = Qs::RedisItem.new(@queue.redis_key, Factory.string)
453
- @worker_pool_spy.add_work(@redis_item)
452
+ @queue_item = Qs::QueueItem.new(@queue.redis_key, Factory.string)
453
+ @worker_pool_spy.add_work(@queue_item)
454
454
 
455
455
  @daemon.halt true
456
456
  end
@@ -463,8 +463,8 @@ module Qs::Daemon
463
463
  should "requeue any work left on the pool" do
464
464
  call = @client_spy.calls.last
465
465
  assert_equal :prepend, call.command
466
- assert_equal @redis_item.queue_redis_key, call.args.first
467
- assert_equal @redis_item.serialized_payload, call.args.last
466
+ assert_equal @queue_item.queue_redis_key, call.args.first
467
+ assert_equal @queue_item.encoded_payload, call.args.last
468
468
  end
469
469
 
470
470
  should "stop the work loop thread" do
@@ -501,8 +501,8 @@ module Qs::Daemon
501
501
 
502
502
  # cause the daemon to loop, its sleeping on the original block_dequeue
503
503
  # call that happened before the stub
504
- @redis_item = Qs::RedisItem.new(@queue.redis_key, Factory.string)
505
- @client_spy.append(@redis_item.queue_redis_key, @redis_item.serialized_payload)
504
+ @queue_item = Qs::QueueItem.new(@queue.redis_key, Factory.string)
505
+ @client_spy.append(@queue_item.queue_redis_key, @queue_item.encoded_payload)
506
506
  end
507
507
 
508
508
  should "shutdown the worker pool" do
@@ -513,8 +513,8 @@ module Qs::Daemon
513
513
  should "requeue any work left on the pool" do
514
514
  call = @client_spy.calls.last
515
515
  assert_equal :prepend, call.command
516
- assert_equal @redis_item.queue_redis_key, call.args.first
517
- assert_equal @redis_item.serialized_payload, call.args.last
516
+ assert_equal @queue_item.queue_redis_key, call.args.first
517
+ assert_equal @queue_item.encoded_payload, call.args.last
518
518
  end
519
519
 
520
520
  should "stop the work loop thread" do
@@ -667,11 +667,11 @@ module Qs::Daemon
667
667
  TestHandler = Class.new
668
668
 
669
669
  class PayloadHandlerSpy
670
- attr_reader :daemon_data, :redis_item, :run_called
670
+ attr_reader :daemon_data, :queue_item, :run_called
671
671
 
672
- def initialize(daemon_data, redis_item)
672
+ def initialize(daemon_data, queue_item)
673
673
  @daemon_data = daemon_data
674
- @redis_item = redis_item
674
+ @queue_item = queue_item
675
675
  @run_called = false
676
676
  end
677
677
 
@@ -0,0 +1,163 @@
1
+ require 'assert'
2
+ require 'qs/dispatch_job_handler'
3
+
4
+ require 'qs/job_handler_test_helpers'
5
+
6
+ module Qs::DispatchJobHandler
7
+
8
+ class UnitTests < Assert::Context
9
+ include Qs::JobHandler::TestHelpers
10
+
11
+ desc "Qs::DispatchJobHandler"
12
+ setup do
13
+ Qs.init
14
+ @handler_class = Class.new do
15
+ include Qs::DispatchJobHandler
16
+ end
17
+ end
18
+ teardown do
19
+ Qs.reset!
20
+ end
21
+ subject{ @handler_class }
22
+
23
+ should "be a job handler" do
24
+ assert_includes Qs::JobHandler, subject
25
+ end
26
+
27
+ end
28
+
29
+ class InitSetupTests < UnitTests
30
+ desc "when init"
31
+ setup do
32
+ @job = Factory.dispatch_job(:publisher => Factory.string)
33
+ @queue_names = Factory.integer(3).times.map{ Factory.string }
34
+ Assert.stub(Qs, :event_subscribers){ @queue_names }
35
+
36
+ @push_calls = []
37
+ Assert.stub(Qs, :push){ |*args| @push_calls << PushCall.new(*args) }
38
+
39
+ @logger_spy = LoggerSpy.new
40
+ @runner_args = {
41
+ :job => @job,
42
+ :params => @job.params,
43
+ :logger => @logger_spy
44
+ }
45
+ end
46
+ subject{ @handler }
47
+
48
+ end
49
+
50
+ class InitTests < InitSetupTests
51
+ setup do
52
+ @runner = test_runner(@handler_class, @runner_args)
53
+ @handler = @runner.handler
54
+ end
55
+
56
+ should "know its event and subscribed queue names" do
57
+ assert_equal @job.event, subject.event
58
+ assert_equal @queue_names, subject.subscribed_queue_names
59
+ end
60
+
61
+ end
62
+
63
+ class RunTests < InitTests
64
+ desc "and run"
65
+ setup do
66
+ @runner.run
67
+ end
68
+
69
+ should "push the events payload to all of the subscribed queue names" do
70
+ assert_equal @queue_names, @push_calls.map(&:queue_name)
71
+ exp = Qs::Payload.event_hash(subject.event)
72
+ assert_equal [exp], @push_calls.map(&:payload).uniq
73
+ end
74
+
75
+ should "log the queues it dispatches to" do
76
+ exp = [
77
+ "Dispatching #{subject.event.route_name}",
78
+ " params: #{subject.event.params.inspect}",
79
+ " publisher: #{subject.event.publisher}",
80
+ " published at: #{subject.event.published_at}",
81
+ "Found #{subject.subscribed_queue_names.size} subscribed queue(s):",
82
+ @queue_names.map{ |queue_name| " => #{queue_name}" }
83
+ ].flatten.join("\n")
84
+ assert_equal exp, @logger_spy.messages.join("\n")
85
+ end
86
+
87
+ end
88
+
89
+ class RunWithDispatchesThatErrorTests < InitSetupTests
90
+ desc "and run with dispatches that error"
91
+ setup do
92
+ @fail_queue_names = Factory.integer(3).times.map{ Factory.string }
93
+ @dispatch_error = RuntimeError.new(Factory.text)
94
+ payload_hash = Qs::Payload.event_hash(@job.event)
95
+ @fail_queue_names.each do |queue_name|
96
+ Assert.stub(Qs, :push).with(queue_name, payload_hash) do
97
+ raise @dispatch_error
98
+ end
99
+ end
100
+
101
+ @success_queue_names = @queue_names.dup
102
+ # add the fail queue names to the front to test that they don't cause
103
+ # the other queues not to be dispatched to
104
+ @queue_names = @fail_queue_names + @success_queue_names
105
+
106
+ @runner = test_runner(@handler_class, @runner_args)
107
+ @handler = @runner.handler
108
+
109
+ @exception = nil
110
+ begin; @runner.run; rescue => @exception; end
111
+ end
112
+
113
+ should "raise a dispatch error after trying to dispatch to every queue" do
114
+ assert_equal @success_queue_names, @push_calls.map(&:queue_name)
115
+
116
+ assert_instance_of DispatchError, @exception
117
+ descriptions = @fail_queue_names.map do |queue_name|
118
+ "#{queue_name} - #{@dispatch_error.class}: #{@dispatch_error.message}"
119
+ end
120
+ exp = "#{subject.event.route_name} event wasn't dispatched to:\n" \
121
+ " #{descriptions.join("\n ")}"
122
+ assert_equal exp, @exception.message
123
+ exp = @fail_queue_names.map do |queue_name|
124
+ FailedDispatch.new(queue_name, @dispatch_error)
125
+ end
126
+ assert_equal exp, @exception.failed_dispatches
127
+ end
128
+
129
+ should "log the queues it dispatches to and the errors it encounters" do
130
+ exp = [
131
+ "Dispatching #{subject.event.route_name}",
132
+ " params: #{subject.event.params.inspect}",
133
+ " publisher: #{subject.event.publisher}",
134
+ " published at: #{subject.event.published_at}",
135
+ "Found #{subject.subscribed_queue_names.size} subscribed queue(s):",
136
+ @fail_queue_names.map{ |queue_name| " => #{queue_name} (failed)" },
137
+ @success_queue_names.map{ |queue_name| " => #{queue_name}" },
138
+ "Failed to dispatch the event to #{@fail_queue_names.size} subscribed queues",
139
+ @fail_queue_names.map do |queue_name|
140
+ [ queue_name,
141
+ " #{@dispatch_error.class}: #{@dispatch_error.message}",
142
+ " #{@dispatch_error.backtrace.first}"
143
+ ]
144
+ end
145
+ ].flatten.join("\n")
146
+ assert_equal exp, @logger_spy.messages.join("\n")
147
+ end
148
+
149
+ end
150
+
151
+ PushCall = Struct.new(:queue_name, :payload)
152
+
153
+ class LoggerSpy
154
+ attr_reader :messages
155
+
156
+ def initialize
157
+ @messages = []
158
+ end
159
+
160
+ def info(message); @messages << message; end
161
+ end
162
+
163
+ end
@@ -0,0 +1,75 @@
1
+ require 'assert'
2
+ require 'qs/dispatch_job'
3
+
4
+ require 'qs/event'
5
+ require 'qs/job'
6
+
7
+ class Qs::DispatchJob
8
+
9
+ class UnitTests < Assert::Context
10
+ desc "Qs::DispatchJob"
11
+ setup do
12
+ @job_class = Qs::DispatchJob
13
+ end
14
+ subject{ @job_class }
15
+
16
+ should "be a job" do
17
+ assert Qs::DispatchJob < Qs::Job
18
+ end
19
+
20
+ end
21
+
22
+ class InitTests < UnitTests
23
+ desc "when init"
24
+ setup do
25
+ @event_channel = Factory.string
26
+ @event_name = Factory.string
27
+ @event_params = { Factory.string => Factory.string }
28
+ @event_publisher = Factory.string
29
+ @created_at = Factory.time
30
+ @job = @job_class.new(@event_channel, @event_name, {
31
+ :event_params => @event_params,
32
+ :event_publisher => @event_publisher,
33
+ :created_at => @created_at
34
+ })
35
+ end
36
+ teardown do
37
+ Qs.reset!
38
+ end
39
+ subject{ @job }
40
+
41
+ should have_imeths :event
42
+
43
+ should "know its name, params and created at" do
44
+ assert_equal Qs.dispatcher_job_name, subject.name
45
+ exp = {
46
+ 'event_channel' => @event_channel,
47
+ 'event_name' => @event_name,
48
+ 'event_params' => @event_params,
49
+ 'event_publisher' => @event_publisher
50
+ }
51
+ assert_equal exp, subject.params
52
+ assert_equal @created_at, subject.created_at
53
+ end
54
+
55
+ should "default its event params and event publisher" do
56
+ Qs.config.event_publisher = Factory.string
57
+ job = @job_class.new(@event_channel, @event_name)
58
+ assert_equal({}, job.params['event_params'])
59
+ assert_equal Qs.event_publisher, job.params['event_publisher']
60
+ end
61
+
62
+ should "know how to build an event from its params" do
63
+ event = subject.event
64
+ exp = Qs::Event.new(@event_channel, @event_name, {
65
+ :params => @event_params,
66
+ :publisher => @event_publisher,
67
+ :published_at => @created_at
68
+ })
69
+ assert_equal exp, event
70
+ assert_same event, subject.event
71
+ end
72
+
73
+ end
74
+
75
+ end
@@ -0,0 +1,42 @@
1
+ require 'assert'
2
+ require 'qs/dispatcher_queue'
3
+
4
+ module Qs::DispatcherQueue
5
+
6
+ class UnitTests < Assert::Context
7
+ desc "Qs::DispatcherQueue"
8
+ subject{ Qs::DispatcherQueue }
9
+
10
+ should have_imeths :new
11
+
12
+ should "build a dispatcher queue" do
13
+ options = {
14
+ :queue_class => Class.new(Qs::Queue),
15
+ :queue_name => Factory.string,
16
+ :job_name => Factory.string,
17
+ :job_handler_class_name => Factory.string
18
+ }
19
+ dispatcher_queue = subject.new(options)
20
+ assert_instance_of options[:queue_class], dispatcher_queue
21
+ assert_equal options[:queue_name], dispatcher_queue.name
22
+
23
+ route = dispatcher_queue.routes.last
24
+ assert_instance_of Qs::Route, route
25
+ exp = Qs::Message::RouteId.new(Qs::Job::PAYLOAD_TYPE, options[:job_name])
26
+ assert_equal exp, route.id
27
+ assert_equal options[:job_handler_class_name], route.handler_class_name
28
+ end
29
+
30
+ end
31
+
32
+ class RunDispatchJobTests < UnitTests
33
+ desc "RunDispatchJob"
34
+ subject{ RunDispatchJob }
35
+
36
+ should "be a dispatch job handler" do
37
+ assert_includes Qs::DispatchJobHandler, subject
38
+ end
39
+
40
+ end
41
+
42
+ end