chore-core 3.2.3 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/LICENSE.txt +1 -1
- data/README.md +170 -153
- data/chore-core.gemspec +2 -3
- data/lib/chore.rb +20 -0
- data/lib/chore/cli.rb +1 -2
- data/lib/chore/configuration.rb +1 -1
- data/lib/chore/consumer.rb +41 -9
- data/lib/chore/job.rb +2 -0
- data/lib/chore/publisher.rb +18 -2
- data/lib/chore/queues/filesystem/consumer.rb +18 -13
- data/lib/chore/queues/filesystem/publisher.rb +1 -1
- data/lib/chore/queues/sqs.rb +22 -13
- data/lib/chore/queues/sqs/consumer.rb +61 -33
- data/lib/chore/queues/sqs/publisher.rb +26 -17
- data/lib/chore/strategies/consumer/batcher.rb +6 -6
- data/lib/chore/strategies/consumer/single_consumer_strategy.rb +5 -5
- data/lib/chore/strategies/consumer/threaded_consumer_strategy.rb +6 -6
- data/lib/chore/strategies/consumer/throttled_consumer_strategy.rb +10 -11
- data/lib/chore/strategies/worker/helpers/ipc.rb +0 -1
- data/lib/chore/unit_of_work.rb +2 -1
- data/lib/chore/version.rb +3 -3
- data/lib/chore/worker.rb +4 -4
- data/spec/chore/consumer_spec.rb +1 -1
- data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb +5 -7
- data/spec/chore/queues/sqs/consumer_spec.rb +117 -76
- data/spec/chore/queues/sqs/publisher_spec.rb +49 -60
- data/spec/chore/queues/sqs_spec.rb +32 -41
- data/spec/chore/strategies/consumer/single_consumer_strategy_spec.rb +3 -3
- data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb +6 -6
- data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb +1 -1
- data/spec/chore/strategies/worker/single_worker_strategy_spec.rb +1 -1
- data/spec/chore/worker_spec.rb +21 -21
- data/spec/spec_helper.rb +1 -1
- data/spec/support/queues/sqs/fake_objects.rb +18 -0
- metadata +9 -13
@@ -3,21 +3,26 @@ require 'chore/publisher'
|
|
3
3
|
module Chore
|
4
4
|
module Queues
|
5
5
|
module SQS
|
6
|
-
|
7
6
|
# SQS Publisher, for writing messages to SQS from Chore
|
8
7
|
class Publisher < Chore::Publisher
|
9
8
|
@@reset_next = true
|
10
9
|
|
10
|
+
# @param [Hash] opts Publisher options
|
11
11
|
def initialize(opts={})
|
12
12
|
super
|
13
13
|
@sqs_queues = {}
|
14
14
|
@sqs_queue_urls = {}
|
15
15
|
end
|
16
16
|
|
17
|
-
#
|
17
|
+
# Publishes a message to an SQS queue
|
18
|
+
#
|
19
|
+
# @param [String] queue_name Name of the SQS queue
|
20
|
+
# @param [Hash] job Job instance definition, will be encoded to JSON
|
21
|
+
#
|
22
|
+
# @return [struct Aws::SQS::Types::SendMessageResult]
|
18
23
|
def publish(queue_name,job)
|
19
|
-
queue =
|
20
|
-
queue.send_message(encode_job(job))
|
24
|
+
queue = queue(queue_name)
|
25
|
+
queue.send_message(message_body: encode_job(job))
|
21
26
|
end
|
22
27
|
|
23
28
|
# Sets a flag that instructs the publisher to reset the connection the next time it's used
|
@@ -25,29 +30,33 @@ module Chore
|
|
25
30
|
@@reset_next = true
|
26
31
|
end
|
27
32
|
|
28
|
-
|
33
|
+
private
|
34
|
+
|
35
|
+
# SQS API client object
|
36
|
+
#
|
37
|
+
# @return [Aws::SQS::Client]
|
29
38
|
def sqs
|
30
|
-
@sqs ||=
|
31
|
-
:access_key_id => Chore.config.aws_access_key,
|
32
|
-
:secret_access_key => Chore.config.aws_secret_key,
|
33
|
-
:logger => Chore.logger,
|
34
|
-
:log_level => :debug)
|
39
|
+
@sqs ||= Chore::Queues::SQS.sqs_client
|
35
40
|
end
|
36
41
|
|
37
|
-
# Retrieves the SQS queue
|
42
|
+
# Retrieves the SQS queue object. The method will cache the results to prevent round trips on subsequent calls
|
43
|
+
#
|
38
44
|
# If <tt>reset_connection!</tt> has been called, this will result in the connection being re-initialized,
|
39
45
|
# as well as clear any cached results from prior calls
|
46
|
+
#
|
47
|
+
# @param [String] name Name of SQS queue
|
48
|
+
#
|
49
|
+
# @return [Aws::SQS::Queue]
|
40
50
|
def queue(name)
|
41
|
-
|
42
|
-
|
43
|
-
p.empty!
|
44
|
-
end
|
51
|
+
if @@reset_next
|
52
|
+
Aws.empty_connection_pools!
|
45
53
|
@sqs = nil
|
46
54
|
@@reset_next = false
|
47
55
|
@sqs_queues = {}
|
48
56
|
end
|
49
|
-
|
50
|
-
@
|
57
|
+
|
58
|
+
@sqs_queue_urls[name] ||= sqs.get_queue_url(queue_name: name).queue_url
|
59
|
+
@sqs_queues[name] ||= Aws::SQS::Queue.new(url: @sqs_queue_urls[name], client: sqs)
|
51
60
|
end
|
52
61
|
end
|
53
62
|
end
|
@@ -15,17 +15,17 @@ module Chore
|
|
15
15
|
@running = true
|
16
16
|
end
|
17
17
|
|
18
|
-
# The main entry point of the Batcher, <tt>schedule</tt> begins a thread with the provided +batch_timeout+
|
19
|
-
# as the only argument. While the Batcher is running, it will attempt to check if either the batch is full,
|
18
|
+
# The main entry point of the Batcher, <tt>schedule</tt> begins a thread with the provided +batch_timeout+
|
19
|
+
# as the only argument. While the Batcher is running, it will attempt to check if either the batch is full,
|
20
20
|
# or if the +batch_timeout+ has elapsed since the oldest message was added. If either case is true, the
|
21
21
|
# items in the batch will be executed.
|
22
|
-
#
|
22
|
+
#
|
23
23
|
# Calling <tt>stop</tt> will cause the thread to finish it's current check, and exit
|
24
24
|
def schedule(batch_timeout)
|
25
25
|
@thread = Thread.new(batch_timeout) do |timeout|
|
26
|
-
Chore.logger.info "Batching
|
26
|
+
Chore.logger.info "Batching thread starting with #{batch_timeout} second timeout"
|
27
27
|
while @running do
|
28
|
-
begin
|
28
|
+
begin
|
29
29
|
oldest_item = @batch.first
|
30
30
|
timestamp = oldest_item && oldest_item.created_at
|
31
31
|
Chore.logger.debug "Oldest message in batch: #{timestamp}, size: #{@batch.size}"
|
@@ -33,7 +33,7 @@ module Chore
|
|
33
33
|
Chore.logger.debug "Batching timeout reached (#{timestamp + timeout}), current size: #{@batch.size}"
|
34
34
|
self.execute(true)
|
35
35
|
end
|
36
|
-
sleep(1)
|
36
|
+
sleep(1)
|
37
37
|
rescue => e
|
38
38
|
Chore.logger.error "Batcher#schedule raised an exception: #{e.inspect}"
|
39
39
|
end
|
@@ -10,16 +10,16 @@ module Chore
|
|
10
10
|
end
|
11
11
|
|
12
12
|
# Begins fetching from the configured queue by way of the configured Consumer. This can only be used if you have a
|
13
|
-
# single queue which can be kept up with at a relatively low volume. If you have more than a single queue
|
14
|
-
# it will raise an exception.
|
13
|
+
# single queue which can be kept up with at a relatively low volume. If you have more than a single queue
|
14
|
+
# configured, it will raise an exception.
|
15
15
|
def fetch
|
16
16
|
Chore.logger.debug "Starting up consumer strategy: #{self.class.name}"
|
17
17
|
queues = Chore.config.queues
|
18
18
|
raise "When using SingleConsumerStrategy only one queue can be defined. Queues: #{queues}" unless queues.size == 1
|
19
|
-
|
19
|
+
|
20
20
|
@consumer = Chore.config.consumer.new(queues.first)
|
21
|
-
@consumer.consume do |
|
22
|
-
work = UnitOfWork.new(
|
21
|
+
@consumer.consume do |message_id, message_receipt_handle, queue_name, queue_timeout, body, previous_attempts|
|
22
|
+
work = UnitOfWork.new(message_id, message_receipt_handle, queue_name, queue_timeout, body, previous_attempts, @consumer)
|
23
23
|
@fetcher.manager.assign(work)
|
24
24
|
end
|
25
25
|
end
|
@@ -23,7 +23,7 @@ module Chore
|
|
23
23
|
Chore.logger.debug "Starting up consumer strategy: #{self.class.name}"
|
24
24
|
threads = []
|
25
25
|
Chore.config.queues.each do |queue|
|
26
|
-
Chore.config.threads_per_queue.times do
|
26
|
+
Chore.config.threads_per_queue.times do
|
27
27
|
if running?
|
28
28
|
threads << start_consumer_thread(queue)
|
29
29
|
end
|
@@ -32,7 +32,7 @@ module Chore
|
|
32
32
|
|
33
33
|
threads.each(&:join)
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
# If the ThreadedConsumerStrategy is currently running <tt>stop!</tt> will begin signalling it to stop
|
37
37
|
# It will stop the batcher from forking more work, as well as set a flag which will disable it's own consuming
|
38
38
|
# threads once they finish with their current work.
|
@@ -49,21 +49,21 @@ module Chore
|
|
49
49
|
@running
|
50
50
|
end
|
51
51
|
|
52
|
-
private
|
52
|
+
private
|
53
53
|
# Starts a consumer thread for polling the given +queue+.
|
54
54
|
# If <tt>stop!<tt> is called, the threads will shut themsevles down.
|
55
55
|
def start_consumer_thread(queue)
|
56
56
|
t = Thread.new(queue) do |tQueue|
|
57
57
|
begin
|
58
58
|
consumer = Chore.config.consumer.new(tQueue)
|
59
|
-
consumer.consume do |
|
59
|
+
consumer.consume do |message_id, message_receipt_handle, queue_name, queue_timeout, body, previous_attempts|
|
60
60
|
# Quick hack to force this thread to end it's work
|
61
61
|
# if we're shutting down. Could be delayed due to the
|
62
62
|
# weird sometimes-blocking nature of SQS.
|
63
63
|
consumer.stop if !running?
|
64
|
-
Chore.logger.debug { "Got message: #{
|
64
|
+
Chore.logger.debug { "Got message: #{message_id}"}
|
65
65
|
|
66
|
-
work = UnitOfWork.new(
|
66
|
+
work = UnitOfWork.new(message_id, message_receipt_handle, queue_name, queue_timeout, body, previous_attempts, consumer)
|
67
67
|
Chore.run_hooks_for(:consumed_from_source, work)
|
68
68
|
@batcher.add(work)
|
69
69
|
end
|
@@ -65,7 +65,7 @@ module Chore
|
|
65
65
|
end
|
66
66
|
|
67
67
|
# Gives work back to the queue in case it couldn't be assigned
|
68
|
-
#
|
68
|
+
#
|
69
69
|
# This will go into a separate queue so that it will be prioritized
|
70
70
|
# over other work that hasn't been attempted yet. It also avoids
|
71
71
|
# a deadlock where @queue is full and the master is waiting to return
|
@@ -100,22 +100,21 @@ module Chore
|
|
100
100
|
end
|
101
101
|
|
102
102
|
def create_work_units(consumer)
|
103
|
-
consumer.consume do |
|
104
|
-
# Note: The unit of work object contains a consumer object that when
|
105
|
-
# used to consume from SQS, would have a mutex (that comes as a part
|
106
|
-
# of the AWS sdk); When sending these objects across from one process
|
107
|
-
# to another, we cannot send this across (becasue of the mutex). To
|
103
|
+
consumer.consume do |message_id, message_receipt_handle, queue, timeout, body, previous_attempts|
|
104
|
+
# Note: The unit of work object contains a consumer object that when
|
105
|
+
# used to consume from SQS, would have a mutex (that comes as a part
|
106
|
+
# of the AWS sdk); When sending these objects across from one process
|
107
|
+
# to another, we cannot send this across (becasue of the mutex). To
|
108
108
|
# work around this, we simply ignore the consumer object when creating
|
109
|
-
# the unit of work object, and when the worker recieves the work
|
110
|
-
# object, it assigns it a consumer object.
|
109
|
+
# the unit of work object, and when the worker recieves the work
|
110
|
+
# object, it assigns it a consumer object.
|
111
111
|
# (to allow for communication back to the queue it was consumed from)
|
112
|
-
work = UnitOfWork.new(
|
113
|
-
previous_attempts)
|
112
|
+
work = UnitOfWork.new(message_id, message_receipt_handle, queue, timeout, body, previous_attempts)
|
114
113
|
Chore.run_hooks_for(:consumed_from_source, work)
|
115
114
|
@queue.push(work) if running?
|
116
115
|
Chore.run_hooks_for(:added_to_queue, work)
|
117
116
|
end
|
118
117
|
end
|
119
|
-
end #
|
118
|
+
end # ThrottledConsumerStrategy
|
120
119
|
end
|
121
120
|
end # Chore
|
data/lib/chore/unit_of_work.rb
CHANGED
@@ -2,13 +2,14 @@ module Chore
|
|
2
2
|
# Simple class to hold job processing information.
|
3
3
|
# Has six attributes:
|
4
4
|
# * +:id+ The queue implementation specific identifier for this message.
|
5
|
+
# * +:receipt_handle+ The queue implementation specific identifier for the receipt of this message.
|
5
6
|
# * +:queue_name+ The name of the queue the job came from
|
6
7
|
# * +:queue_timeout+ The time (in seconds) before the job will get re-enqueued if not processed
|
7
8
|
# * +:message+ The actual data of the message.
|
8
9
|
# * +:previous_attempts+ The number of times the work has been attempted previously.
|
9
10
|
# * +:consumer+ The consumer instance used to fetch this message. Most queue implementations won't need access to this, but some (RabbitMQ) will. So we
|
10
11
|
# make sure to pass it along with each message. This instance will be used by the Worker for things like <tt>complete</tt> and </tt>reject</tt>.
|
11
|
-
class UnitOfWork < Struct.new(:id
|
12
|
+
class UnitOfWork < Struct.new(:id, :receipt_handle, :queue_name, :queue_timeout, :message, :previous_attempts, :consumer, :decoded_message, :klass)
|
12
13
|
# The time at which this unit of work was created
|
13
14
|
attr_accessor :created_at
|
14
15
|
|
data/lib/chore/version.rb
CHANGED
data/lib/chore/worker.rb
CHANGED
@@ -56,7 +56,7 @@ module Chore
|
|
56
56
|
|
57
57
|
if item.consumer.duplicate_message?(dedupe_key, item.klass, item.queue_timeout)
|
58
58
|
Chore.logger.info { "Found and deleted duplicate job #{item.klass}"}
|
59
|
-
item.consumer.complete(item.id)
|
59
|
+
item.consumer.complete(item.id, item.receipt_handle)
|
60
60
|
return true
|
61
61
|
end
|
62
62
|
end
|
@@ -89,7 +89,7 @@ module Chore
|
|
89
89
|
Chore.logger.error { "Failed to run job for #{item.message} with error: #{e.message} #{e.backtrace * "\n"}" }
|
90
90
|
if item.current_attempt >= Chore.config.max_attempts
|
91
91
|
Chore.run_hooks_for(:on_permanent_failure,item.queue_name,item.message,e)
|
92
|
-
item.consumer.complete(item.id)
|
92
|
+
item.consumer.complete(item.id, item.receipt_handle)
|
93
93
|
else
|
94
94
|
Chore.run_hooks_for(:on_failure,item.message,e)
|
95
95
|
item.consumer.reject(item.id)
|
@@ -112,7 +112,7 @@ module Chore
|
|
112
112
|
begin
|
113
113
|
Chore.logger.info { "Running job #{klass} with params #{message}"}
|
114
114
|
perform_job(klass,message)
|
115
|
-
item.consumer.complete(item.id)
|
115
|
+
item.consumer.complete(item.id, item.receipt_handle)
|
116
116
|
Chore.logger.info { "Finished job #{klass} with params #{message}"}
|
117
117
|
klass.run_hooks_for(:after_perform, message)
|
118
118
|
Chore.run_hooks_for(:worker_ended, item)
|
@@ -141,7 +141,7 @@ module Chore
|
|
141
141
|
Chore.logger.error { "Failed to run job #{item.message} with error: #{e.message} at #{e.backtrace * "\n"}" }
|
142
142
|
if item.current_attempt >= klass.options[:max_attempts]
|
143
143
|
klass.run_hooks_for(:on_permanent_failure,item.queue_name,message,e)
|
144
|
-
item.consumer.complete(item.id)
|
144
|
+
item.consumer.complete(item.id, item.receipt_handle)
|
145
145
|
else
|
146
146
|
klass.run_hooks_for(:on_failure, message, e)
|
147
147
|
item.consumer.reject(item.id)
|
data/spec/chore/consumer_spec.rb
CHANGED
@@ -31,6 +31,6 @@ describe Chore::Consumer do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
it 'should not have an implemented complete method' do
|
34
|
-
expect { consumer.complete(message) }.to raise_error(NotImplementedError)
|
34
|
+
expect { consumer.complete(message, nil) }.to raise_error(NotImplementedError)
|
35
35
|
end
|
36
36
|
end
|
@@ -121,7 +121,7 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
121
121
|
end
|
122
122
|
|
123
123
|
it "should consume a published job and yield the job to the handler block" do
|
124
|
-
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 0)
|
124
|
+
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, anything, 'test-queue', 60, test_job_hash.to_json, 0)
|
125
125
|
end
|
126
126
|
|
127
127
|
context "rejecting a job" do
|
@@ -136,7 +136,7 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
136
136
|
expect(rejected).to be true
|
137
137
|
|
138
138
|
Timecop.freeze(Time.now + 61) do
|
139
|
-
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 1)
|
139
|
+
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, anything, 'test-queue', 60, test_job_hash.to_json, 1)
|
140
140
|
end
|
141
141
|
end
|
142
142
|
end
|
@@ -145,12 +145,11 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
145
145
|
let!(:consumer_run_for_two_messages) { allow(consumer).to receive(:running?).and_return(true, false,true,false) }
|
146
146
|
|
147
147
|
it "should remove job on completion" do
|
148
|
-
|
148
|
+
|
149
149
|
consumer.consume do |job_id, queue_name, job_hash|
|
150
|
+
expect(File).to receive(:delete).with(kind_of(String))
|
150
151
|
consumer.complete(job_id)
|
151
|
-
completed = true
|
152
152
|
end
|
153
|
-
expect(completed).to be true
|
154
153
|
|
155
154
|
expect { |b| consumer.consume(&b) }.to_not yield_control
|
156
155
|
end
|
@@ -160,7 +159,7 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
160
159
|
let(:timeout) { 30 }
|
161
160
|
|
162
161
|
it "should consume a published job and yield the job to the handler block" do
|
163
|
-
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 30, test_job_hash.to_json, 0)
|
162
|
+
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, anything, 'test-queue', 30, test_job_hash.to_json, 0)
|
164
163
|
end
|
165
164
|
end
|
166
165
|
end
|
@@ -172,4 +171,3 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
172
171
|
end
|
173
172
|
end
|
174
173
|
end
|
175
|
-
|
@@ -1,131 +1,170 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Chore::Queues::SQS::Consumer do
|
4
|
-
|
5
|
-
|
6
|
-
let(:queues) { double("queues") }
|
7
|
-
let(:queue) { double("test_queue", :visibility_timeout=>10, :url=>"test_queue", :name=>"test_queue") }
|
4
|
+
include_context 'fake objects'
|
5
|
+
|
8
6
|
let(:options) { {} }
|
9
7
|
let(:consumer) { Chore::Queues::SQS::Consumer.new(queue_name) }
|
10
|
-
let(:
|
11
|
-
let(:
|
12
|
-
|
13
|
-
let(:
|
14
|
-
|
8
|
+
let(:job) { {'class' => 'TestJob', 'args'=>[1,2,'3']} }
|
9
|
+
let(:backoff_func) { Proc.new { 2 + 2 } }
|
10
|
+
|
11
|
+
let(:receive_message_result) { Aws::SQS::Message::Collection.new([message], size: 1) }
|
12
|
+
|
13
|
+
let(:message) do
|
14
|
+
Aws::SQS::Message.new(
|
15
|
+
message_id: 'message id',
|
16
|
+
receipt_handle: "receipt_handle",
|
17
|
+
body: job.to_json,
|
18
|
+
data: job,
|
19
|
+
queue: queue,
|
20
|
+
queue_url: queue_url,
|
21
|
+
)
|
22
|
+
end
|
15
23
|
|
16
|
-
|
17
|
-
|
18
|
-
|
24
|
+
# Since a message handler is required (but not validated), this convenience method lets us
|
25
|
+
# effectively stub the block.
|
26
|
+
def consume(&block)
|
27
|
+
block = Proc.new{} unless block_given?
|
28
|
+
consumer.consume(&block)
|
29
|
+
end
|
19
30
|
|
20
|
-
|
21
|
-
allow(
|
22
|
-
allow(
|
23
|
-
allow(
|
31
|
+
before do
|
32
|
+
allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
|
33
|
+
allow(Aws::SQS::Queue).to receive(:new).and_return(queue)
|
34
|
+
allow(queue).to receive(:receive_messages).and_return(receive_message_result)
|
35
|
+
allow(message).to receive(:attributes).and_return({ 'ApproximateReceiveCount' => rand(10) })
|
24
36
|
end
|
25
37
|
|
26
38
|
describe "consuming messages" do
|
27
|
-
|
28
|
-
|
29
|
-
let!(:queue_contain_messages) { allow(queue).to receive(:receive_messages).and_return(message) }
|
30
|
-
|
31
|
-
it 'should configure sqs' do
|
32
|
-
allow(Chore.config).to receive(:aws_access_key).and_return('key')
|
33
|
-
allow(Chore.config).to receive(:aws_secret_key).and_return('secret')
|
34
|
-
|
35
|
-
expect(AWS::SQS).to receive(:new).with(
|
36
|
-
:access_key_id => 'key',
|
37
|
-
:secret_access_key => 'secret'
|
38
|
-
).and_return(sqs)
|
39
|
-
consumer.consume
|
39
|
+
before do
|
40
|
+
allow(consumer).to receive(:running?).and_return(true, false)
|
40
41
|
end
|
41
42
|
|
42
|
-
|
43
|
-
|
43
|
+
context "should create objects for interacting with the SQS API" do
|
44
|
+
it 'should create an sqs client' do
|
45
|
+
expect(queue).to receive(:receive_messages)
|
46
|
+
consume
|
47
|
+
end
|
44
48
|
|
45
|
-
|
46
|
-
|
47
|
-
|
49
|
+
it "should only create an sqs client when one doesn't exist" do
|
50
|
+
allow(consumer).to receive(:running?).and_return(true, true, true, true, false, true, true)
|
51
|
+
expect(Aws::SQS::Client).to receive(:new).exactly(:once)
|
52
|
+
consume
|
53
|
+
end
|
48
54
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
55
|
+
it 'should look up the queue url based on the queue name' do
|
56
|
+
expect(sqs).to receive(:get_queue_url).with(queue_name: queue_name)
|
57
|
+
consume
|
58
|
+
end
|
53
59
|
|
54
|
-
|
55
|
-
|
56
|
-
|
60
|
+
it 'should create a queue object' do
|
61
|
+
expect(consumer.send(:queue)).to_not be_nil
|
62
|
+
consume
|
63
|
+
end
|
57
64
|
end
|
58
65
|
|
59
66
|
context "should receive a message from the queue" do
|
60
|
-
|
61
67
|
it 'should use the default size of 10 when no queue_polling_size is specified' do
|
62
|
-
expect(queue).to receive(:receive_messages).with(
|
63
|
-
|
68
|
+
expect(queue).to receive(:receive_messages).with(
|
69
|
+
:max_number_of_messages => 10,
|
70
|
+
:attribute_names => ['ApproximateReceiveCount']
|
71
|
+
).and_return(message)
|
72
|
+
consume
|
64
73
|
end
|
65
74
|
|
66
75
|
it 'should respect the queue_polling_size when specified' do
|
67
76
|
allow(Chore.config).to receive(:queue_polling_size).and_return(5)
|
68
|
-
expect(queue).to receive(:receive_messages).with(
|
69
|
-
|
77
|
+
expect(queue).to receive(:receive_messages).with(
|
78
|
+
:max_number_of_messages => 5,
|
79
|
+
:attribute_names => ['ApproximateReceiveCount']
|
80
|
+
)
|
81
|
+
consume
|
70
82
|
end
|
71
83
|
end
|
72
84
|
|
73
|
-
it "should check the uniqueness of the message" do
|
74
|
-
allow_any_instance_of(Chore::DuplicateDetector).to receive(:found_duplicate?).with(message_data).and_return(false)
|
75
|
-
consumer.consume
|
76
|
-
end
|
77
|
-
|
78
|
-
it "should yield the message to the handler block" do
|
79
|
-
expect { |b| consumer.consume(&b) }.to yield_with_args('handle', queue_name, 10, 'message body', 0)
|
80
|
-
end
|
81
|
-
|
82
|
-
it 'should not yield for a dupe message' do
|
83
|
-
allow_any_instance_of(Chore::DuplicateDetector).to receive(:found_duplicate?).with(message_data).and_return(true)
|
84
|
-
expect {|b| consumer.consume(&b) }.not_to yield_control
|
85
|
-
end
|
86
|
-
|
87
85
|
context 'with no messages' do
|
88
|
-
|
89
|
-
|
86
|
+
before do
|
87
|
+
allow(consumer).to receive(:handle_messages).and_return([])
|
88
|
+
end
|
90
89
|
|
91
90
|
it 'should sleep' do
|
92
91
|
expect(consumer).to receive(:sleep).with(1)
|
93
|
-
|
92
|
+
consume
|
94
93
|
end
|
95
94
|
end
|
96
95
|
|
97
96
|
context 'with messages' do
|
98
|
-
|
99
|
-
|
97
|
+
before do
|
98
|
+
allow(consumer).to receive(:duplicate_message?).and_return(false)
|
99
|
+
allow(queue).to receive(:receive_messages).and_return(message)
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should check the uniqueness of the message" do
|
103
|
+
expect(consumer).to receive(:duplicate_message?)
|
104
|
+
consume
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should yield the message to the handler block" do
|
108
|
+
expect { |b| consume(&b) }
|
109
|
+
.to yield_with_args(
|
110
|
+
message.message_id,
|
111
|
+
message.receipt_handle,
|
112
|
+
queue_name,
|
113
|
+
queue.attributes['VisibilityTimeout'].to_i,
|
114
|
+
message.body,
|
115
|
+
message.attributes['ApproximateReceiveCount'].to_i - 1
|
116
|
+
)
|
117
|
+
end
|
100
118
|
|
101
119
|
it 'should not sleep' do
|
102
120
|
expect(consumer).to_not receive(:sleep)
|
103
|
-
|
121
|
+
consume
|
104
122
|
end
|
123
|
+
|
124
|
+
context 'with duplicates' do
|
125
|
+
before do
|
126
|
+
allow(consumer).to receive(:duplicate_message?).and_return(true)
|
127
|
+
end
|
128
|
+
|
129
|
+
it 'should not yield for a dupe message' do
|
130
|
+
expect {|b| consume(&b) }.not_to yield_control
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
describe "completing work" do
|
137
|
+
it 'deletes the message from the queue' do
|
138
|
+
expect(queue).to receive(:delete_messages).with(entries: [{id: message.message_id, receipt_handle: message.receipt_handle}])
|
139
|
+
consumer.complete(message.message_id, message.receipt_handle)
|
105
140
|
end
|
106
141
|
end
|
107
142
|
|
108
143
|
describe '#delay' do
|
109
|
-
let(:item) { Chore::UnitOfWork.new(message.
|
110
|
-
let(:
|
144
|
+
let(:item) { Chore::UnitOfWork.new(message.message_id, message.receipt_handle, message.queue, 60, message.body, 0, consumer) }
|
145
|
+
let(:entries) do
|
146
|
+
[
|
147
|
+
{ id: item.id, receipt_handle: item.receipt_handle, visibility_timeout: backoff_func.call(item) },
|
148
|
+
]
|
149
|
+
end
|
111
150
|
|
112
151
|
it 'changes the visiblity of the message' do
|
113
|
-
expect(queue).to receive(:
|
152
|
+
expect(queue).to receive(:change_message_visibility_batch).with(entries: entries)
|
114
153
|
consumer.delay(item, backoff_func)
|
115
154
|
end
|
116
155
|
end
|
117
156
|
|
118
157
|
describe '#reset_connection!' do
|
119
158
|
it 'should reset the connection after a call to reset_connection!' do
|
120
|
-
expect(
|
121
|
-
expect(pool).to receive(:empty!)
|
159
|
+
expect(Aws).to receive(:empty_connection_pools!)
|
122
160
|
Chore::Queues::SQS::Consumer.reset_connection!
|
123
161
|
consumer.send(:queue)
|
124
162
|
end
|
125
163
|
|
126
164
|
it 'should not reset the connection between calls' do
|
127
|
-
|
128
|
-
|
165
|
+
expect(Aws).to receive(:empty_connection_pools!).once
|
166
|
+
q = consumer.send(:queue)
|
167
|
+
expect(consumer.send(:queue)).to be(q)
|
129
168
|
end
|
130
169
|
|
131
170
|
it 'should reconfigure sqs' do
|
@@ -133,13 +172,15 @@ describe Chore::Queues::SQS::Consumer do
|
|
133
172
|
allow_any_instance_of(Chore::DuplicateDetector).to receive(:found_duplicate?).and_return(false)
|
134
173
|
|
135
174
|
allow(queue).to receive(:receive_messages).and_return(message)
|
136
|
-
|
175
|
+
allow(sqs).to receive(:receive_message).with({:attribute_names=>["ApproximateReceiveCount"], :max_number_of_messages=>10, :queue_url=>queue_url})
|
176
|
+
|
177
|
+
consume
|
137
178
|
|
138
179
|
Chore::Queues::SQS::Consumer.reset_connection!
|
139
|
-
allow(
|
180
|
+
allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
|
140
181
|
|
141
182
|
expect(consumer).to receive(:running?).and_return(true, false)
|
142
|
-
|
183
|
+
consume
|
143
184
|
end
|
144
185
|
end
|
145
186
|
end
|