chore-core 3.2.3 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/LICENSE.txt +1 -1
- data/README.md +170 -153
- data/chore-core.gemspec +2 -3
- data/lib/chore.rb +20 -0
- data/lib/chore/cli.rb +1 -2
- data/lib/chore/configuration.rb +1 -1
- data/lib/chore/consumer.rb +41 -9
- data/lib/chore/job.rb +2 -0
- data/lib/chore/publisher.rb +18 -2
- data/lib/chore/queues/filesystem/consumer.rb +18 -13
- data/lib/chore/queues/filesystem/publisher.rb +1 -1
- data/lib/chore/queues/sqs.rb +22 -13
- data/lib/chore/queues/sqs/consumer.rb +61 -33
- data/lib/chore/queues/sqs/publisher.rb +26 -17
- data/lib/chore/strategies/consumer/batcher.rb +6 -6
- data/lib/chore/strategies/consumer/single_consumer_strategy.rb +5 -5
- data/lib/chore/strategies/consumer/threaded_consumer_strategy.rb +6 -6
- data/lib/chore/strategies/consumer/throttled_consumer_strategy.rb +10 -11
- data/lib/chore/strategies/worker/helpers/ipc.rb +0 -1
- data/lib/chore/unit_of_work.rb +2 -1
- data/lib/chore/version.rb +3 -3
- data/lib/chore/worker.rb +4 -4
- data/spec/chore/consumer_spec.rb +1 -1
- data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb +5 -7
- data/spec/chore/queues/sqs/consumer_spec.rb +117 -76
- data/spec/chore/queues/sqs/publisher_spec.rb +49 -60
- data/spec/chore/queues/sqs_spec.rb +32 -41
- data/spec/chore/strategies/consumer/single_consumer_strategy_spec.rb +3 -3
- data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb +6 -6
- data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb +1 -1
- data/spec/chore/strategies/worker/single_worker_strategy_spec.rb +1 -1
- data/spec/chore/worker_spec.rb +21 -21
- data/spec/spec_helper.rb +1 -1
- data/spec/support/queues/sqs/fake_objects.rb +18 -0
- metadata +9 -13
@@ -3,21 +3,26 @@ require 'chore/publisher'
|
|
3
3
|
module Chore
|
4
4
|
module Queues
|
5
5
|
module SQS
|
6
|
-
|
7
6
|
# SQS Publisher, for writing messages to SQS from Chore
|
8
7
|
class Publisher < Chore::Publisher
|
9
8
|
@@reset_next = true
|
10
9
|
|
10
|
+
# @param [Hash] opts Publisher options
|
11
11
|
def initialize(opts={})
|
12
12
|
super
|
13
13
|
@sqs_queues = {}
|
14
14
|
@sqs_queue_urls = {}
|
15
15
|
end
|
16
16
|
|
17
|
-
#
|
17
|
+
# Publishes a message to an SQS queue
|
18
|
+
#
|
19
|
+
# @param [String] queue_name Name of the SQS queue
|
20
|
+
# @param [Hash] job Job instance definition, will be encoded to JSON
|
21
|
+
#
|
22
|
+
# @return [struct Aws::SQS::Types::SendMessageResult]
|
18
23
|
def publish(queue_name,job)
|
19
|
-
queue =
|
20
|
-
queue.send_message(encode_job(job))
|
24
|
+
queue = queue(queue_name)
|
25
|
+
queue.send_message(message_body: encode_job(job))
|
21
26
|
end
|
22
27
|
|
23
28
|
# Sets a flag that instructs the publisher to reset the connection the next time it's used
|
@@ -25,29 +30,33 @@ module Chore
|
|
25
30
|
@@reset_next = true
|
26
31
|
end
|
27
32
|
|
28
|
-
|
33
|
+
private
|
34
|
+
|
35
|
+
# SQS API client object
|
36
|
+
#
|
37
|
+
# @return [Aws::SQS::Client]
|
29
38
|
def sqs
|
30
|
-
@sqs ||=
|
31
|
-
:access_key_id => Chore.config.aws_access_key,
|
32
|
-
:secret_access_key => Chore.config.aws_secret_key,
|
33
|
-
:logger => Chore.logger,
|
34
|
-
:log_level => :debug)
|
39
|
+
@sqs ||= Chore::Queues::SQS.sqs_client
|
35
40
|
end
|
36
41
|
|
37
|
-
# Retrieves the SQS queue
|
42
|
+
# Retrieves the SQS queue object. The method will cache the results to prevent round trips on subsequent calls
|
43
|
+
#
|
38
44
|
# If <tt>reset_connection!</tt> has been called, this will result in the connection being re-initialized,
|
39
45
|
# as well as clear any cached results from prior calls
|
46
|
+
#
|
47
|
+
# @param [String] name Name of SQS queue
|
48
|
+
#
|
49
|
+
# @return [Aws::SQS::Queue]
|
40
50
|
def queue(name)
|
41
|
-
|
42
|
-
|
43
|
-
p.empty!
|
44
|
-
end
|
51
|
+
if @@reset_next
|
52
|
+
Aws.empty_connection_pools!
|
45
53
|
@sqs = nil
|
46
54
|
@@reset_next = false
|
47
55
|
@sqs_queues = {}
|
48
56
|
end
|
49
|
-
|
50
|
-
@
|
57
|
+
|
58
|
+
@sqs_queue_urls[name] ||= sqs.get_queue_url(queue_name: name).queue_url
|
59
|
+
@sqs_queues[name] ||= Aws::SQS::Queue.new(url: @sqs_queue_urls[name], client: sqs)
|
51
60
|
end
|
52
61
|
end
|
53
62
|
end
|
@@ -15,17 +15,17 @@ module Chore
|
|
15
15
|
@running = true
|
16
16
|
end
|
17
17
|
|
18
|
-
# The main entry point of the Batcher, <tt>schedule</tt> begins a thread with the provided +batch_timeout+
|
19
|
-
# as the only argument. While the Batcher is running, it will attempt to check if either the batch is full,
|
18
|
+
# The main entry point of the Batcher, <tt>schedule</tt> begins a thread with the provided +batch_timeout+
|
19
|
+
# as the only argument. While the Batcher is running, it will attempt to check if either the batch is full,
|
20
20
|
# or if the +batch_timeout+ has elapsed since the oldest message was added. If either case is true, the
|
21
21
|
# items in the batch will be executed.
|
22
|
-
#
|
22
|
+
#
|
23
23
|
# Calling <tt>stop</tt> will cause the thread to finish it's current check, and exit
|
24
24
|
def schedule(batch_timeout)
|
25
25
|
@thread = Thread.new(batch_timeout) do |timeout|
|
26
|
-
Chore.logger.info "Batching
|
26
|
+
Chore.logger.info "Batching thread starting with #{batch_timeout} second timeout"
|
27
27
|
while @running do
|
28
|
-
begin
|
28
|
+
begin
|
29
29
|
oldest_item = @batch.first
|
30
30
|
timestamp = oldest_item && oldest_item.created_at
|
31
31
|
Chore.logger.debug "Oldest message in batch: #{timestamp}, size: #{@batch.size}"
|
@@ -33,7 +33,7 @@ module Chore
|
|
33
33
|
Chore.logger.debug "Batching timeout reached (#{timestamp + timeout}), current size: #{@batch.size}"
|
34
34
|
self.execute(true)
|
35
35
|
end
|
36
|
-
sleep(1)
|
36
|
+
sleep(1)
|
37
37
|
rescue => e
|
38
38
|
Chore.logger.error "Batcher#schedule raised an exception: #{e.inspect}"
|
39
39
|
end
|
@@ -10,16 +10,16 @@ module Chore
|
|
10
10
|
end
|
11
11
|
|
12
12
|
# Begins fetching from the configured queue by way of the configured Consumer. This can only be used if you have a
|
13
|
-
# single queue which can be kept up with at a relatively low volume. If you have more than a single queue
|
14
|
-
# it will raise an exception.
|
13
|
+
# single queue which can be kept up with at a relatively low volume. If you have more than a single queue
|
14
|
+
# configured, it will raise an exception.
|
15
15
|
def fetch
|
16
16
|
Chore.logger.debug "Starting up consumer strategy: #{self.class.name}"
|
17
17
|
queues = Chore.config.queues
|
18
18
|
raise "When using SingleConsumerStrategy only one queue can be defined. Queues: #{queues}" unless queues.size == 1
|
19
|
-
|
19
|
+
|
20
20
|
@consumer = Chore.config.consumer.new(queues.first)
|
21
|
-
@consumer.consume do |
|
22
|
-
work = UnitOfWork.new(
|
21
|
+
@consumer.consume do |message_id, message_receipt_handle, queue_name, queue_timeout, body, previous_attempts|
|
22
|
+
work = UnitOfWork.new(message_id, message_receipt_handle, queue_name, queue_timeout, body, previous_attempts, @consumer)
|
23
23
|
@fetcher.manager.assign(work)
|
24
24
|
end
|
25
25
|
end
|
@@ -23,7 +23,7 @@ module Chore
|
|
23
23
|
Chore.logger.debug "Starting up consumer strategy: #{self.class.name}"
|
24
24
|
threads = []
|
25
25
|
Chore.config.queues.each do |queue|
|
26
|
-
Chore.config.threads_per_queue.times do
|
26
|
+
Chore.config.threads_per_queue.times do
|
27
27
|
if running?
|
28
28
|
threads << start_consumer_thread(queue)
|
29
29
|
end
|
@@ -32,7 +32,7 @@ module Chore
|
|
32
32
|
|
33
33
|
threads.each(&:join)
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
# If the ThreadedConsumerStrategy is currently running <tt>stop!</tt> will begin signalling it to stop
|
37
37
|
# It will stop the batcher from forking more work, as well as set a flag which will disable it's own consuming
|
38
38
|
# threads once they finish with their current work.
|
@@ -49,21 +49,21 @@ module Chore
|
|
49
49
|
@running
|
50
50
|
end
|
51
51
|
|
52
|
-
private
|
52
|
+
private
|
53
53
|
# Starts a consumer thread for polling the given +queue+.
|
54
54
|
# If <tt>stop!<tt> is called, the threads will shut themsevles down.
|
55
55
|
def start_consumer_thread(queue)
|
56
56
|
t = Thread.new(queue) do |tQueue|
|
57
57
|
begin
|
58
58
|
consumer = Chore.config.consumer.new(tQueue)
|
59
|
-
consumer.consume do |
|
59
|
+
consumer.consume do |message_id, message_receipt_handle, queue_name, queue_timeout, body, previous_attempts|
|
60
60
|
# Quick hack to force this thread to end it's work
|
61
61
|
# if we're shutting down. Could be delayed due to the
|
62
62
|
# weird sometimes-blocking nature of SQS.
|
63
63
|
consumer.stop if !running?
|
64
|
-
Chore.logger.debug { "Got message: #{
|
64
|
+
Chore.logger.debug { "Got message: #{message_id}"}
|
65
65
|
|
66
|
-
work = UnitOfWork.new(
|
66
|
+
work = UnitOfWork.new(message_id, message_receipt_handle, queue_name, queue_timeout, body, previous_attempts, consumer)
|
67
67
|
Chore.run_hooks_for(:consumed_from_source, work)
|
68
68
|
@batcher.add(work)
|
69
69
|
end
|
@@ -65,7 +65,7 @@ module Chore
|
|
65
65
|
end
|
66
66
|
|
67
67
|
# Gives work back to the queue in case it couldn't be assigned
|
68
|
-
#
|
68
|
+
#
|
69
69
|
# This will go into a separate queue so that it will be prioritized
|
70
70
|
# over other work that hasn't been attempted yet. It also avoids
|
71
71
|
# a deadlock where @queue is full and the master is waiting to return
|
@@ -100,22 +100,21 @@ module Chore
|
|
100
100
|
end
|
101
101
|
|
102
102
|
def create_work_units(consumer)
|
103
|
-
consumer.consume do |
|
104
|
-
# Note: The unit of work object contains a consumer object that when
|
105
|
-
# used to consume from SQS, would have a mutex (that comes as a part
|
106
|
-
# of the AWS sdk); When sending these objects across from one process
|
107
|
-
# to another, we cannot send this across (becasue of the mutex). To
|
103
|
+
consumer.consume do |message_id, message_receipt_handle, queue, timeout, body, previous_attempts|
|
104
|
+
# Note: The unit of work object contains a consumer object that when
|
105
|
+
# used to consume from SQS, would have a mutex (that comes as a part
|
106
|
+
# of the AWS sdk); When sending these objects across from one process
|
107
|
+
# to another, we cannot send this across (becasue of the mutex). To
|
108
108
|
# work around this, we simply ignore the consumer object when creating
|
109
|
-
# the unit of work object, and when the worker recieves the work
|
110
|
-
# object, it assigns it a consumer object.
|
109
|
+
# the unit of work object, and when the worker recieves the work
|
110
|
+
# object, it assigns it a consumer object.
|
111
111
|
# (to allow for communication back to the queue it was consumed from)
|
112
|
-
work = UnitOfWork.new(
|
113
|
-
previous_attempts)
|
112
|
+
work = UnitOfWork.new(message_id, message_receipt_handle, queue, timeout, body, previous_attempts)
|
114
113
|
Chore.run_hooks_for(:consumed_from_source, work)
|
115
114
|
@queue.push(work) if running?
|
116
115
|
Chore.run_hooks_for(:added_to_queue, work)
|
117
116
|
end
|
118
117
|
end
|
119
|
-
end #
|
118
|
+
end # ThrottledConsumerStrategy
|
120
119
|
end
|
121
120
|
end # Chore
|
data/lib/chore/unit_of_work.rb
CHANGED
@@ -2,13 +2,14 @@ module Chore
|
|
2
2
|
# Simple class to hold job processing information.
|
3
3
|
# Has six attributes:
|
4
4
|
# * +:id+ The queue implementation specific identifier for this message.
|
5
|
+
# * +:receipt_handle+ The queue implementation specific identifier for the receipt of this message.
|
5
6
|
# * +:queue_name+ The name of the queue the job came from
|
6
7
|
# * +:queue_timeout+ The time (in seconds) before the job will get re-enqueued if not processed
|
7
8
|
# * +:message+ The actual data of the message.
|
8
9
|
# * +:previous_attempts+ The number of times the work has been attempted previously.
|
9
10
|
# * +:consumer+ The consumer instance used to fetch this message. Most queue implementations won't need access to this, but some (RabbitMQ) will. So we
|
10
11
|
# make sure to pass it along with each message. This instance will be used by the Worker for things like <tt>complete</tt> and </tt>reject</tt>.
|
11
|
-
class UnitOfWork < Struct.new(:id
|
12
|
+
class UnitOfWork < Struct.new(:id, :receipt_handle, :queue_name, :queue_timeout, :message, :previous_attempts, :consumer, :decoded_message, :klass)
|
12
13
|
# The time at which this unit of work was created
|
13
14
|
attr_accessor :created_at
|
14
15
|
|
data/lib/chore/version.rb
CHANGED
data/lib/chore/worker.rb
CHANGED
@@ -56,7 +56,7 @@ module Chore
|
|
56
56
|
|
57
57
|
if item.consumer.duplicate_message?(dedupe_key, item.klass, item.queue_timeout)
|
58
58
|
Chore.logger.info { "Found and deleted duplicate job #{item.klass}"}
|
59
|
-
item.consumer.complete(item.id)
|
59
|
+
item.consumer.complete(item.id, item.receipt_handle)
|
60
60
|
return true
|
61
61
|
end
|
62
62
|
end
|
@@ -89,7 +89,7 @@ module Chore
|
|
89
89
|
Chore.logger.error { "Failed to run job for #{item.message} with error: #{e.message} #{e.backtrace * "\n"}" }
|
90
90
|
if item.current_attempt >= Chore.config.max_attempts
|
91
91
|
Chore.run_hooks_for(:on_permanent_failure,item.queue_name,item.message,e)
|
92
|
-
item.consumer.complete(item.id)
|
92
|
+
item.consumer.complete(item.id, item.receipt_handle)
|
93
93
|
else
|
94
94
|
Chore.run_hooks_for(:on_failure,item.message,e)
|
95
95
|
item.consumer.reject(item.id)
|
@@ -112,7 +112,7 @@ module Chore
|
|
112
112
|
begin
|
113
113
|
Chore.logger.info { "Running job #{klass} with params #{message}"}
|
114
114
|
perform_job(klass,message)
|
115
|
-
item.consumer.complete(item.id)
|
115
|
+
item.consumer.complete(item.id, item.receipt_handle)
|
116
116
|
Chore.logger.info { "Finished job #{klass} with params #{message}"}
|
117
117
|
klass.run_hooks_for(:after_perform, message)
|
118
118
|
Chore.run_hooks_for(:worker_ended, item)
|
@@ -141,7 +141,7 @@ module Chore
|
|
141
141
|
Chore.logger.error { "Failed to run job #{item.message} with error: #{e.message} at #{e.backtrace * "\n"}" }
|
142
142
|
if item.current_attempt >= klass.options[:max_attempts]
|
143
143
|
klass.run_hooks_for(:on_permanent_failure,item.queue_name,message,e)
|
144
|
-
item.consumer.complete(item.id)
|
144
|
+
item.consumer.complete(item.id, item.receipt_handle)
|
145
145
|
else
|
146
146
|
klass.run_hooks_for(:on_failure, message, e)
|
147
147
|
item.consumer.reject(item.id)
|
data/spec/chore/consumer_spec.rb
CHANGED
@@ -31,6 +31,6 @@ describe Chore::Consumer do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
it 'should not have an implemented complete method' do
|
34
|
-
expect { consumer.complete(message) }.to raise_error(NotImplementedError)
|
34
|
+
expect { consumer.complete(message, nil) }.to raise_error(NotImplementedError)
|
35
35
|
end
|
36
36
|
end
|
@@ -121,7 +121,7 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
121
121
|
end
|
122
122
|
|
123
123
|
it "should consume a published job and yield the job to the handler block" do
|
124
|
-
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 0)
|
124
|
+
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, anything, 'test-queue', 60, test_job_hash.to_json, 0)
|
125
125
|
end
|
126
126
|
|
127
127
|
context "rejecting a job" do
|
@@ -136,7 +136,7 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
136
136
|
expect(rejected).to be true
|
137
137
|
|
138
138
|
Timecop.freeze(Time.now + 61) do
|
139
|
-
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 1)
|
139
|
+
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, anything, 'test-queue', 60, test_job_hash.to_json, 1)
|
140
140
|
end
|
141
141
|
end
|
142
142
|
end
|
@@ -145,12 +145,11 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
145
145
|
let!(:consumer_run_for_two_messages) { allow(consumer).to receive(:running?).and_return(true, false,true,false) }
|
146
146
|
|
147
147
|
it "should remove job on completion" do
|
148
|
-
|
148
|
+
|
149
149
|
consumer.consume do |job_id, queue_name, job_hash|
|
150
|
+
expect(File).to receive(:delete).with(kind_of(String))
|
150
151
|
consumer.complete(job_id)
|
151
|
-
completed = true
|
152
152
|
end
|
153
|
-
expect(completed).to be true
|
154
153
|
|
155
154
|
expect { |b| consumer.consume(&b) }.to_not yield_control
|
156
155
|
end
|
@@ -160,7 +159,7 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
160
159
|
let(:timeout) { 30 }
|
161
160
|
|
162
161
|
it "should consume a published job and yield the job to the handler block" do
|
163
|
-
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 30, test_job_hash.to_json, 0)
|
162
|
+
expect { |b| consumer.consume(&b) }.to yield_with_args(anything, anything, 'test-queue', 30, test_job_hash.to_json, 0)
|
164
163
|
end
|
165
164
|
end
|
166
165
|
end
|
@@ -172,4 +171,3 @@ describe Chore::Queues::Filesystem::Consumer do
|
|
172
171
|
end
|
173
172
|
end
|
174
173
|
end
|
175
|
-
|
@@ -1,131 +1,170 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Chore::Queues::SQS::Consumer do
|
4
|
-
|
5
|
-
|
6
|
-
let(:queues) { double("queues") }
|
7
|
-
let(:queue) { double("test_queue", :visibility_timeout=>10, :url=>"test_queue", :name=>"test_queue") }
|
4
|
+
include_context 'fake objects'
|
5
|
+
|
8
6
|
let(:options) { {} }
|
9
7
|
let(:consumer) { Chore::Queues::SQS::Consumer.new(queue_name) }
|
10
|
-
let(:
|
11
|
-
let(:
|
12
|
-
|
13
|
-
let(:
|
14
|
-
|
8
|
+
let(:job) { {'class' => 'TestJob', 'args'=>[1,2,'3']} }
|
9
|
+
let(:backoff_func) { Proc.new { 2 + 2 } }
|
10
|
+
|
11
|
+
let(:receive_message_result) { Aws::SQS::Message::Collection.new([message], size: 1) }
|
12
|
+
|
13
|
+
let(:message) do
|
14
|
+
Aws::SQS::Message.new(
|
15
|
+
message_id: 'message id',
|
16
|
+
receipt_handle: "receipt_handle",
|
17
|
+
body: job.to_json,
|
18
|
+
data: job,
|
19
|
+
queue: queue,
|
20
|
+
queue_url: queue_url,
|
21
|
+
)
|
22
|
+
end
|
15
23
|
|
16
|
-
|
17
|
-
|
18
|
-
|
24
|
+
# Since a message handler is required (but not validated), this convenience method lets us
|
25
|
+
# effectively stub the block.
|
26
|
+
def consume(&block)
|
27
|
+
block = Proc.new{} unless block_given?
|
28
|
+
consumer.consume(&block)
|
29
|
+
end
|
19
30
|
|
20
|
-
|
21
|
-
allow(
|
22
|
-
allow(
|
23
|
-
allow(
|
31
|
+
before do
|
32
|
+
allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
|
33
|
+
allow(Aws::SQS::Queue).to receive(:new).and_return(queue)
|
34
|
+
allow(queue).to receive(:receive_messages).and_return(receive_message_result)
|
35
|
+
allow(message).to receive(:attributes).and_return({ 'ApproximateReceiveCount' => rand(10) })
|
24
36
|
end
|
25
37
|
|
26
38
|
describe "consuming messages" do
|
27
|
-
|
28
|
-
|
29
|
-
let!(:queue_contain_messages) { allow(queue).to receive(:receive_messages).and_return(message) }
|
30
|
-
|
31
|
-
it 'should configure sqs' do
|
32
|
-
allow(Chore.config).to receive(:aws_access_key).and_return('key')
|
33
|
-
allow(Chore.config).to receive(:aws_secret_key).and_return('secret')
|
34
|
-
|
35
|
-
expect(AWS::SQS).to receive(:new).with(
|
36
|
-
:access_key_id => 'key',
|
37
|
-
:secret_access_key => 'secret'
|
38
|
-
).and_return(sqs)
|
39
|
-
consumer.consume
|
39
|
+
before do
|
40
|
+
allow(consumer).to receive(:running?).and_return(true, false)
|
40
41
|
end
|
41
42
|
|
42
|
-
|
43
|
-
|
43
|
+
context "should create objects for interacting with the SQS API" do
|
44
|
+
it 'should create an sqs client' do
|
45
|
+
expect(queue).to receive(:receive_messages)
|
46
|
+
consume
|
47
|
+
end
|
44
48
|
|
45
|
-
|
46
|
-
|
47
|
-
|
49
|
+
it "should only create an sqs client when one doesn't exist" do
|
50
|
+
allow(consumer).to receive(:running?).and_return(true, true, true, true, false, true, true)
|
51
|
+
expect(Aws::SQS::Client).to receive(:new).exactly(:once)
|
52
|
+
consume
|
53
|
+
end
|
48
54
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
55
|
+
it 'should look up the queue url based on the queue name' do
|
56
|
+
expect(sqs).to receive(:get_queue_url).with(queue_name: queue_name)
|
57
|
+
consume
|
58
|
+
end
|
53
59
|
|
54
|
-
|
55
|
-
|
56
|
-
|
60
|
+
it 'should create a queue object' do
|
61
|
+
expect(consumer.send(:queue)).to_not be_nil
|
62
|
+
consume
|
63
|
+
end
|
57
64
|
end
|
58
65
|
|
59
66
|
context "should receive a message from the queue" do
|
60
|
-
|
61
67
|
it 'should use the default size of 10 when no queue_polling_size is specified' do
|
62
|
-
expect(queue).to receive(:receive_messages).with(
|
63
|
-
|
68
|
+
expect(queue).to receive(:receive_messages).with(
|
69
|
+
:max_number_of_messages => 10,
|
70
|
+
:attribute_names => ['ApproximateReceiveCount']
|
71
|
+
).and_return(message)
|
72
|
+
consume
|
64
73
|
end
|
65
74
|
|
66
75
|
it 'should respect the queue_polling_size when specified' do
|
67
76
|
allow(Chore.config).to receive(:queue_polling_size).and_return(5)
|
68
|
-
expect(queue).to receive(:receive_messages).with(
|
69
|
-
|
77
|
+
expect(queue).to receive(:receive_messages).with(
|
78
|
+
:max_number_of_messages => 5,
|
79
|
+
:attribute_names => ['ApproximateReceiveCount']
|
80
|
+
)
|
81
|
+
consume
|
70
82
|
end
|
71
83
|
end
|
72
84
|
|
73
|
-
it "should check the uniqueness of the message" do
|
74
|
-
allow_any_instance_of(Chore::DuplicateDetector).to receive(:found_duplicate?).with(message_data).and_return(false)
|
75
|
-
consumer.consume
|
76
|
-
end
|
77
|
-
|
78
|
-
it "should yield the message to the handler block" do
|
79
|
-
expect { |b| consumer.consume(&b) }.to yield_with_args('handle', queue_name, 10, 'message body', 0)
|
80
|
-
end
|
81
|
-
|
82
|
-
it 'should not yield for a dupe message' do
|
83
|
-
allow_any_instance_of(Chore::DuplicateDetector).to receive(:found_duplicate?).with(message_data).and_return(true)
|
84
|
-
expect {|b| consumer.consume(&b) }.not_to yield_control
|
85
|
-
end
|
86
|
-
|
87
85
|
context 'with no messages' do
|
88
|
-
|
89
|
-
|
86
|
+
before do
|
87
|
+
allow(consumer).to receive(:handle_messages).and_return([])
|
88
|
+
end
|
90
89
|
|
91
90
|
it 'should sleep' do
|
92
91
|
expect(consumer).to receive(:sleep).with(1)
|
93
|
-
|
92
|
+
consume
|
94
93
|
end
|
95
94
|
end
|
96
95
|
|
97
96
|
context 'with messages' do
|
98
|
-
|
99
|
-
|
97
|
+
before do
|
98
|
+
allow(consumer).to receive(:duplicate_message?).and_return(false)
|
99
|
+
allow(queue).to receive(:receive_messages).and_return(message)
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should check the uniqueness of the message" do
|
103
|
+
expect(consumer).to receive(:duplicate_message?)
|
104
|
+
consume
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should yield the message to the handler block" do
|
108
|
+
expect { |b| consume(&b) }
|
109
|
+
.to yield_with_args(
|
110
|
+
message.message_id,
|
111
|
+
message.receipt_handle,
|
112
|
+
queue_name,
|
113
|
+
queue.attributes['VisibilityTimeout'].to_i,
|
114
|
+
message.body,
|
115
|
+
message.attributes['ApproximateReceiveCount'].to_i - 1
|
116
|
+
)
|
117
|
+
end
|
100
118
|
|
101
119
|
it 'should not sleep' do
|
102
120
|
expect(consumer).to_not receive(:sleep)
|
103
|
-
|
121
|
+
consume
|
104
122
|
end
|
123
|
+
|
124
|
+
context 'with duplicates' do
|
125
|
+
before do
|
126
|
+
allow(consumer).to receive(:duplicate_message?).and_return(true)
|
127
|
+
end
|
128
|
+
|
129
|
+
it 'should not yield for a dupe message' do
|
130
|
+
expect {|b| consume(&b) }.not_to yield_control
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
describe "completing work" do
|
137
|
+
it 'deletes the message from the queue' do
|
138
|
+
expect(queue).to receive(:delete_messages).with(entries: [{id: message.message_id, receipt_handle: message.receipt_handle}])
|
139
|
+
consumer.complete(message.message_id, message.receipt_handle)
|
105
140
|
end
|
106
141
|
end
|
107
142
|
|
108
143
|
describe '#delay' do
|
109
|
-
let(:item) { Chore::UnitOfWork.new(message.
|
110
|
-
let(:
|
144
|
+
let(:item) { Chore::UnitOfWork.new(message.message_id, message.receipt_handle, message.queue, 60, message.body, 0, consumer) }
|
145
|
+
let(:entries) do
|
146
|
+
[
|
147
|
+
{ id: item.id, receipt_handle: item.receipt_handle, visibility_timeout: backoff_func.call(item) },
|
148
|
+
]
|
149
|
+
end
|
111
150
|
|
112
151
|
it 'changes the visiblity of the message' do
|
113
|
-
expect(queue).to receive(:
|
152
|
+
expect(queue).to receive(:change_message_visibility_batch).with(entries: entries)
|
114
153
|
consumer.delay(item, backoff_func)
|
115
154
|
end
|
116
155
|
end
|
117
156
|
|
118
157
|
describe '#reset_connection!' do
|
119
158
|
it 'should reset the connection after a call to reset_connection!' do
|
120
|
-
expect(
|
121
|
-
expect(pool).to receive(:empty!)
|
159
|
+
expect(Aws).to receive(:empty_connection_pools!)
|
122
160
|
Chore::Queues::SQS::Consumer.reset_connection!
|
123
161
|
consumer.send(:queue)
|
124
162
|
end
|
125
163
|
|
126
164
|
it 'should not reset the connection between calls' do
|
127
|
-
|
128
|
-
|
165
|
+
expect(Aws).to receive(:empty_connection_pools!).once
|
166
|
+
q = consumer.send(:queue)
|
167
|
+
expect(consumer.send(:queue)).to be(q)
|
129
168
|
end
|
130
169
|
|
131
170
|
it 'should reconfigure sqs' do
|
@@ -133,13 +172,15 @@ describe Chore::Queues::SQS::Consumer do
|
|
133
172
|
allow_any_instance_of(Chore::DuplicateDetector).to receive(:found_duplicate?).and_return(false)
|
134
173
|
|
135
174
|
allow(queue).to receive(:receive_messages).and_return(message)
|
136
|
-
|
175
|
+
allow(sqs).to receive(:receive_message).with({:attribute_names=>["ApproximateReceiveCount"], :max_number_of_messages=>10, :queue_url=>queue_url})
|
176
|
+
|
177
|
+
consume
|
137
178
|
|
138
179
|
Chore::Queues::SQS::Consumer.reset_connection!
|
139
|
-
allow(
|
180
|
+
allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
|
140
181
|
|
141
182
|
expect(consumer).to receive(:running?).and_return(true, false)
|
142
|
-
|
183
|
+
consume
|
143
184
|
end
|
144
185
|
end
|
145
186
|
end
|