sqewer 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +10 -0
- data/DETAILS.md +10 -0
- data/README.md +6 -0
- data/example.env +2 -1
- data/lib/sqewer/connection.rb +18 -15
- data/lib/sqewer/isolator.rb +32 -6
- data/lib/sqewer/version.rb +1 -1
- data/lib/sqewer/worker.rb +53 -52
- data/spec/sqewer/cli_spec.rb +5 -2
- data/spec/sqewer/connection_spec.rb +11 -15
- data/spec/sqewer/worker_spec.rb +38 -25
- data/sqewer.gemspec +4 -3
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7431fac224bee95320e7fa75a1503511a7175ff1
|
|
4
|
+
data.tar.gz: fdf4629e080fd39e0acc9ac7d1b941e9270e4348
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: efc4ae288acade906f671179499918ff130dc4d1feee736aa5a17b875e07d9e6c51958d87ce159ef1c6f020332804ac9ef8935f2f8106c27e7ba565512bd407d
|
|
7
|
+
data.tar.gz: c0fdf7e2640365bc2d8c8f8e0496375013c5febf50ebf1abd94ea28c5eb7c9655682b8bde976c628d724dbd9b55edb1d6a92c2d1f2ba7d27c925ff42ab26efe8
|
data/.travis.yml
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
rvm:
|
|
2
|
+
- 2.1.5
|
|
3
|
+
- 2.2.2
|
|
4
|
+
sudo: false
|
|
5
|
+
cache: bundler
|
|
6
|
+
env:
|
|
7
|
+
global:
|
|
8
|
+
- secure: E+uNoJmvjq5+3/PYUs2a1bRh7bCjAbcYEIKJzOVgIiI+gWB4dLpTKwVW/fqJbjCDIGPw9OwiwphN51zLEo0ie5AJIkavcjWRcdazKq8usokAz/A5Ilru5YvGXJ5UQtF8TJNsZfJGOGMJwL5d/MsDSd3/sjy88wrFiSeeNlHA06KDBrGdZXE4/dOb5Z84cSxPaD7nDaGP36V3Uv88X272oezODm9Mp+Fsf4goGoU9I9PBDD36pCXAlpURWpyWMKa2TzEXfgN43TYxta4p+bu6BA871df31j3BfVKwY1K/59eN9HqW8xWF9rgcgLo3NgQuaoqde3ryCE4jqKymjtG1/R5bdbQsn07R7/OagQMPVkbn07Aq+X6Q8GVTkrMl829z87QiMETNnzpsHl/1RD3DVnUo4k+XmqLbwW6NRl452j3eEHsiMvxf1vt8Qn1hu61qm98WGVNCGFWX/eAu/iTpvDcQOkoUuUEfGnXPaGeM8GT7tNdKptT8gsg/H0Dwb4d0qwA8uAEWttmZBmI9145hpnIk+hkDkH22dzhTus8d7CfFDSpu7s3y9434wiCYF5pkugyWJb5dd24d00ssgF5gqhLU/s48WcBwipl8mMGu/H/5ezh/pG9dL2uDIUjKRrUybE/vmDJGy8ZEDnOwJ04NJ701vQjFybgZRXXyU5ESEJQ=
|
|
9
|
+
- secure: hq1T5MsHGmEnfQjOJlu/GzUPJ4WH0+lZOHezb4i0rFmRtWnmOMO+YUA34q9P6VQROk5KcAiRKtqZYmxZwTjyJVk2DTcJkvUAVzWBHD+MZAsH/lql+lnUJTuZRapYI85LChmEZkDMhYxZYfxTqGTbE4GjLtVtGN8zmTRStLxtvqyfiaU0SuGzyw9wfso/QhqHadQAQ7NttTYyiSiO2vHxK7rhmNi99MxAeVL3qZ3H2mcypSEE7CZd+nFa2CQxh/Guce0i+N2PDYTB6dsvA4xA5Xh/e+han3g4x/sRpic2C/9HS2zRYf0iN20+pFyotp5SaW3o5Q5S3iUqQtyZoR2lN1YMd4csyyKp/0G4QOjAPIN3FIef/RTVLqdfZXaord4iAg9GN9eKdtgYuS7JgyDl0VOmTpS2RYuDP+PGR2HRXXVc0gHt2aFl/oy+DTXh6emcOkpZtRZlbHqtVz5WXZZftm8niId+UXQmJuIQU+YgIiFDmviafapwjaYoXHCv8QMAGqNBP5EWDfxvYyPtGhQrj37sRYpfnUCkdJCiZvZLhQgQ0aIvR3KJIFpXtQmWaaT3e7u07B6weBB7Icto6WflfKtYHvVd1YfOX02/J8byhJbzRY+Qjqrq+5aQR1L3AKbmv8AooZdcm/CrEx6MP7xuIM0fjp1oU4cKQe74KX/KuqE=
|
|
10
|
+
- secure: fWzpY63ZAXOtjEJuYoya/mwzJ7fZQJGsN6hhamXorOhUgaeP2vdC7/6gK3PPMIXOJ5NXPiANGbAUWKgLLWJ3F7rK2y9UCUWTzvE2CK/13vUV9de7byQfAjfHo7wjDJuPLved+WmSM7ASVjzGq9wQS8MTeYUymSNVY3xiZcvNTO2XY9AAu32SSWDs1ayXiCGE9DyN5WEvEEPixHqpplZLvSqTtKtEZVdP/a3tPuzogYFvoJ/ZEfG9LRr4nnbwuKCaTljpYaZc/8yFbhhJgCrTuOKb0gB/UFeMjd0WX6MQWofzIj152d0ghCTlgtESR2LQJdSXA2BO0ZzSqYj8NKTAcZ8IsaUj+6kBgfQGHu43TF/dB877v8pY55HS8a3gFPDK/6OZhRT7UcFZgle1qvX422CD196fRHktSBcDTx9jQsG8r7Ray9oJygXQr6tKhrRkjMkkBAiBd/GkIJkJu5RecmlLOZMJ+YeYa95kWDUXwo3HGK9MxtFyuplSRYk4mQ1jbrJXdmufQuTazl2FYL+elOUeUb8ej3iniUcmyDGU6tZx84aZTbBOaovmeQqr3pXIaXYMhvkRGLD3Ky7SNZN9ZzFczRWcKStyQal+BYK1R3RU8cD0m0PEpUxV5ERP50E3GyN/0d0bg22LywK9aWDHOhCzjI0emVvSAn8ZiY78SSo=
|
data/DETAILS.md
CHANGED
|
@@ -155,6 +155,16 @@ S3 bucket notifications coming into the same queue):
|
|
|
155
155
|
The `Sqewer::CLI` module that you run from the commandline handler application accepts the
|
|
156
156
|
same options as the `Worker` constructor, so everything stays configurable.
|
|
157
157
|
|
|
158
|
+
## Threads versus processes
|
|
159
|
+
|
|
160
|
+
You can elect to use process isolation per job. Each job will then run in a forked process, on which
|
|
161
|
+
the main worker process will `wait()`. To do so, pass `Sqewer::Isolator.process` as the `isolator:`
|
|
162
|
+
option to the Worker
|
|
163
|
+
|
|
164
|
+
proc_worker = Sqewer::Worker.new(isolator: Sqewer::Isolator.process)
|
|
165
|
+
|
|
166
|
+
By default the system is working with threads only, as processes obviously have some overhead.
|
|
167
|
+
|
|
158
168
|
## Execution and serialization wrappers (middleware)
|
|
159
169
|
|
|
160
170
|
You can wrap job processing in middleware. A full-featured middleware class looks like this:
|
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
An AWS SQS-based queue processor, for highly distributed job engines.
|
|
2
2
|
|
|
3
|
+
[](https://travis-ci.org/WeTransfer/sqewer)
|
|
4
|
+
|
|
3
5
|
## The shortest introduction possible
|
|
4
6
|
|
|
5
7
|
In your environment, set `SQS_QUEUE_URL`. Then, define a job class:
|
|
@@ -44,6 +46,10 @@ Submitting jobs from other jobs (the job will go to the same queue the parent jo
|
|
|
44
46
|
|
|
45
47
|
The messages will only be deleted from SQS once the job execution completes without raising an exception.
|
|
46
48
|
|
|
49
|
+
## Requirements
|
|
50
|
+
|
|
51
|
+
Ruby 2.1+, version 2 of the AWS SDK.
|
|
52
|
+
|
|
47
53
|
## Detailed usage instructions
|
|
48
54
|
|
|
49
55
|
For more detailed usage information, see [DETAILS.md](./DETAILS.md)
|
data/example.env
CHANGED
data/lib/sqewer/connection.rb
CHANGED
|
@@ -9,6 +9,17 @@ class Sqewer::Connection
|
|
|
9
9
|
DEFAULT_TIMEOUT_SECONDS = 5
|
|
10
10
|
BATCH_RECEIVE_SIZE = 10
|
|
11
11
|
|
|
12
|
+
# A wrapper for most important properties of the received message
|
|
13
|
+
class Message < Struct.new(:receipt_handle, :body)
|
|
14
|
+
def inspect
|
|
15
|
+
body.inspect
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def has_body?
|
|
19
|
+
body && !body.empty?
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
12
23
|
# Returns the default adapter, connected to the queue set via the `SQS_QUEUE_URL`
|
|
13
24
|
# environment variable.
|
|
14
25
|
def self.default
|
|
@@ -25,22 +36,14 @@ class Sqewer::Connection
|
|
|
25
36
|
@queue_url = queue_url
|
|
26
37
|
end
|
|
27
38
|
|
|
28
|
-
#
|
|
39
|
+
# Receive at most 10 messages from the queue, and return the array of Message objects.
|
|
29
40
|
#
|
|
30
|
-
# @
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# from the caller.
|
|
37
|
-
poller.poll(max_number_of_messages: BATCH_RECEIVE_SIZE, skip_delete: true,
|
|
38
|
-
idle_timeout: timeout.to_i, wait_time_seconds: timeout.to_i) do | sqs_messages |
|
|
39
|
-
|
|
40
|
-
sqs_messages.each do | sqs_message |
|
|
41
|
-
yield [sqs_message.receipt_handle, sqs_message.body]
|
|
42
|
-
end
|
|
43
|
-
|
|
41
|
+
# @return [Array<Message>] an array of Message objects
|
|
42
|
+
def receive_messages
|
|
43
|
+
client = ::Aws::SQS::Client.new
|
|
44
|
+
response = client.receive_message(queue_url: @queue_url, wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: 10)
|
|
45
|
+
response.messages.map do | message |
|
|
46
|
+
Message.new(message.receipt_handle, message.body)
|
|
44
47
|
end
|
|
45
48
|
end
|
|
46
49
|
|
data/lib/sqewer/isolator.rb
CHANGED
|
@@ -3,11 +3,13 @@
|
|
|
3
3
|
# in threads (a-la Sidekiq).
|
|
4
4
|
class Sqewer::Isolator
|
|
5
5
|
# Used for running each job in a separate process.
|
|
6
|
-
class PerProcess
|
|
6
|
+
class PerProcess < self
|
|
7
7
|
# The method called to isolate a particular job flow (both instantiation and execution)
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
#
|
|
9
|
+
# @see {Isolator#perform}
|
|
10
|
+
def perform(*)
|
|
11
|
+
require 'exceptional_fork' unless defined?(ExceptionalFork)
|
|
12
|
+
ExceptionalFork.fork_and_wait { super }
|
|
11
13
|
end
|
|
12
14
|
end
|
|
13
15
|
|
|
@@ -27,7 +29,31 @@ class Sqewer::Isolator
|
|
|
27
29
|
end
|
|
28
30
|
|
|
29
31
|
# The method called to isolate a particular job flow (both instantiation and execution)
|
|
30
|
-
|
|
31
|
-
|
|
32
|
+
#
|
|
33
|
+
# @param worker[Sqewer::Worker] the worker that is running the jobs
|
|
34
|
+
# @param message[Sqewer::Connection::Message] the message that is being processed
|
|
35
|
+
def perform(worker, message)
|
|
36
|
+
|
|
37
|
+
submitter_class, execution_context_class,
|
|
38
|
+
middleware_stack, connection, serializer, logger =
|
|
39
|
+
worker.submitter_class, worker.execution_context_class,
|
|
40
|
+
worker.middleware_stack, worker.connection, worker.serializer, worker.logger
|
|
41
|
+
|
|
42
|
+
job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
|
|
43
|
+
serializer.unserialize(message.body)
|
|
44
|
+
end
|
|
45
|
+
return unless job
|
|
46
|
+
|
|
47
|
+
submitter = submitter_class.new(connection, serializer)
|
|
48
|
+
context = execution_context_class.new(submitter, {'logger' => @logger})
|
|
49
|
+
|
|
50
|
+
t = Time.now
|
|
51
|
+
middleware_stack.around_execution(job, context) do
|
|
52
|
+
job.method(:run).arity.zero? ? job.run : job.run(context)
|
|
53
|
+
end
|
|
54
|
+
logger.info { "[worker] Finished #{job.inspect} in %0.2fs" % (Time.now - t) }
|
|
55
|
+
rescue => e
|
|
56
|
+
logger.error { "[worker] Failed #{job.inspect} with a #{e}" } if job
|
|
57
|
+
raise e
|
|
32
58
|
end
|
|
33
59
|
end
|
data/lib/sqewer/version.rb
CHANGED
data/lib/sqewer/worker.rb
CHANGED
|
@@ -9,6 +9,30 @@ class Sqewer::Worker
|
|
|
9
9
|
SLEEP_SECONDS_ON_EMPTY_QUEUE = 1
|
|
10
10
|
THROTTLE_FACTOR = 2
|
|
11
11
|
|
|
12
|
+
# @return [Logger] The logger used for job execution
|
|
13
|
+
attr_reader :logger
|
|
14
|
+
|
|
15
|
+
# @return [Sqewer::Connection] The connection for sending and receiving messages
|
|
16
|
+
attr_reader :connection
|
|
17
|
+
|
|
18
|
+
# @return [Sqewer::Serializer] The serializer for unmarshalling and marshalling
|
|
19
|
+
attr_reader :serializer
|
|
20
|
+
|
|
21
|
+
# @return [Sqewer::MiddlewareStack] The stack used when executing the job
|
|
22
|
+
attr_reader :middleware_stack
|
|
23
|
+
|
|
24
|
+
# @return [Class] The class to use when instantiating the execution context
|
|
25
|
+
attr_reader :execution_context_class
|
|
26
|
+
|
|
27
|
+
# @return [Class] The class used to create the Submitter used by jobs to spawn other jobs
|
|
28
|
+
attr_reader :submitter_class
|
|
29
|
+
|
|
30
|
+
# @return [#perform] The isolator to use when executing each job
|
|
31
|
+
attr_reader :isolator
|
|
32
|
+
|
|
33
|
+
# @return [Fixnum] the number of threads to spin up
|
|
34
|
+
attr_reader :num_threads
|
|
35
|
+
|
|
12
36
|
# Returns the default Worker instance, configured based on the default components
|
|
13
37
|
#
|
|
14
38
|
# @return [Sqewer::Worker]
|
|
@@ -77,29 +101,24 @@ class Sqewer::Worker
|
|
|
77
101
|
end
|
|
78
102
|
end
|
|
79
103
|
|
|
80
|
-
# Create
|
|
81
|
-
#
|
|
82
|
-
# to "suspend" the polling loop in the SQS client when the local buffer queue fills up.
|
|
104
|
+
# Create the provider thread. When the execution queue is exhausted,
|
|
105
|
+
# grab new messages and place them on the local queue.
|
|
83
106
|
provider = Thread.new do
|
|
84
|
-
feeder_fiber = Fiber.new do
|
|
85
|
-
loop do
|
|
86
|
-
break if @state.in_state?(:stopping)
|
|
87
|
-
@connection.poll do |message_id, message_body|
|
|
88
|
-
break if @state.in_state?(:stopping)
|
|
89
|
-
Fiber.yield([message_id, message_body])
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
end
|
|
93
|
-
|
|
94
107
|
loop do
|
|
95
|
-
break if !feeder_fiber.alive?
|
|
96
108
|
break if stopping?
|
|
97
109
|
|
|
98
|
-
if
|
|
99
|
-
|
|
110
|
+
if queue_has_capacity?
|
|
111
|
+
messages = @connection.receive_messages
|
|
112
|
+
if messages.any?
|
|
113
|
+
messages.each {|m| @execution_queue << m }
|
|
114
|
+
@logger.debug { "[worker] Received and buffered %d messages" % messages.length } if messages.any?
|
|
115
|
+
else
|
|
116
|
+
@logger.debug { "[worker] No messages received" }
|
|
117
|
+
Thread.pass
|
|
118
|
+
end
|
|
100
119
|
else
|
|
101
|
-
@logger.debug "Suspending poller (%d items buffered)" % @execution_queue.length
|
|
102
|
-
sleep
|
|
120
|
+
@logger.debug { "[worker] Suspending poller (%d items buffered)" % @execution_queue.length }
|
|
121
|
+
sleep 1
|
|
103
122
|
Thread.pass
|
|
104
123
|
end
|
|
105
124
|
end
|
|
@@ -154,46 +173,28 @@ class Sqewer::Worker
|
|
|
154
173
|
@state.in_state?(:stopping)
|
|
155
174
|
end
|
|
156
175
|
|
|
176
|
+
def queue_has_capacity?
|
|
177
|
+
@execution_queue.length < (@num_threads * THROTTLE_FACTOR)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def handle_message(message)
|
|
181
|
+
return unless message.receipt_handle
|
|
182
|
+
return @connection.delete_message(message.receipt_handle) unless message.has_body?
|
|
183
|
+
@isolator.perform(self, message)
|
|
184
|
+
@connection.delete_message(message.receipt_handle)
|
|
185
|
+
end
|
|
157
186
|
|
|
158
187
|
def take_and_execute
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
return @connection.delete_message(message_id) unless message_body && !message_body.empty?
|
|
162
|
-
|
|
163
|
-
@isolator.isolate do
|
|
164
|
-
job = @middleware_stack.around_deserialization(@serializer, message_id, message_body) do
|
|
165
|
-
@serializer.unserialize(message_body)
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
if job # if the serializer returns a nil or false
|
|
169
|
-
t = Time.now
|
|
170
|
-
submitter = @submitter_class.new(@connection, @serializer)
|
|
171
|
-
context = @execution_context_class.new(submitter, {STR_logger => @logger})
|
|
172
|
-
|
|
173
|
-
begin
|
|
174
|
-
@middleware_stack.around_execution(job, context) do
|
|
175
|
-
job.method(:run).arity.zero? ? job.run : job.run(context)
|
|
176
|
-
end
|
|
177
|
-
@logger.info { "[worker] Finished #{job.inspect} in %0.2fs" % (Time.now - t) }
|
|
178
|
-
rescue => e
|
|
179
|
-
@logger.error { "[worker] Failed #{job.inspect} with a #{e}" }
|
|
180
|
-
raise e
|
|
181
|
-
end
|
|
182
|
-
end
|
|
183
|
-
end
|
|
184
|
-
|
|
185
|
-
@connection.delete_message(message_id)
|
|
188
|
+
message = @execution_queue.pop(nonblock=true)
|
|
189
|
+
handle_message(message)
|
|
186
190
|
rescue ThreadError # Queue is empty
|
|
187
191
|
sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
|
|
188
192
|
Thread.pass
|
|
189
|
-
rescue SystemExit, SignalException, Interrupt => e # Time to quit
|
|
190
|
-
@logger.error { "[worker] Signaled, will quit the consumer" }
|
|
191
|
-
return
|
|
192
193
|
rescue => e # anything else, at or below StandardError that does not need us to quit
|
|
193
|
-
@logger.
|
|
194
|
-
@logger.
|
|
195
|
-
@logger.
|
|
196
|
-
e.backtrace.each { |s| @logger.
|
|
194
|
+
@logger.error { "[worker] Failed #{message.inspect} with #{e}" }
|
|
195
|
+
@logger.error(e.class)
|
|
196
|
+
@logger.error(e.message)
|
|
197
|
+
e.backtrace.each { |s| @logger.error{"\t#{s}"} }
|
|
197
198
|
end
|
|
198
199
|
|
|
199
200
|
STR_logger = 'logger'
|
data/spec/sqewer/cli_spec.rb
CHANGED
|
@@ -6,7 +6,8 @@ describe Sqewer::CLI, :sqs => true do
|
|
|
6
6
|
submitter = Sqewer::Connection.default
|
|
7
7
|
|
|
8
8
|
stderr = Tempfile.new('worker-stderr')
|
|
9
|
-
|
|
9
|
+
stderr.sync = true
|
|
10
|
+
|
|
10
11
|
pid = fork { $stderr.reopen(stderr); exec("ruby #{__dir__}/cli_app.rb") }
|
|
11
12
|
|
|
12
13
|
Thread.new do
|
|
@@ -16,8 +17,9 @@ describe Sqewer::CLI, :sqs => true do
|
|
|
16
17
|
end
|
|
17
18
|
end
|
|
18
19
|
|
|
19
|
-
sleep
|
|
20
|
+
sleep 8 # Give it some time to process all the jobs
|
|
20
21
|
Process.kill("USR1", pid)
|
|
22
|
+
sleep 2
|
|
21
23
|
|
|
22
24
|
generated_files = Dir.glob('*-result')
|
|
23
25
|
expect(generated_files).not_to be_empty
|
|
@@ -32,6 +34,7 @@ describe Sqewer::CLI, :sqs => true do
|
|
|
32
34
|
submitter = Sqewer::Connection.default
|
|
33
35
|
|
|
34
36
|
stderr = Tempfile.new('worker-stderr')
|
|
37
|
+
stderr.sync = true
|
|
35
38
|
|
|
36
39
|
pid = fork { $stderr.reopen(stderr); exec("ruby #{__dir__}/cli_app.rb") }
|
|
37
40
|
|
|
@@ -31,27 +31,23 @@ describe Sqewer::Connection do
|
|
|
31
31
|
end
|
|
32
32
|
end
|
|
33
33
|
|
|
34
|
-
describe '#
|
|
34
|
+
describe '#receive_messages' do
|
|
35
35
|
it 'uses the batched receive feature' do
|
|
36
36
|
s = described_class.new('https://fake-queue')
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
expect(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
double('SQSMessage', receipt_handle: SecureRandom.hex(4), body: 'Some message')
|
|
44
|
-
end
|
|
45
|
-
# Yields arrays of messages, so...
|
|
46
|
-
blk.call(bulk)
|
|
38
|
+
fake_sqs_client = double('Client')
|
|
39
|
+
expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
|
|
40
|
+
|
|
41
|
+
fake_messages = (1..5).map {
|
|
42
|
+
double(receipt_handle: SecureRandom.hex(4), body: SecureRandom.random_bytes(128))
|
|
47
43
|
}
|
|
44
|
+
fake_response = double(messages: fake_messages)
|
|
48
45
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
receives << [sqs_message_handle, sqs_message_body]
|
|
52
|
-
end
|
|
46
|
+
expect(fake_sqs_client).to receive(:receive_message).with({:queue_url=>"https://fake-queue", :wait_time_seconds=>5,
|
|
47
|
+
:max_number_of_messages=>10}).and_return(fake_response)
|
|
53
48
|
|
|
54
|
-
|
|
49
|
+
messages = s.receive_messages
|
|
50
|
+
expect(messages.length).to eq(5)
|
|
55
51
|
end
|
|
56
52
|
end
|
|
57
53
|
end
|
data/spec/sqewer/worker_spec.rb
CHANGED
|
@@ -1,8 +1,21 @@
|
|
|
1
1
|
require_relative '../spec_helper'
|
|
2
2
|
|
|
3
3
|
describe Sqewer::Worker, :sqs => true do
|
|
4
|
-
let(:
|
|
4
|
+
let(:test_logger) {
|
|
5
|
+
$stderr.sync = true
|
|
6
|
+
ENV['SHOW_TEST_LOGS'] ? Logger.new($stderr) : Logger.new(StringIO.new(''))
|
|
7
|
+
}
|
|
5
8
|
|
|
9
|
+
it 'has all the necessary attributes' do
|
|
10
|
+
attrs = [:logger, :connection, :serializer, :middleware_stack,
|
|
11
|
+
:execution_context_class, :submitter_class, :isolator, :num_threads]
|
|
12
|
+
default_worker = described_class.default
|
|
13
|
+
attrs.each do | attr_name |
|
|
14
|
+
expect(default_worker).to respond_to(attr_name)
|
|
15
|
+
expect(default_worker.public_send(attr_name)).not_to be_nil
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
6
19
|
it 'supports .default' do
|
|
7
20
|
default_worker = described_class.default
|
|
8
21
|
expect(default_worker).to respond_to(:start)
|
|
@@ -15,7 +28,7 @@ describe Sqewer::Worker, :sqs => true do
|
|
|
15
28
|
end
|
|
16
29
|
|
|
17
30
|
it 'can go through the full cycle of initialize, start, stop, start, stop' do
|
|
18
|
-
worker = described_class.new(logger:
|
|
31
|
+
worker = described_class.new(logger: test_logger)
|
|
19
32
|
worker.start
|
|
20
33
|
worker.stop
|
|
21
34
|
worker.start
|
|
@@ -34,17 +47,11 @@ describe Sqewer::Worker, :sqs => true do
|
|
|
34
47
|
client = Aws::SQS::Client.new
|
|
35
48
|
client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: '{"foo":')
|
|
36
49
|
|
|
37
|
-
|
|
38
|
-
logger_to_string = Logger.new(StringIO.new(logger_output))
|
|
39
|
-
|
|
40
|
-
worker = described_class.new(logger: logger_to_string)
|
|
50
|
+
worker = described_class.new(logger: test_logger)
|
|
41
51
|
|
|
42
52
|
worker.start
|
|
43
53
|
sleep 2
|
|
44
54
|
worker.stop
|
|
45
|
-
|
|
46
|
-
expect(logger_output).to include('unexpected token at \'{"foo":')
|
|
47
|
-
expect(logger_output).to include('Stopping (clean shutdown)')
|
|
48
55
|
end
|
|
49
56
|
end
|
|
50
57
|
|
|
@@ -55,22 +62,19 @@ describe Sqewer::Worker, :sqs => true do
|
|
|
55
62
|
client = Aws::SQS::Client.new
|
|
56
63
|
client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
|
|
57
64
|
|
|
58
|
-
|
|
59
|
-
logger_to_string = Logger.new(StringIO.new(logger_output))
|
|
60
|
-
|
|
61
|
-
worker = described_class.new(logger: logger_to_string)
|
|
65
|
+
worker = described_class.new(logger: test_logger)
|
|
62
66
|
|
|
63
67
|
worker.start
|
|
64
68
|
sleep 2
|
|
65
69
|
worker.stop
|
|
66
70
|
|
|
67
|
-
expect(logger_output).to include('uninitialized constant UnknownJobClass')
|
|
68
|
-
expect(logger_output).to include('Stopping (clean shutdown)')
|
|
71
|
+
# expect(logger_output).to include('uninitialized constant UnknownJobClass')
|
|
72
|
+
# expect(logger_output).to include('Stopping (clean shutdown)')
|
|
69
73
|
end
|
|
70
74
|
end
|
|
71
75
|
|
|
72
76
|
context 'with a job that spawns another job' do
|
|
73
|
-
it 'sets up the processing pipeline so that jobs can execute in sequence' do
|
|
77
|
+
it 'sets up the processing pipeline so that jobs can execute in sequence (with threads)' do
|
|
74
78
|
class SecondaryJob
|
|
75
79
|
def run
|
|
76
80
|
File.open('secondary-job-run','w') {}
|
|
@@ -88,9 +92,7 @@ describe Sqewer::Worker, :sqs => true do
|
|
|
88
92
|
client = Aws::SQS::Client.new
|
|
89
93
|
client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
|
|
90
94
|
|
|
91
|
-
|
|
92
|
-
logger_to_string = Logger.new(StringIO.new(logger_output))
|
|
93
|
-
worker = described_class.new(logger: logger_to_string, num_threads: 8)
|
|
95
|
+
worker = described_class.new(logger: test_logger, num_threads: 8)
|
|
94
96
|
|
|
95
97
|
worker.start
|
|
96
98
|
|
|
@@ -100,18 +102,30 @@ describe Sqewer::Worker, :sqs => true do
|
|
|
100
102
|
|
|
101
103
|
File.unlink('initial-job-run')
|
|
102
104
|
File.unlink('secondary-job-run')
|
|
103
|
-
expect(true).to eq(true)
|
|
104
105
|
ensure
|
|
105
106
|
worker.stop
|
|
106
107
|
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
it 'sets up the processing pipeline so that jobs can execute in sequence (with processes)' do
|
|
111
|
+
class SecondaryJob
|
|
112
|
+
def run
|
|
113
|
+
File.open('secondary-job-run','w') {}
|
|
114
|
+
end
|
|
115
|
+
end
|
|
107
116
|
|
|
108
|
-
|
|
117
|
+
class InitialJob
|
|
118
|
+
def run(executor)
|
|
119
|
+
File.open('initial-job-run','w') {}
|
|
120
|
+
executor.submit!(SecondaryJob.new)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
payload = JSON.dump({job_class: 'InitialJob'})
|
|
109
125
|
client = Aws::SQS::Client.new
|
|
110
126
|
client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
|
|
111
127
|
|
|
112
|
-
|
|
113
|
-
logger_to_string = Logger.new(StringIO.new(logger_output))
|
|
114
|
-
worker = described_class.new(logger: logger_to_string, num_threads: 8, isolator: Sqewer::Isolator.process)
|
|
128
|
+
worker = described_class.new(logger: test_logger, num_threads: 8, isolator: Sqewer::Isolator.process)
|
|
115
129
|
|
|
116
130
|
worker.start
|
|
117
131
|
|
|
@@ -121,7 +135,6 @@ describe Sqewer::Worker, :sqs => true do
|
|
|
121
135
|
|
|
122
136
|
File.unlink('initial-job-run')
|
|
123
137
|
File.unlink('secondary-job-run')
|
|
124
|
-
expect(true).to eq(true)
|
|
125
138
|
ensure
|
|
126
139
|
worker.stop
|
|
127
140
|
end
|
data/sqewer.gemspec
CHANGED
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
|
-
# stub: sqewer
|
|
5
|
+
# stub: sqewer 2.0.0 ruby lib
|
|
6
6
|
|
|
7
7
|
Gem::Specification.new do |s|
|
|
8
8
|
s.name = "sqewer"
|
|
9
|
-
s.version = "
|
|
9
|
+
s.version = "2.0.0"
|
|
10
10
|
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
12
12
|
s.require_paths = ["lib"]
|
|
13
13
|
s.authors = ["Julik Tarkhanov"]
|
|
14
|
-
s.date = "2016-01-
|
|
14
|
+
s.date = "2016-01-21"
|
|
15
15
|
s.description = "Process jobs from SQS"
|
|
16
16
|
s.email = "me@julik.nl"
|
|
17
17
|
s.extra_rdoc_files = [
|
|
@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
|
|
|
19
19
|
]
|
|
20
20
|
s.files = [
|
|
21
21
|
".gitlab-ci.yml",
|
|
22
|
+
".travis.yml",
|
|
22
23
|
".yardopts",
|
|
23
24
|
"DETAILS.md",
|
|
24
25
|
"FAQ.md",
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: sqewer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Julik Tarkhanov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-01-
|
|
11
|
+
date: 2016-01-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: aws-sdk
|
|
@@ -186,6 +186,7 @@ extra_rdoc_files:
|
|
|
186
186
|
- README.md
|
|
187
187
|
files:
|
|
188
188
|
- ".gitlab-ci.yml"
|
|
189
|
+
- ".travis.yml"
|
|
189
190
|
- ".yardopts"
|
|
190
191
|
- DETAILS.md
|
|
191
192
|
- FAQ.md
|