sqewer 1.0.0 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +10 -0
- data/DETAILS.md +10 -0
- data/README.md +6 -0
- data/example.env +2 -1
- data/lib/sqewer/connection.rb +18 -15
- data/lib/sqewer/isolator.rb +32 -6
- data/lib/sqewer/version.rb +1 -1
- data/lib/sqewer/worker.rb +53 -52
- data/spec/sqewer/cli_spec.rb +5 -2
- data/spec/sqewer/connection_spec.rb +11 -15
- data/spec/sqewer/worker_spec.rb +38 -25
- data/sqewer.gemspec +4 -3
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7431fac224bee95320e7fa75a1503511a7175ff1
|
4
|
+
data.tar.gz: fdf4629e080fd39e0acc9ac7d1b941e9270e4348
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: efc4ae288acade906f671179499918ff130dc4d1feee736aa5a17b875e07d9e6c51958d87ce159ef1c6f020332804ac9ef8935f2f8106c27e7ba565512bd407d
|
7
|
+
data.tar.gz: c0fdf7e2640365bc2d8c8f8e0496375013c5febf50ebf1abd94ea28c5eb7c9655682b8bde976c628d724dbd9b55edb1d6a92c2d1f2ba7d27c925ff42ab26efe8
|
data/.travis.yml
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
rvm:
|
2
|
+
- 2.1.5
|
3
|
+
- 2.2.2
|
4
|
+
sudo: false
|
5
|
+
cache: bundler
|
6
|
+
env:
|
7
|
+
global:
|
8
|
+
- secure: E+uNoJmvjq5+3/PYUs2a1bRh7bCjAbcYEIKJzOVgIiI+gWB4dLpTKwVW/fqJbjCDIGPw9OwiwphN51zLEo0ie5AJIkavcjWRcdazKq8usokAz/A5Ilru5YvGXJ5UQtF8TJNsZfJGOGMJwL5d/MsDSd3/sjy88wrFiSeeNlHA06KDBrGdZXE4/dOb5Z84cSxPaD7nDaGP36V3Uv88X272oezODm9Mp+Fsf4goGoU9I9PBDD36pCXAlpURWpyWMKa2TzEXfgN43TYxta4p+bu6BA871df31j3BfVKwY1K/59eN9HqW8xWF9rgcgLo3NgQuaoqde3ryCE4jqKymjtG1/R5bdbQsn07R7/OagQMPVkbn07Aq+X6Q8GVTkrMl829z87QiMETNnzpsHl/1RD3DVnUo4k+XmqLbwW6NRl452j3eEHsiMvxf1vt8Qn1hu61qm98WGVNCGFWX/eAu/iTpvDcQOkoUuUEfGnXPaGeM8GT7tNdKptT8gsg/H0Dwb4d0qwA8uAEWttmZBmI9145hpnIk+hkDkH22dzhTus8d7CfFDSpu7s3y9434wiCYF5pkugyWJb5dd24d00ssgF5gqhLU/s48WcBwipl8mMGu/H/5ezh/pG9dL2uDIUjKRrUybE/vmDJGy8ZEDnOwJ04NJ701vQjFybgZRXXyU5ESEJQ=
|
9
|
+
- secure: hq1T5MsHGmEnfQjOJlu/GzUPJ4WH0+lZOHezb4i0rFmRtWnmOMO+YUA34q9P6VQROk5KcAiRKtqZYmxZwTjyJVk2DTcJkvUAVzWBHD+MZAsH/lql+lnUJTuZRapYI85LChmEZkDMhYxZYfxTqGTbE4GjLtVtGN8zmTRStLxtvqyfiaU0SuGzyw9wfso/QhqHadQAQ7NttTYyiSiO2vHxK7rhmNi99MxAeVL3qZ3H2mcypSEE7CZd+nFa2CQxh/Guce0i+N2PDYTB6dsvA4xA5Xh/e+han3g4x/sRpic2C/9HS2zRYf0iN20+pFyotp5SaW3o5Q5S3iUqQtyZoR2lN1YMd4csyyKp/0G4QOjAPIN3FIef/RTVLqdfZXaord4iAg9GN9eKdtgYuS7JgyDl0VOmTpS2RYuDP+PGR2HRXXVc0gHt2aFl/oy+DTXh6emcOkpZtRZlbHqtVz5WXZZftm8niId+UXQmJuIQU+YgIiFDmviafapwjaYoXHCv8QMAGqNBP5EWDfxvYyPtGhQrj37sRYpfnUCkdJCiZvZLhQgQ0aIvR3KJIFpXtQmWaaT3e7u07B6weBB7Icto6WflfKtYHvVd1YfOX02/J8byhJbzRY+Qjqrq+5aQR1L3AKbmv8AooZdcm/CrEx6MP7xuIM0fjp1oU4cKQe74KX/KuqE=
|
10
|
+
- secure: fWzpY63ZAXOtjEJuYoya/mwzJ7fZQJGsN6hhamXorOhUgaeP2vdC7/6gK3PPMIXOJ5NXPiANGbAUWKgLLWJ3F7rK2y9UCUWTzvE2CK/13vUV9de7byQfAjfHo7wjDJuPLved+WmSM7ASVjzGq9wQS8MTeYUymSNVY3xiZcvNTO2XY9AAu32SSWDs1ayXiCGE9DyN5WEvEEPixHqpplZLvSqTtKtEZVdP/a3tPuzogYFvoJ/ZEfG9LRr4nnbwuKCaTljpYaZc/8yFbhhJgCrTuOKb0gB/UFeMjd0WX6MQWofzIj152d0ghCTlgtESR2LQJdSXA2BO0ZzSqYj8NKTAcZ8IsaUj+6kBgfQGHu43TF/dB877v8pY55HS8a3gFPDK/6OZhRT7UcFZgle1qvX422CD196fRHktSBcDTx9jQsG8r7Ray9oJygXQr6tKhrRkjMkkBAiBd/GkIJkJu5RecmlLOZMJ+YeYa95kWDUXwo3HGK9MxtFyuplSRYk4mQ1jbrJXdmufQuTazl2FYL+elOUeUb8ej3iniUcmyDGU6tZx84aZTbBOaovmeQqr3pXIaXYMhvkRGLD3Ky7SNZN9ZzFczRWcKStyQal+BYK1R3RU8cD0m0PEpUxV5ERP50E3GyN/0d0bg22LywK9aWDHOhCzjI0emVvSAn8ZiY78SSo=
|
data/DETAILS.md
CHANGED
@@ -155,6 +155,16 @@ S3 bucket notifications coming into the same queue):
|
|
155
155
|
The `Sqewer::CLI` module that you run from the commandline handler application accepts the
|
156
156
|
same options as the `Worker` constructor, so everything stays configurable.
|
157
157
|
|
158
|
+
## Threads versus processes
|
159
|
+
|
160
|
+
You can elect to use process isolation per job. Each job will then run in a forked process, on which
|
161
|
+
the main worker process will `wait()`. To do so, pass `Sqewer::Isolator.process` as the `isolator:`
|
162
|
+
option to the Worker
|
163
|
+
|
164
|
+
proc_worker = Sqewer::Worker.new(isolator: Sqewer::Isolator.process)
|
165
|
+
|
166
|
+
By default the system is working with threads only, as processes obviously have some overhead.
|
167
|
+
|
158
168
|
## Execution and serialization wrappers (middleware)
|
159
169
|
|
160
170
|
You can wrap job processing in middleware. A full-featured middleware class looks like this:
|
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
An AWS SQS-based queue processor, for highly distributed job engines.
|
2
2
|
|
3
|
+
[![Build Status](https://travis-ci.org/WeTransfer/sqewer.svg?branch=master)](https://travis-ci.org/WeTransfer/sqewer)
|
4
|
+
|
3
5
|
## The shortest introduction possible
|
4
6
|
|
5
7
|
In your environment, set `SQS_QUEUE_URL`. Then, define a job class:
|
@@ -44,6 +46,10 @@ Submitting jobs from other jobs (the job will go to the same queue the parent jo
|
|
44
46
|
|
45
47
|
The messages will only be deleted from SQS once the job execution completes without raising an exception.
|
46
48
|
|
49
|
+
## Requirements
|
50
|
+
|
51
|
+
Ruby 2.1+, version 2 of the AWS SDK.
|
52
|
+
|
47
53
|
## Detailed usage instructions
|
48
54
|
|
49
55
|
For more detailed usage information, see [DETAILS.md](./DETAILS.md)
|
data/example.env
CHANGED
data/lib/sqewer/connection.rb
CHANGED
@@ -9,6 +9,17 @@ class Sqewer::Connection
|
|
9
9
|
DEFAULT_TIMEOUT_SECONDS = 5
|
10
10
|
BATCH_RECEIVE_SIZE = 10
|
11
11
|
|
12
|
+
# A wrapper for most important properties of the received message
|
13
|
+
class Message < Struct.new(:receipt_handle, :body)
|
14
|
+
def inspect
|
15
|
+
body.inspect
|
16
|
+
end
|
17
|
+
|
18
|
+
def has_body?
|
19
|
+
body && !body.empty?
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
12
23
|
# Returns the default adapter, connected to the queue set via the `SQS_QUEUE_URL`
|
13
24
|
# environment variable.
|
14
25
|
def self.default
|
@@ -25,22 +36,14 @@ class Sqewer::Connection
|
|
25
36
|
@queue_url = queue_url
|
26
37
|
end
|
27
38
|
|
28
|
-
#
|
39
|
+
# Receive at most 10 messages from the queue, and return the array of Message objects.
|
29
40
|
#
|
30
|
-
# @
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
# from the caller.
|
37
|
-
poller.poll(max_number_of_messages: BATCH_RECEIVE_SIZE, skip_delete: true,
|
38
|
-
idle_timeout: timeout.to_i, wait_time_seconds: timeout.to_i) do | sqs_messages |
|
39
|
-
|
40
|
-
sqs_messages.each do | sqs_message |
|
41
|
-
yield [sqs_message.receipt_handle, sqs_message.body]
|
42
|
-
end
|
43
|
-
|
41
|
+
# @return [Array<Message>] an array of Message objects
|
42
|
+
def receive_messages
|
43
|
+
client = ::Aws::SQS::Client.new
|
44
|
+
response = client.receive_message(queue_url: @queue_url, wait_time_seconds: DEFAULT_TIMEOUT_SECONDS, max_number_of_messages: 10)
|
45
|
+
response.messages.map do | message |
|
46
|
+
Message.new(message.receipt_handle, message.body)
|
44
47
|
end
|
45
48
|
end
|
46
49
|
|
data/lib/sqewer/isolator.rb
CHANGED
@@ -3,11 +3,13 @@
|
|
3
3
|
# in threads (a-la Sidekiq).
|
4
4
|
class Sqewer::Isolator
|
5
5
|
# Used for running each job in a separate process.
|
6
|
-
class PerProcess
|
6
|
+
class PerProcess < self
|
7
7
|
# The method called to isolate a particular job flow (both instantiation and execution)
|
8
|
-
|
9
|
-
|
10
|
-
|
8
|
+
#
|
9
|
+
# @see {Isolator#perform}
|
10
|
+
def perform(*)
|
11
|
+
require 'exceptional_fork' unless defined?(ExceptionalFork)
|
12
|
+
ExceptionalFork.fork_and_wait { super }
|
11
13
|
end
|
12
14
|
end
|
13
15
|
|
@@ -27,7 +29,31 @@ class Sqewer::Isolator
|
|
27
29
|
end
|
28
30
|
|
29
31
|
# The method called to isolate a particular job flow (both instantiation and execution)
|
30
|
-
|
31
|
-
|
32
|
+
#
|
33
|
+
# @param worker[Sqewer::Worker] the worker that is running the jobs
|
34
|
+
# @param message[Sqewer::Connection::Message] the message that is being processed
|
35
|
+
def perform(worker, message)
|
36
|
+
|
37
|
+
submitter_class, execution_context_class,
|
38
|
+
middleware_stack, connection, serializer, logger =
|
39
|
+
worker.submitter_class, worker.execution_context_class,
|
40
|
+
worker.middleware_stack, worker.connection, worker.serializer, worker.logger
|
41
|
+
|
42
|
+
job = middleware_stack.around_deserialization(serializer, message.receipt_handle, message.body) do
|
43
|
+
serializer.unserialize(message.body)
|
44
|
+
end
|
45
|
+
return unless job
|
46
|
+
|
47
|
+
submitter = submitter_class.new(connection, serializer)
|
48
|
+
context = execution_context_class.new(submitter, {'logger' => @logger})
|
49
|
+
|
50
|
+
t = Time.now
|
51
|
+
middleware_stack.around_execution(job, context) do
|
52
|
+
job.method(:run).arity.zero? ? job.run : job.run(context)
|
53
|
+
end
|
54
|
+
logger.info { "[worker] Finished #{job.inspect} in %0.2fs" % (Time.now - t) }
|
55
|
+
rescue => e
|
56
|
+
logger.error { "[worker] Failed #{job.inspect} with a #{e}" } if job
|
57
|
+
raise e
|
32
58
|
end
|
33
59
|
end
|
data/lib/sqewer/version.rb
CHANGED
data/lib/sqewer/worker.rb
CHANGED
@@ -9,6 +9,30 @@ class Sqewer::Worker
|
|
9
9
|
SLEEP_SECONDS_ON_EMPTY_QUEUE = 1
|
10
10
|
THROTTLE_FACTOR = 2
|
11
11
|
|
12
|
+
# @return [Logger] The logger used for job execution
|
13
|
+
attr_reader :logger
|
14
|
+
|
15
|
+
# @return [Sqewer::Connection] The connection for sending and receiving messages
|
16
|
+
attr_reader :connection
|
17
|
+
|
18
|
+
# @return [Sqewer::Serializer] The serializer for unmarshalling and marshalling
|
19
|
+
attr_reader :serializer
|
20
|
+
|
21
|
+
# @return [Sqewer::MiddlewareStack] The stack used when executing the job
|
22
|
+
attr_reader :middleware_stack
|
23
|
+
|
24
|
+
# @return [Class] The class to use when instantiating the execution context
|
25
|
+
attr_reader :execution_context_class
|
26
|
+
|
27
|
+
# @return [Class] The class used to create the Submitter used by jobs to spawn other jobs
|
28
|
+
attr_reader :submitter_class
|
29
|
+
|
30
|
+
# @return [#perform] The isolator to use when executing each job
|
31
|
+
attr_reader :isolator
|
32
|
+
|
33
|
+
# @return [Fixnum] the number of threads to spin up
|
34
|
+
attr_reader :num_threads
|
35
|
+
|
12
36
|
# Returns the default Worker instance, configured based on the default components
|
13
37
|
#
|
14
38
|
# @return [Sqewer::Worker]
|
@@ -77,29 +101,24 @@ class Sqewer::Worker
|
|
77
101
|
end
|
78
102
|
end
|
79
103
|
|
80
|
-
# Create
|
81
|
-
#
|
82
|
-
# to "suspend" the polling loop in the SQS client when the local buffer queue fills up.
|
104
|
+
# Create the provider thread. When the execution queue is exhausted,
|
105
|
+
# grab new messages and place them on the local queue.
|
83
106
|
provider = Thread.new do
|
84
|
-
feeder_fiber = Fiber.new do
|
85
|
-
loop do
|
86
|
-
break if @state.in_state?(:stopping)
|
87
|
-
@connection.poll do |message_id, message_body|
|
88
|
-
break if @state.in_state?(:stopping)
|
89
|
-
Fiber.yield([message_id, message_body])
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
107
|
loop do
|
95
|
-
break if !feeder_fiber.alive?
|
96
108
|
break if stopping?
|
97
109
|
|
98
|
-
if
|
99
|
-
|
110
|
+
if queue_has_capacity?
|
111
|
+
messages = @connection.receive_messages
|
112
|
+
if messages.any?
|
113
|
+
messages.each {|m| @execution_queue << m }
|
114
|
+
@logger.debug { "[worker] Received and buffered %d messages" % messages.length } if messages.any?
|
115
|
+
else
|
116
|
+
@logger.debug { "[worker] No messages received" }
|
117
|
+
Thread.pass
|
118
|
+
end
|
100
119
|
else
|
101
|
-
@logger.debug "Suspending poller (%d items buffered)" % @execution_queue.length
|
102
|
-
sleep
|
120
|
+
@logger.debug { "[worker] Suspending poller (%d items buffered)" % @execution_queue.length }
|
121
|
+
sleep 1
|
103
122
|
Thread.pass
|
104
123
|
end
|
105
124
|
end
|
@@ -154,46 +173,28 @@ class Sqewer::Worker
|
|
154
173
|
@state.in_state?(:stopping)
|
155
174
|
end
|
156
175
|
|
176
|
+
def queue_has_capacity?
|
177
|
+
@execution_queue.length < (@num_threads * THROTTLE_FACTOR)
|
178
|
+
end
|
179
|
+
|
180
|
+
def handle_message(message)
|
181
|
+
return unless message.receipt_handle
|
182
|
+
return @connection.delete_message(message.receipt_handle) unless message.has_body?
|
183
|
+
@isolator.perform(self, message)
|
184
|
+
@connection.delete_message(message.receipt_handle)
|
185
|
+
end
|
157
186
|
|
158
187
|
def take_and_execute
|
159
|
-
|
160
|
-
|
161
|
-
return @connection.delete_message(message_id) unless message_body && !message_body.empty?
|
162
|
-
|
163
|
-
@isolator.isolate do
|
164
|
-
job = @middleware_stack.around_deserialization(@serializer, message_id, message_body) do
|
165
|
-
@serializer.unserialize(message_body)
|
166
|
-
end
|
167
|
-
|
168
|
-
if job # if the serializer returns a nil or false
|
169
|
-
t = Time.now
|
170
|
-
submitter = @submitter_class.new(@connection, @serializer)
|
171
|
-
context = @execution_context_class.new(submitter, {STR_logger => @logger})
|
172
|
-
|
173
|
-
begin
|
174
|
-
@middleware_stack.around_execution(job, context) do
|
175
|
-
job.method(:run).arity.zero? ? job.run : job.run(context)
|
176
|
-
end
|
177
|
-
@logger.info { "[worker] Finished #{job.inspect} in %0.2fs" % (Time.now - t) }
|
178
|
-
rescue => e
|
179
|
-
@logger.error { "[worker] Failed #{job.inspect} with a #{e}" }
|
180
|
-
raise e
|
181
|
-
end
|
182
|
-
end
|
183
|
-
end
|
184
|
-
|
185
|
-
@connection.delete_message(message_id)
|
188
|
+
message = @execution_queue.pop(nonblock=true)
|
189
|
+
handle_message(message)
|
186
190
|
rescue ThreadError # Queue is empty
|
187
191
|
sleep SLEEP_SECONDS_ON_EMPTY_QUEUE
|
188
192
|
Thread.pass
|
189
|
-
rescue SystemExit, SignalException, Interrupt => e # Time to quit
|
190
|
-
@logger.error { "[worker] Signaled, will quit the consumer" }
|
191
|
-
return
|
192
193
|
rescue => e # anything else, at or below StandardError that does not need us to quit
|
193
|
-
@logger.
|
194
|
-
@logger.
|
195
|
-
@logger.
|
196
|
-
e.backtrace.each { |s| @logger.
|
194
|
+
@logger.error { "[worker] Failed #{message.inspect} with #{e}" }
|
195
|
+
@logger.error(e.class)
|
196
|
+
@logger.error(e.message)
|
197
|
+
e.backtrace.each { |s| @logger.error{"\t#{s}"} }
|
197
198
|
end
|
198
199
|
|
199
200
|
STR_logger = 'logger'
|
data/spec/sqewer/cli_spec.rb
CHANGED
@@ -6,7 +6,8 @@ describe Sqewer::CLI, :sqs => true do
|
|
6
6
|
submitter = Sqewer::Connection.default
|
7
7
|
|
8
8
|
stderr = Tempfile.new('worker-stderr')
|
9
|
-
|
9
|
+
stderr.sync = true
|
10
|
+
|
10
11
|
pid = fork { $stderr.reopen(stderr); exec("ruby #{__dir__}/cli_app.rb") }
|
11
12
|
|
12
13
|
Thread.new do
|
@@ -16,8 +17,9 @@ describe Sqewer::CLI, :sqs => true do
|
|
16
17
|
end
|
17
18
|
end
|
18
19
|
|
19
|
-
sleep
|
20
|
+
sleep 8 # Give it some time to process all the jobs
|
20
21
|
Process.kill("USR1", pid)
|
22
|
+
sleep 2
|
21
23
|
|
22
24
|
generated_files = Dir.glob('*-result')
|
23
25
|
expect(generated_files).not_to be_empty
|
@@ -32,6 +34,7 @@ describe Sqewer::CLI, :sqs => true do
|
|
32
34
|
submitter = Sqewer::Connection.default
|
33
35
|
|
34
36
|
stderr = Tempfile.new('worker-stderr')
|
37
|
+
stderr.sync = true
|
35
38
|
|
36
39
|
pid = fork { $stderr.reopen(stderr); exec("ruby #{__dir__}/cli_app.rb") }
|
37
40
|
|
@@ -31,27 +31,23 @@ describe Sqewer::Connection do
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
|
-
describe '#
|
34
|
+
describe '#receive_messages' do
|
35
35
|
it 'uses the batched receive feature' do
|
36
36
|
s = described_class.new('https://fake-queue')
|
37
37
|
|
38
|
-
|
39
|
-
expect(
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
double('SQSMessage', receipt_handle: SecureRandom.hex(4), body: 'Some message')
|
44
|
-
end
|
45
|
-
# Yields arrays of messages, so...
|
46
|
-
blk.call(bulk)
|
38
|
+
fake_sqs_client = double('Client')
|
39
|
+
expect(Aws::SQS::Client).to receive(:new) { fake_sqs_client }
|
40
|
+
|
41
|
+
fake_messages = (1..5).map {
|
42
|
+
double(receipt_handle: SecureRandom.hex(4), body: SecureRandom.random_bytes(128))
|
47
43
|
}
|
44
|
+
fake_response = double(messages: fake_messages)
|
48
45
|
|
49
|
-
|
50
|
-
|
51
|
-
receives << [sqs_message_handle, sqs_message_body]
|
52
|
-
end
|
46
|
+
expect(fake_sqs_client).to receive(:receive_message).with({:queue_url=>"https://fake-queue", :wait_time_seconds=>5,
|
47
|
+
:max_number_of_messages=>10}).and_return(fake_response)
|
53
48
|
|
54
|
-
|
49
|
+
messages = s.receive_messages
|
50
|
+
expect(messages.length).to eq(5)
|
55
51
|
end
|
56
52
|
end
|
57
53
|
end
|
data/spec/sqewer/worker_spec.rb
CHANGED
@@ -1,8 +1,21 @@
|
|
1
1
|
require_relative '../spec_helper'
|
2
2
|
|
3
3
|
describe Sqewer::Worker, :sqs => true do
|
4
|
-
let(:
|
4
|
+
let(:test_logger) {
|
5
|
+
$stderr.sync = true
|
6
|
+
ENV['SHOW_TEST_LOGS'] ? Logger.new($stderr) : Logger.new(StringIO.new(''))
|
7
|
+
}
|
5
8
|
|
9
|
+
it 'has all the necessary attributes' do
|
10
|
+
attrs = [:logger, :connection, :serializer, :middleware_stack,
|
11
|
+
:execution_context_class, :submitter_class, :isolator, :num_threads]
|
12
|
+
default_worker = described_class.default
|
13
|
+
attrs.each do | attr_name |
|
14
|
+
expect(default_worker).to respond_to(attr_name)
|
15
|
+
expect(default_worker.public_send(attr_name)).not_to be_nil
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
6
19
|
it 'supports .default' do
|
7
20
|
default_worker = described_class.default
|
8
21
|
expect(default_worker).to respond_to(:start)
|
@@ -15,7 +28,7 @@ describe Sqewer::Worker, :sqs => true do
|
|
15
28
|
end
|
16
29
|
|
17
30
|
it 'can go through the full cycle of initialize, start, stop, start, stop' do
|
18
|
-
worker = described_class.new(logger:
|
31
|
+
worker = described_class.new(logger: test_logger)
|
19
32
|
worker.start
|
20
33
|
worker.stop
|
21
34
|
worker.start
|
@@ -34,17 +47,11 @@ describe Sqewer::Worker, :sqs => true do
|
|
34
47
|
client = Aws::SQS::Client.new
|
35
48
|
client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: '{"foo":')
|
36
49
|
|
37
|
-
|
38
|
-
logger_to_string = Logger.new(StringIO.new(logger_output))
|
39
|
-
|
40
|
-
worker = described_class.new(logger: logger_to_string)
|
50
|
+
worker = described_class.new(logger: test_logger)
|
41
51
|
|
42
52
|
worker.start
|
43
53
|
sleep 2
|
44
54
|
worker.stop
|
45
|
-
|
46
|
-
expect(logger_output).to include('unexpected token at \'{"foo":')
|
47
|
-
expect(logger_output).to include('Stopping (clean shutdown)')
|
48
55
|
end
|
49
56
|
end
|
50
57
|
|
@@ -55,22 +62,19 @@ describe Sqewer::Worker, :sqs => true do
|
|
55
62
|
client = Aws::SQS::Client.new
|
56
63
|
client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
|
57
64
|
|
58
|
-
|
59
|
-
logger_to_string = Logger.new(StringIO.new(logger_output))
|
60
|
-
|
61
|
-
worker = described_class.new(logger: logger_to_string)
|
65
|
+
worker = described_class.new(logger: test_logger)
|
62
66
|
|
63
67
|
worker.start
|
64
68
|
sleep 2
|
65
69
|
worker.stop
|
66
70
|
|
67
|
-
expect(logger_output).to include('uninitialized constant UnknownJobClass')
|
68
|
-
expect(logger_output).to include('Stopping (clean shutdown)')
|
71
|
+
# expect(logger_output).to include('uninitialized constant UnknownJobClass')
|
72
|
+
# expect(logger_output).to include('Stopping (clean shutdown)')
|
69
73
|
end
|
70
74
|
end
|
71
75
|
|
72
76
|
context 'with a job that spawns another job' do
|
73
|
-
it 'sets up the processing pipeline so that jobs can execute in sequence' do
|
77
|
+
it 'sets up the processing pipeline so that jobs can execute in sequence (with threads)' do
|
74
78
|
class SecondaryJob
|
75
79
|
def run
|
76
80
|
File.open('secondary-job-run','w') {}
|
@@ -88,9 +92,7 @@ describe Sqewer::Worker, :sqs => true do
|
|
88
92
|
client = Aws::SQS::Client.new
|
89
93
|
client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
|
90
94
|
|
91
|
-
|
92
|
-
logger_to_string = Logger.new(StringIO.new(logger_output))
|
93
|
-
worker = described_class.new(logger: logger_to_string, num_threads: 8)
|
95
|
+
worker = described_class.new(logger: test_logger, num_threads: 8)
|
94
96
|
|
95
97
|
worker.start
|
96
98
|
|
@@ -100,18 +102,30 @@ describe Sqewer::Worker, :sqs => true do
|
|
100
102
|
|
101
103
|
File.unlink('initial-job-run')
|
102
104
|
File.unlink('secondary-job-run')
|
103
|
-
expect(true).to eq(true)
|
104
105
|
ensure
|
105
106
|
worker.stop
|
106
107
|
end
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'sets up the processing pipeline so that jobs can execute in sequence (with processes)' do
|
111
|
+
class SecondaryJob
|
112
|
+
def run
|
113
|
+
File.open('secondary-job-run','w') {}
|
114
|
+
end
|
115
|
+
end
|
107
116
|
|
108
|
-
|
117
|
+
class InitialJob
|
118
|
+
def run(executor)
|
119
|
+
File.open('initial-job-run','w') {}
|
120
|
+
executor.submit!(SecondaryJob.new)
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
payload = JSON.dump({job_class: 'InitialJob'})
|
109
125
|
client = Aws::SQS::Client.new
|
110
126
|
client.send_message(queue_url: ENV.fetch('SQS_QUEUE_URL'), message_body: payload)
|
111
127
|
|
112
|
-
|
113
|
-
logger_to_string = Logger.new(StringIO.new(logger_output))
|
114
|
-
worker = described_class.new(logger: logger_to_string, num_threads: 8, isolator: Sqewer::Isolator.process)
|
128
|
+
worker = described_class.new(logger: test_logger, num_threads: 8, isolator: Sqewer::Isolator.process)
|
115
129
|
|
116
130
|
worker.start
|
117
131
|
|
@@ -121,7 +135,6 @@ describe Sqewer::Worker, :sqs => true do
|
|
121
135
|
|
122
136
|
File.unlink('initial-job-run')
|
123
137
|
File.unlink('secondary-job-run')
|
124
|
-
expect(true).to eq(true)
|
125
138
|
ensure
|
126
139
|
worker.stop
|
127
140
|
end
|
data/sqewer.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: sqewer
|
5
|
+
# stub: sqewer 2.0.0 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "sqewer"
|
9
|
-
s.version = "
|
9
|
+
s.version = "2.0.0"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Julik Tarkhanov"]
|
14
|
-
s.date = "2016-01-
|
14
|
+
s.date = "2016-01-21"
|
15
15
|
s.description = "Process jobs from SQS"
|
16
16
|
s.email = "me@julik.nl"
|
17
17
|
s.extra_rdoc_files = [
|
@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
|
|
19
19
|
]
|
20
20
|
s.files = [
|
21
21
|
".gitlab-ci.yml",
|
22
|
+
".travis.yml",
|
22
23
|
".yardopts",
|
23
24
|
"DETAILS.md",
|
24
25
|
"FAQ.md",
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sqewer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Julik Tarkhanov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|
@@ -186,6 +186,7 @@ extra_rdoc_files:
|
|
186
186
|
- README.md
|
187
187
|
files:
|
188
188
|
- ".gitlab-ci.yml"
|
189
|
+
- ".travis.yml"
|
189
190
|
- ".yardopts"
|
190
191
|
- DETAILS.md
|
191
192
|
- FAQ.md
|