sqewer 5.0.0 → 5.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/ACTIVE_JOB.md +64 -0
- data/FAQ.md +0 -4
- data/Gemfile +4 -0
- data/README.md +4 -0
- data/Rakefile +1 -0
- data/bin/sqewer +7 -0
- data/bin/sqewer_rails +10 -0
- data/lib/sqewer.rb +18 -1
- data/lib/sqewer/atomic_counter.rb +2 -2
- data/lib/sqewer/cli.rb +3 -3
- data/lib/sqewer/connection.rb +16 -16
- data/lib/sqewer/connection_messagebox.rb +4 -4
- data/lib/sqewer/execution_context.rb +5 -5
- data/lib/sqewer/extensions/active_job_adapter.rb +78 -0
- data/lib/sqewer/{contrib → extensions}/appsignal_wrapper.rb +4 -4
- data/lib/sqewer/extensions/railtie.rb +12 -0
- data/lib/sqewer/middleware_stack.rb +7 -7
- data/lib/sqewer/null_logger.rb +1 -1
- data/lib/sqewer/resubmit.rb +17 -0
- data/lib/sqewer/serializer.rb +25 -25
- data/lib/sqewer/simple_job.rb +11 -11
- data/lib/sqewer/state_lock.rb +2 -2
- data/lib/sqewer/submitter.rb +17 -3
- data/lib/sqewer/version.rb +1 -1
- data/lib/sqewer/worker.rb +47 -47
- data/spec/spec_helper.rb +8 -2
- data/spec/sqewer/active_job_spec.rb +113 -0
- data/spec/sqewer/cli_spec.rb +48 -31
- data/spec/sqewer/serializer_spec.rb +51 -56
- data/spec/sqewer/submitter_spec.rb +18 -0
- data/spec/sqewer/worker_spec.rb +11 -9
- data/sqewer.gemspec +21 -5
- metadata +55 -5
- data/lib/sqewer/contrib/performable.rb +0 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ddea2036ab00b9c68e83ed4bcd64bf7a46ab5114
|
4
|
+
data.tar.gz: 9a0efcf0a664e35d628d1c602f7fb0339490233a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 432aa4dca6829cf25a5a402b0d8da73cfc3719aba51479068fb7572bbac0badf15319970f97c015b95dcf3bcce25b1004233a0ecbf5e969ef238c4ce4db2a775
|
7
|
+
data.tar.gz: 2bce53adecc4d6e869b05c0a9d0cdafae7a4cf021b1e73f7ad25cbf061ac7ddd1fb61a2f571bc2a1c61415be35ea6900ae915a2ed8d577df26bb2c29208de0b1
|
data/.travis.yml
CHANGED
@@ -8,3 +8,4 @@ env:
|
|
8
8
|
- secure: E+uNoJmvjq5+3/PYUs2a1bRh7bCjAbcYEIKJzOVgIiI+gWB4dLpTKwVW/fqJbjCDIGPw9OwiwphN51zLEo0ie5AJIkavcjWRcdazKq8usokAz/A5Ilru5YvGXJ5UQtF8TJNsZfJGOGMJwL5d/MsDSd3/sjy88wrFiSeeNlHA06KDBrGdZXE4/dOb5Z84cSxPaD7nDaGP36V3Uv88X272oezODm9Mp+Fsf4goGoU9I9PBDD36pCXAlpURWpyWMKa2TzEXfgN43TYxta4p+bu6BA871df31j3BfVKwY1K/59eN9HqW8xWF9rgcgLo3NgQuaoqde3ryCE4jqKymjtG1/R5bdbQsn07R7/OagQMPVkbn07Aq+X6Q8GVTkrMl829z87QiMETNnzpsHl/1RD3DVnUo4k+XmqLbwW6NRl452j3eEHsiMvxf1vt8Qn1hu61qm98WGVNCGFWX/eAu/iTpvDcQOkoUuUEfGnXPaGeM8GT7tNdKptT8gsg/H0Dwb4d0qwA8uAEWttmZBmI9145hpnIk+hkDkH22dzhTus8d7CfFDSpu7s3y9434wiCYF5pkugyWJb5dd24d00ssgF5gqhLU/s48WcBwipl8mMGu/H/5ezh/pG9dL2uDIUjKRrUybE/vmDJGy8ZEDnOwJ04NJ701vQjFybgZRXXyU5ESEJQ=
|
9
9
|
- secure: hq1T5MsHGmEnfQjOJlu/GzUPJ4WH0+lZOHezb4i0rFmRtWnmOMO+YUA34q9P6VQROk5KcAiRKtqZYmxZwTjyJVk2DTcJkvUAVzWBHD+MZAsH/lql+lnUJTuZRapYI85LChmEZkDMhYxZYfxTqGTbE4GjLtVtGN8zmTRStLxtvqyfiaU0SuGzyw9wfso/QhqHadQAQ7NttTYyiSiO2vHxK7rhmNi99MxAeVL3qZ3H2mcypSEE7CZd+nFa2CQxh/Guce0i+N2PDYTB6dsvA4xA5Xh/e+han3g4x/sRpic2C/9HS2zRYf0iN20+pFyotp5SaW3o5Q5S3iUqQtyZoR2lN1YMd4csyyKp/0G4QOjAPIN3FIef/RTVLqdfZXaord4iAg9GN9eKdtgYuS7JgyDl0VOmTpS2RYuDP+PGR2HRXXVc0gHt2aFl/oy+DTXh6emcOkpZtRZlbHqtVz5WXZZftm8niId+UXQmJuIQU+YgIiFDmviafapwjaYoXHCv8QMAGqNBP5EWDfxvYyPtGhQrj37sRYpfnUCkdJCiZvZLhQgQ0aIvR3KJIFpXtQmWaaT3e7u07B6weBB7Icto6WflfKtYHvVd1YfOX02/J8byhJbzRY+Qjqrq+5aQR1L3AKbmv8AooZdcm/CrEx6MP7xuIM0fjp1oU4cKQe74KX/KuqE=
|
10
10
|
- secure: fWzpY63ZAXOtjEJuYoya/mwzJ7fZQJGsN6hhamXorOhUgaeP2vdC7/6gK3PPMIXOJ5NXPiANGbAUWKgLLWJ3F7rK2y9UCUWTzvE2CK/13vUV9de7byQfAjfHo7wjDJuPLved+WmSM7ASVjzGq9wQS8MTeYUymSNVY3xiZcvNTO2XY9AAu32SSWDs1ayXiCGE9DyN5WEvEEPixHqpplZLvSqTtKtEZVdP/a3tPuzogYFvoJ/ZEfG9LRr4nnbwuKCaTljpYaZc/8yFbhhJgCrTuOKb0gB/UFeMjd0WX6MQWofzIj152d0ghCTlgtESR2LQJdSXA2BO0ZzSqYj8NKTAcZ8IsaUj+6kBgfQGHu43TF/dB877v8pY55HS8a3gFPDK/6OZhRT7UcFZgle1qvX422CD196fRHktSBcDTx9jQsG8r7Ray9oJygXQr6tKhrRkjMkkBAiBd/GkIJkJu5RecmlLOZMJ+YeYa95kWDUXwo3HGK9MxtFyuplSRYk4mQ1jbrJXdmufQuTazl2FYL+elOUeUb8ej3iniUcmyDGU6tZx84aZTbBOaovmeQqr3pXIaXYMhvkRGLD3Ky7SNZN9ZzFczRWcKStyQal+BYK1R3RU8cD0m0PEpUxV5ERP50E3GyN/0d0bg22LywK9aWDHOhCzjI0emVvSAn8ZiY78SSo=
|
11
|
+
- AWS_REGION=eu-central-1
|
data/ACTIVE_JOB.md
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# Sqewer with ActiveJob
|
2
|
+
|
3
|
+
This gem includes a queue adapter for usage with ActiveJob in Rails 4.2+. The functionality
|
4
|
+
is well-tested and should function for any well-conforming ActiveJob subclasses.
|
5
|
+
|
6
|
+
To run the default `sqewer` worker setup against your Rails application, first set it as the
|
7
|
+
executing backend for ActiveJob in your Rails app configuration, set your `SQS_QUEUE_URL`
|
8
|
+
in the environment variables, and make sure you can access it using your default (envvar-based
|
9
|
+
or machine role based) AWS credentials. Then, set sqewer as the adapter for ActiveJob:
|
10
|
+
|
11
|
+
class Application < Rails::Application
|
12
|
+
...
|
13
|
+
config.active_job.queue_adapter = :sqewer
|
14
|
+
end
|
15
|
+
|
16
|
+
and then run
|
17
|
+
|
18
|
+
$ bundle exec sqewer_rails
|
19
|
+
|
20
|
+
in your rails source tree, via a foreman Procfile or similar. If you want to run your own worker binary
|
21
|
+
for executing the jobs, be aware that you _have_ to eager-load your Rails application's code explicitly
|
22
|
+
before the Sqewer worker is started. The worker is threaded and any kind of autoloading does not generally
|
23
|
+
play nice with threading. So do not forget to add this in your worker code:
|
24
|
+
|
25
|
+
Rails.application.eager_load!
|
26
|
+
|
27
|
+
For handling error reporting within your Sqewer worker, set up a middleware stack as described in the documentation.
|
28
|
+
|
29
|
+
## ActiveJob feature support matrix
|
30
|
+
|
31
|
+
Compared to the matrix of features as seen in the
|
32
|
+
[official ActiveJob documentation](http://edgeapi.rubyonrails.org/classes/ActiveJob/QueueAdapters.html)
|
33
|
+
`sqewer` has the following support for various ActiveJob options, in comparison to the builtin
|
34
|
+
ActiveJob adapters:
|
35
|
+
|
36
|
+
| | Async | Queues | Delayed | Priorities | Timeout | Retries |
|
37
|
+
|-------------------|-------|--------|------------|------------|---------|---------|
|
38
|
+
| sqewer | Yes | No | Yes | No | No | Global |
|
39
|
+
| // | // | // | // | // | // | // |
|
40
|
+
| Active Job Async | Yes | Yes | Yes | No | No | No |
|
41
|
+
| Active Job Inline | No | Yes | N/A | N/A | N/A | N/A |
|
42
|
+
|
43
|
+
Retries are set up globally for the entire SQS queue. There is no specific queue setting per job,
|
44
|
+
since all the messages go to the queue available to `Sqewer.submit!`.
|
45
|
+
|
46
|
+
There is no timeout handling, if you need it you may want to implement it within your jobs proper.
|
47
|
+
Retries are handled on Sqewer level for as many deliveries as your SQS settings permit.
|
48
|
+
|
49
|
+
## Delay handling
|
50
|
+
|
51
|
+
Delayed execution is handled via a combination
|
52
|
+
of the `delay_seconds` SQS parameter and the `_execute_after` job key (see the serializer documentation
|
53
|
+
in Sqewer for more). In a nutshell - if you postpone a job by less than 900 seconds, the standard delivery
|
54
|
+
delay option will be used - and the job will become visible for workers on the SQS queue only after this period.
|
55
|
+
|
56
|
+
If a larger delay is used, the job will receive an additional field called `_execute_after`, which will contain
|
57
|
+
a UNIX timestamp in seconds of when it must be executed at the earliest. In addition, the maximum permitted SQS
|
58
|
+
delivery delay will be set for it. If the job then gets redelivered, Sqewer will automatically put it back on the
|
59
|
+
queue with the same maximum delay, and will continue doing so for as long as necessary.
|
60
|
+
|
61
|
+
Note that this will incur extra receives and sends on the queue, and even though it is not substantial,
|
62
|
+
it will not be free. We think that this is an acceptable workaround for now, though. If you want a better approach,
|
63
|
+
you may be better off using a Rails scheduling system and use a cron job or similar to spin up your enqueue
|
64
|
+
for the actual, executable background task.
|
data/FAQ.md
CHANGED
@@ -3,10 +3,6 @@
|
|
3
3
|
This document tries to answer some questions that may arise when reading or using the library. Hopefully
|
4
4
|
this can provide some answers with regards to how things are put together.
|
5
5
|
|
6
|
-
## Why no ActiveJob?
|
7
|
-
|
8
|
-
An adapter will be added in the future
|
9
|
-
|
10
6
|
## Why separate `new` and `run` methods instead of just `perform`?
|
11
7
|
|
12
8
|
Because the job needs access to the execution context of the worker. It turned out that keeping the context
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -58,6 +58,10 @@ For more detailed usage information, see [DETAILS.md](./DETAILS.md)
|
|
58
58
|
|
59
59
|
Please see [FAQ.md](./FAQ.md). This might explain some decisions behind the library in greater detail.
|
60
60
|
|
61
|
+
## Usage with Rails via ActiveJob
|
62
|
+
|
63
|
+
Please see [ACTIVE_JOB.md](./ACTIVE_JOB.md) for the exact description.
|
64
|
+
|
61
65
|
## Contributing to the library
|
62
66
|
|
63
67
|
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
|
data/Rakefile
CHANGED
@@ -22,6 +22,7 @@ Jeweler::Tasks.new do |gem|
|
|
22
22
|
gem.summary = %Q{A full-featured library for all them worker needs}
|
23
23
|
gem.email = "me@julik.nl"
|
24
24
|
gem.authors = ["Julik Tarkhanov"]
|
25
|
+
gem.executables = ["sqewer", "sqewer_rails"]
|
25
26
|
# dependencies defined in Gemfile
|
26
27
|
end
|
27
28
|
Jeweler::RubygemsDotOrgTasks.new
|
data/bin/sqewer
ADDED
data/bin/sqewer_rails
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require_relative '../lib/sqewer'
|
3
|
+
require_relative '../lib/sqewer/extensions/active_job_adapter'
|
4
|
+
require_relative '../lib/sqewer/extensions/rails'
|
5
|
+
|
6
|
+
# Make sure all the classes are preloaded before the threads are spun up
|
7
|
+
Rails.application.eager_load!
|
8
|
+
|
9
|
+
# ...and start
|
10
|
+
Sqewer::CLI.start(Sqewer::Worker.new(logger: Rails.logger))
|
data/lib/sqewer.rb
CHANGED
@@ -1,11 +1,28 @@
|
|
1
1
|
# The enclosing module for the library
|
2
2
|
module Sqewer
|
3
|
-
|
3
|
+
# Eager-load everything except extensions
|
4
|
+
Dir.glob(__dir__ + '/**/*.rb').each do |path|
|
5
|
+
if path != __FILE__ && File.dirname(path) !~ /\/extensions$/
|
6
|
+
require path
|
7
|
+
end
|
8
|
+
end
|
4
9
|
|
10
|
+
# Loads a particular Sqewer extension that is not loaded
|
11
|
+
# automatically during the gem require.
|
12
|
+
#
|
13
|
+
# @param extension_name[String] the name of the extension to load (like `active_job_adapter`)
|
14
|
+
def self.require_extension(extension_name)
|
15
|
+
path = File.join("sqewer", "extensions", extension_name)
|
16
|
+
require_relative path
|
17
|
+
end
|
18
|
+
|
5
19
|
# Shortcut access to Submitter#submit.
|
6
20
|
#
|
7
21
|
# @see {Sqewer::Submitter#submit!}
|
8
22
|
def self.submit!(*jobs, **options)
|
9
23
|
Sqewer::Submitter.default.submit!(*jobs, **options)
|
10
24
|
end
|
25
|
+
|
26
|
+
# If we are within Rails, load the railtie
|
27
|
+
require_relative 'sqewer/extensions/railtie' if defined?(Rails)
|
11
28
|
end
|
@@ -5,14 +5,14 @@ class Sqewer::AtomicCounter
|
|
5
5
|
def initialize
|
6
6
|
@m, @v = Mutex.new, 0
|
7
7
|
end
|
8
|
-
|
8
|
+
|
9
9
|
# Returns the current value of the counter
|
10
10
|
#
|
11
11
|
# @return [Fixnum] the current value of the counter
|
12
12
|
def to_i
|
13
13
|
@m.synchronize { @v + 0 }
|
14
14
|
end
|
15
|
-
|
15
|
+
|
16
16
|
# Increments the counter
|
17
17
|
#
|
18
18
|
# @return [Fixnum] the current value of the counter
|
data/lib/sqewer/cli.rb
CHANGED
@@ -20,7 +20,7 @@ module Sqewer::CLI
|
|
20
20
|
# Signal not supported
|
21
21
|
end
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
begin
|
25
25
|
worker.start
|
26
26
|
# The worker is non-blocking, so in the main CLI process we select() on the signal
|
@@ -34,7 +34,7 @@ module Sqewer::CLI
|
|
34
34
|
exit 1
|
35
35
|
end
|
36
36
|
end
|
37
|
-
|
37
|
+
|
38
38
|
def handle_signal(worker, sig)
|
39
39
|
case sig
|
40
40
|
when 'USR1', 'TERM'
|
@@ -46,6 +46,6 @@ module Sqewer::CLI
|
|
46
46
|
raise Interrupt
|
47
47
|
end
|
48
48
|
end
|
49
|
-
|
49
|
+
|
50
50
|
extend self
|
51
51
|
end
|
data/lib/sqewer/connection.rb
CHANGED
@@ -8,18 +8,18 @@
|
|
8
8
|
class Sqewer::Connection
|
9
9
|
DEFAULT_TIMEOUT_SECONDS = 5
|
10
10
|
BATCH_RECEIVE_SIZE = 10
|
11
|
-
|
11
|
+
|
12
12
|
# A wrapper for most important properties of the received message
|
13
13
|
class Message < Struct.new(:receipt_handle, :body)
|
14
14
|
def inspect
|
15
15
|
body.inspect
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
def has_body?
|
19
19
|
body && !body.empty?
|
20
20
|
end
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
# Returns the default adapter, connected to the queue set via the `SQS_QUEUE_URL`
|
24
24
|
# environment variable.
|
25
25
|
def self.default
|
@@ -27,7 +27,7 @@ class Sqewer::Connection
|
|
27
27
|
rescue KeyError => e
|
28
28
|
raise "SQS_QUEUE_URL not set in the environment. This is the queue URL that the default that Sqewer uses"
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
# Initializes a new adapter, with access to the SQS queue at the given URL.
|
32
32
|
#
|
33
33
|
# @param queue_url[String] the SQS queue URL (the URL can be copied from your AWS console)
|
@@ -35,7 +35,7 @@ class Sqewer::Connection
|
|
35
35
|
require 'aws-sdk'
|
36
36
|
@queue_url = queue_url
|
37
37
|
end
|
38
|
-
|
38
|
+
|
39
39
|
# Receive at most 10 messages from the queue, and return the array of Message objects.
|
40
40
|
#
|
41
41
|
# @return [Array<Message>] an array of Message objects
|
@@ -46,7 +46,7 @@ class Sqewer::Connection
|
|
46
46
|
Message.new(message.receipt_handle, message.body)
|
47
47
|
end
|
48
48
|
end
|
49
|
-
|
49
|
+
|
50
50
|
# Send a message to the backing queue
|
51
51
|
#
|
52
52
|
# @param message_body[String] the message to send
|
@@ -56,7 +56,7 @@ class Sqewer::Connection
|
|
56
56
|
def send_message(message_body, **kwargs_for_send)
|
57
57
|
send_multiple_messages {|via| via.send_message(message_body, **kwargs_for_send) }
|
58
58
|
end
|
59
|
-
|
59
|
+
|
60
60
|
# Stores the messages for the SQS queue (both deletes and sends), and yields them in allowed batch sizes
|
61
61
|
class MessageBuffer < Struct.new(:messages)
|
62
62
|
MAX_RECORDS = 10
|
@@ -67,7 +67,7 @@ class Sqewer::Connection
|
|
67
67
|
messages.each_slice(MAX_RECORDS){|batch| yield(batch)}
|
68
68
|
end
|
69
69
|
end
|
70
|
-
|
70
|
+
|
71
71
|
# Saves the messages to send to the SQS queue
|
72
72
|
class SendBuffer < MessageBuffer
|
73
73
|
def send_message(message_body, **kwargs_for_send)
|
@@ -78,7 +78,7 @@ class Sqewer::Connection
|
|
78
78
|
messages << m
|
79
79
|
end
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
# Saves the receipt handles to batch-delete from the SQS queue
|
83
83
|
class DeleteBuffer < MessageBuffer
|
84
84
|
def delete_message(receipt_handle)
|
@@ -88,7 +88,7 @@ class Sqewer::Connection
|
|
88
88
|
messages << m
|
89
89
|
end
|
90
90
|
end
|
91
|
-
|
91
|
+
|
92
92
|
# Send multiple messages. If any messages fail to send, an exception will be raised.
|
93
93
|
#
|
94
94
|
# @yield [#send_message] the object you can send messages through (will be flushed at method return)
|
@@ -105,7 +105,7 @@ class Sqewer::Connection
|
|
105
105
|
end
|
106
106
|
end
|
107
107
|
end
|
108
|
-
|
108
|
+
|
109
109
|
# Deletes a message after it has been succesfully decoded and processed
|
110
110
|
#
|
111
111
|
# @param message_identifier[String] the ID of the message to delete. For SQS, it is the receipt handle
|
@@ -113,7 +113,7 @@ class Sqewer::Connection
|
|
113
113
|
def delete_message(message_identifier)
|
114
114
|
delete_multiple_messages {|via| via.delete_message(message_identifier) }
|
115
115
|
end
|
116
|
-
|
116
|
+
|
117
117
|
# Deletes multiple messages after they all have been succesfully decoded and processed.
|
118
118
|
#
|
119
119
|
# @yield [#delete_message] an object you can delete an individual message through
|
@@ -121,7 +121,7 @@ class Sqewer::Connection
|
|
121
121
|
def delete_multiple_messages
|
122
122
|
buffer = DeleteBuffer.new
|
123
123
|
yield(buffer)
|
124
|
-
|
124
|
+
|
125
125
|
buffer.each_batch do | batch |
|
126
126
|
resp = client.delete_message_batch(queue_url: @queue_url, entries: batch)
|
127
127
|
failed = resp.failed
|
@@ -131,9 +131,9 @@ class Sqewer::Connection
|
|
131
131
|
end
|
132
132
|
end
|
133
133
|
end
|
134
|
-
|
134
|
+
|
135
135
|
private
|
136
|
-
|
136
|
+
|
137
137
|
class RetryWrapper < Struct.new(:sqs_client)
|
138
138
|
MAX_RETRIES = 1000
|
139
139
|
# Provide retrying wrappers for all the methods of Aws::SQS::Client that we actually use
|
@@ -153,7 +153,7 @@ class Sqewer::Connection
|
|
153
153
|
end
|
154
154
|
end
|
155
155
|
end
|
156
|
-
|
156
|
+
|
157
157
|
def client
|
158
158
|
@client ||= RetryWrapper.new(Aws::SQS::Client.new)
|
159
159
|
end
|
@@ -23,7 +23,7 @@ class Sqewer::ConnectionMessagebox
|
|
23
23
|
@sends = []
|
24
24
|
@mux = Mutex.new
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
# Saves the given body and the keyword arguments (such as delay_seconds) to be sent into the queue.
|
28
28
|
# If there are more sends in the same flush, they will be batched using batched deletes.G
|
29
29
|
#
|
@@ -33,7 +33,7 @@ class Sqewer::ConnectionMessagebox
|
|
33
33
|
@sends << [message_body, kwargs_for_send]
|
34
34
|
}
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
# Saves the given identifier to be deleted from the queue. If there are more
|
38
38
|
# deletes in the same flush, they will be batched using batched deletes.
|
39
39
|
#
|
@@ -43,7 +43,7 @@ class Sqewer::ConnectionMessagebox
|
|
43
43
|
@deletes << message_identifier
|
44
44
|
}
|
45
45
|
end
|
46
|
-
|
46
|
+
|
47
47
|
# Flushes all the accumulated commands to the queue connection.
|
48
48
|
# First the message sends are going to be flushed, then the message deletes.
|
49
49
|
# All of those will use batching where possible.
|
@@ -52,7 +52,7 @@ class Sqewer::ConnectionMessagebox
|
|
52
52
|
@connection.send_multiple_messages do | buffer |
|
53
53
|
@sends.each { |body, kwargs| buffer.send_message(body, **kwargs) }
|
54
54
|
end
|
55
|
-
|
55
|
+
|
56
56
|
@connection.delete_multiple_messages do | buffer |
|
57
57
|
@deletes.each { |id| buffer.delete_message(id) }
|
58
58
|
end
|
@@ -14,14 +14,14 @@ class Sqewer::ExecutionContext
|
|
14
14
|
@params = {}
|
15
15
|
extra_variables.each_pair{|k, v| self[k] = v }
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
# Submits one or more jobs to the queue
|
19
19
|
#
|
20
20
|
# @see {Sqewer::Submitter#submit!}
|
21
21
|
def submit!(job, **execution_options)
|
22
22
|
@submitter.submit!(job, **execution_options)
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
# Sets a key in the execution environment
|
26
26
|
#
|
27
27
|
# @param key[#to_s] the key to set
|
@@ -29,14 +29,14 @@ class Sqewer::ExecutionContext
|
|
29
29
|
def []=(key, value)
|
30
30
|
@params[key.to_s] = value
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
# Returns a key of the execution environment by name
|
34
34
|
#
|
35
35
|
# @param key[#to_s] the key to get
|
36
36
|
def [](key)
|
37
37
|
@params[key.to_s]
|
38
38
|
end
|
39
|
-
|
39
|
+
|
40
40
|
# Returns a key of the execution environment, or executes the given block
|
41
41
|
# if the key is not set
|
42
42
|
#
|
@@ -45,7 +45,7 @@ class Sqewer::ExecutionContext
|
|
45
45
|
def fetch(key, &blk)
|
46
46
|
@params.fetch(key.to_s, &blk)
|
47
47
|
end
|
48
|
-
|
48
|
+
|
49
49
|
# Returns the logger set in the execution environment, or
|
50
50
|
# the NullLogger if no logger is set. Can be used to supply
|
51
51
|
# a logger prefixed with job parameters per job.
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
|
2
|
+
# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
|
3
|
+
module ActiveJob
|
4
|
+
module QueueAdapters
|
5
|
+
|
6
|
+
# Handle Rails ActiveJob through sqewer.
|
7
|
+
# Set it up like so:
|
8
|
+
#
|
9
|
+
# Rails.application.config.active_job.queue_adapter = :sqewer
|
10
|
+
class SqewerAdapter
|
11
|
+
# Works as a Job for sqewer, and wraps an ActiveJob Worker which responds to perform()
|
12
|
+
class Performable
|
13
|
+
|
14
|
+
# Creates a new Performable using the passed ActiveJob object. The resulting Performable
|
15
|
+
# can be sent to any Sqewer queue.
|
16
|
+
#
|
17
|
+
# @param active_job_worker[ActiveJob::Job] the job you want to convert
|
18
|
+
def self.from_active_job(active_job_worker)
|
19
|
+
# Try to grab the job class immediately, so that an error is raised in the unserializer
|
20
|
+
# if the class is not available
|
21
|
+
klass = active_job_worker.class.to_s
|
22
|
+
Kernel.const_get(klass)
|
23
|
+
new(job: active_job_worker.serialize)
|
24
|
+
end
|
25
|
+
|
26
|
+
def initialize(job:)
|
27
|
+
@job = job
|
28
|
+
end
|
29
|
+
|
30
|
+
def to_h
|
31
|
+
{job: @job}
|
32
|
+
end
|
33
|
+
|
34
|
+
def inspect
|
35
|
+
'<%s>' % [@job.inspect]
|
36
|
+
end
|
37
|
+
|
38
|
+
# Runs the contained ActiveJob.
|
39
|
+
def run
|
40
|
+
job = ActiveSupport::HashWithIndifferentAccess.new(@job)
|
41
|
+
if active_record_defined_and_connected?
|
42
|
+
with_active_record_connection_from_pool { Base.execute(job) }
|
43
|
+
else
|
44
|
+
Base.execute(job)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def with_active_record_connection_from_pool
|
51
|
+
ActiveRecord::Base.connection_pool.with_connection { yield }
|
52
|
+
end
|
53
|
+
|
54
|
+
def active_record_defined_and_connected?
|
55
|
+
defined?(ActiveRecord) && ActiveRecord::Base.connected?
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
class << self
|
61
|
+
def enqueue(active_job) #:nodoc:
|
62
|
+
wrapped_job = Performable.from_active_job(active_job)
|
63
|
+
|
64
|
+
Sqewer.submit!(wrapped_job)
|
65
|
+
end
|
66
|
+
|
67
|
+
def enqueue_at(active_job, timestamp) #:nodoc:
|
68
|
+
wrapped_job = Performable.from_active_job(active_job)
|
69
|
+
|
70
|
+
delta_t = (timestamp - Time.now.to_i).to_i
|
71
|
+
|
72
|
+
Sqewer.submit!(wrapped_job, delay_seconds: delta_t)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|