shoryuken-later 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +6 -14
- data/README.md +21 -106
- data/Rakefile +0 -0
- data/lib/shoryuken/later/active_job_adapter.rb +5 -35
- data/lib/shoryuken/later/cli.rb +52 -47
- data/lib/shoryuken/later/client.rb +0 -0
- data/lib/shoryuken/later/launcher.rb +42 -0
- data/lib/shoryuken/later/manager.rb +138 -0
- data/lib/shoryuken/later/poller.rb +13 -5
- data/lib/shoryuken/later/version.rb +1 -1
- data/lib/shoryuken/later/worker.rb +0 -0
- data/shoryuken-later.gemspec +3 -3
- data/spec/shoryuken/later/client_spec.rb +0 -0
- data/spec/shoryuken/later/poller_spec.rb +16 -2
- data/spec/shoryuken/worker_spec.rb +0 -0
- data/spec/spec_helper.rb +3 -1
- metadata +22 -19
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
metadata.gz: !binary |-
|
9
|
-
NjYxMDExNjQzYjhhNTJmZWE3ZmZmZWMxYjRmN2IwYjU1OTU0Nzg4NTIyOGQx
|
10
|
-
YzM0NzUzNjZjYzAzOTBjNDZiZGEzYTk3ZDU0MjdmM2ZlNmI0OTg5ZWQ4YjM2
|
11
|
-
NWU1Y2U0NDZmMTZkY2QwYjE5MTJlYzJjMmI1MzA5NzVkNGE2YTE=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
MzRiYjI0NDUxMGExNzVmMjA3ZDRlNDlkM2E0NzgzZTc1ZGM5YWE2YzY0OGQy
|
14
|
-
Y2Q0N2QwZjEwMzM2YmUxYzMxYjBkYzUxOWQ0ZjFkNjMwOTNhNjM2MzkxMzdi
|
15
|
-
ZWRiOGRlYjk1MTIzM2IzYjU1ZWJkOWJlNWI2OWQxY2JlMWFiNzg=
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 452bf02360d9ab6eb4a73581d937476a0b49c2d1
|
4
|
+
data.tar.gz: 69348f8206e4c630cbc21b62a0dd9aac83051609
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b9392aa905bf2ea5be0211bf485cf157e34d4f8da7fef8a96ec3590d12fdd9b07ebf1ffd9a597871994c227aacd2e1eba9d91dc755f3976cd966bbf3f7f9814c
|
7
|
+
data.tar.gz: f0d5b96a457574ca36a352809f949255fa794ebecf73486e3789e86342e0e357aa6d859c6e56a2ea5416f0f314e097eeef959ca7e44bfd23850d43939ad90b8f
|
data/README.md
CHANGED
@@ -1,15 +1,26 @@
|
|
1
1
|
# shoryuken-later
|
2
2
|
|
3
3
|
A scheduling plugin for [Shoryuken](https://github.com/phstc/shoryuken) that uses [Dynamo DB](https://aws.amazon.com/dynamodb/)
|
4
|
-
to
|
4
|
+
to schedule messages arbitrarily far into the future.
|
5
5
|
|
6
6
|
## Features
|
7
7
|
|
8
|
-
###
|
8
|
+
### Integration with Shoryuken::Worker
|
9
|
+
|
10
|
+
A new method named `perform_later` is added to `Shoryuken::Worker` allowing messages to be delayed arbitrarily far into the future. If the delay is 15 minutes or less, then the message is enqueued into the specified SQS `:queue` as usual. Otherwise, the message is inserted into the specified DynamoDB `:schedule_table`.
|
11
|
+
|
12
|
+
```ruby
|
13
|
+
require 'shoryuken-later'
|
9
14
|
|
10
|
-
|
15
|
+
class MyWorker
|
16
|
+
include Shoryuken::Worker
|
17
|
+
|
18
|
+
shoryuken_options queue: 'default', schedule_table: 'default_schedule'
|
19
|
+
end
|
11
20
|
|
12
|
-
|
21
|
+
# Schedules a message to be processed 30 minutes from now.
|
22
|
+
MyWorker.perform_later(Time.now + 30 * 60, 'Foobar')
|
23
|
+
```
|
13
24
|
|
14
25
|
### One or more schedule tables
|
15
26
|
|
@@ -27,19 +38,21 @@ later:
|
|
27
38
|
You can use the same configuration file for both `Shoryuken` and `Shoryuken::Later`, because the new configuration options are namespaced.
|
28
39
|
|
29
40
|
```yaml
|
30
|
-
#
|
41
|
+
# This key is used by both Shoryuken and Shoryuken::Later
|
31
42
|
aws:
|
32
43
|
access_key_id: ... # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
|
33
44
|
secret_access_key: ... # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
|
34
45
|
region: us-east-1 # or <%= ENV['AWS_REGION'] %>
|
35
|
-
logfile: some/path/to/file.log
|
36
46
|
|
37
47
|
# This key is only used by Shoryuken::Later
|
38
48
|
later:
|
39
49
|
delay: 5 * 60 # How frequently to poll the schedule table, in seconds.
|
40
|
-
pidfile: some/path/to/file.pid
|
41
50
|
tables:
|
42
51
|
- table1
|
52
|
+
|
53
|
+
# These keys are used by both Shoryuken and Shoryuken::Later
|
54
|
+
logfile: some/path/to/file.log
|
55
|
+
pidfile: some/path/to/file.pid
|
43
56
|
|
44
57
|
# These keys are only used by Shoryuken
|
45
58
|
concurrency: 3
|
@@ -47,102 +60,4 @@ delay: 0
|
|
47
60
|
queues:
|
48
61
|
- [queue1, 1]
|
49
62
|
- [queue2, 2]
|
50
|
-
```
|
51
|
-
|
52
|
-
## Usage
|
53
|
-
|
54
|
-
### Starting the schedule poller
|
55
|
-
|
56
|
-
Start the `shoryuken-later` schedule poller with a command like:
|
57
|
-
|
58
|
-
```shell
|
59
|
-
bundle exec shoryuken-later --config shoryuken.yml
|
60
|
-
```
|
61
|
-
|
62
|
-
Run it as a daemon inside of your Rails app with a command like:
|
63
|
-
|
64
|
-
```shell
|
65
|
-
bundle exec shoryuken-later --config shoryuken.yml --rails --daemon
|
66
|
-
```
|
67
|
-
|
68
|
-
[Command-line options](https://github.com/joekhoobyar/shoryuken-later/wiki/Command-line-options)
|
69
|
-
|
70
|
-
|
71
|
-
### Integration with ActiveJob
|
72
|
-
|
73
|
-
A custom ActiveJob adapter can used to support delaying messages arbitrarily far into the future.
|
74
|
-
|
75
|
-
```ruby
|
76
|
-
# config/application.rb
|
77
|
-
config.active_job.queue_adapter = :shoryuken_later
|
78
|
-
```
|
79
|
-
|
80
|
-
When you use the `:shoryuken_later` queue adapter, jobs to be performed farther than 15 minutes into the future (by setting the `wait` or `wait_until` ActiveJob options), will be inserted into the *default* schedule table. You can set the default schedule table in an initializer.
|
81
|
-
|
82
|
-
```ruby
|
83
|
-
# config/initializers/shoryuken_later.rb
|
84
|
-
Shoryuken::Later.default_table = "#{Rails.env}_myapp_later"
|
85
|
-
```
|
86
|
-
|
87
|
-
|
88
|
-
### Integration with Shoryuken::Worker
|
89
|
-
|
90
|
-
A new method named `perform_later` is added to `Shoryuken::Worker` allowing messages to be delayed arbitrarily far into the future. If the delay is 15 minutes or less, then the message is enqueued into the specified SQS `:queue` as usual. Otherwise, the message is inserted into the specified DynamoDB `:schedule_table`.
|
91
|
-
|
92
|
-
```ruby
|
93
|
-
require 'shoryuken-later'
|
94
|
-
|
95
|
-
class MyWorker
|
96
|
-
include Shoryuken::Worker
|
97
|
-
|
98
|
-
shoryuken_options queue: 'default', schedule_table: 'default_schedule'
|
99
|
-
end
|
100
|
-
|
101
|
-
# Schedules a message to be processed 30 minutes from now.
|
102
|
-
MyWorker.perform_later(Time.now + 30 * 60, 'Foobar')
|
103
|
-
```
|
104
|
-
|
105
|
-
|
106
|
-
## Requirements
|
107
|
-
|
108
|
-
Ruby 1.9 or greater.
|
109
|
-
|
110
|
-
## Installation
|
111
|
-
|
112
|
-
Add this line to your application's Gemfile:
|
113
|
-
|
114
|
-
```ruby
|
115
|
-
gem 'shoryuken-later'
|
116
|
-
```
|
117
|
-
|
118
|
-
Or to get the latest updates:
|
119
|
-
|
120
|
-
```ruby
|
121
|
-
gem 'shoryuken-later', github: 'joekhoobyar/shoryuken-later', branch: 'master'
|
122
|
-
```
|
123
|
-
|
124
|
-
And then execute:
|
125
|
-
|
126
|
-
$ bundle
|
127
|
-
|
128
|
-
Or install it yourself as:
|
129
|
-
|
130
|
-
$ gem install shoryuken-later
|
131
|
-
|
132
|
-
## Documentation
|
133
|
-
|
134
|
-
Learn about using Shoryuken::Later at the [Shoryuken::Later Wiki](https://github.com/joekhoobyar/shoryuken-later/wiki).
|
135
|
-
|
136
|
-
Learn about using Shoryuken at the [Shoryuken Wiki](https://github.com/phstc/shoryuken/wiki).
|
137
|
-
|
138
|
-
## Credits
|
139
|
-
|
140
|
-
[Pablo Cantero](https://github.com/phstc), creator of [Shoryuken](https://github.com/phstc/shoryuken), and [everybody who contributed to it](https://github.com/phstc/shoryuken/graphs/contributors). I borrowed a lot of code from Shoryuken itself as a shortcut to making this gem.
|
141
|
-
|
142
|
-
## Contributing
|
143
|
-
|
144
|
-
1. Fork it ( https://github.com/joekhoobyar/shoryuken-later/fork )
|
145
|
-
2. Create your feature branch (`git checkout -b my-new-feature`)
|
146
|
-
3. Commit your changes (`git commit -am 'Add some feature'`)
|
147
|
-
4. Push to the branch (`git push origin my-new-feature`)
|
148
|
-
5. Create a new Pull Request
|
63
|
+
```
|
data/Rakefile
CHANGED
File without changes
|
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# Build on top of Shoryuken's ActiveJob adapter.
|
2
2
|
|
3
3
|
# @see ActiveJob::QueueAdapter::ShoryukenAdapter
|
4
4
|
|
@@ -20,20 +20,16 @@ module ActiveJob
|
|
20
20
|
# To use Shoryuken::Later set the queue_adapter config to +:shoryuken_later+.
|
21
21
|
#
|
22
22
|
# Rails.application.config.active_job.queue_adapter = :shoryuken_later
|
23
|
-
class ShoryukenLaterAdapter
|
23
|
+
class ShoryukenLaterAdapter < ShoryukenAdapter
|
24
|
+
JobWrapper = ShoryukenAdapter::JobWrapper
|
25
|
+
|
24
26
|
class << self
|
25
|
-
def enqueue(job) #:nodoc:
|
26
|
-
register_worker!(job)
|
27
|
-
|
28
|
-
Shoryuken::Client.send_message(job.queue_name, job.serialize, message_attributes: message_attributes)
|
29
|
-
end
|
30
|
-
|
31
27
|
def enqueue_at(job, timestamp) #:nodoc:
|
32
28
|
register_worker!(job)
|
33
29
|
|
34
30
|
delay = (timestamp - Time.current.to_f).round
|
35
31
|
if delay > 15.minutes
|
36
|
-
Shoryuken::Later::Client.put_item(Shoryuken::Later.default_table, perform_at:
|
32
|
+
Shoryuken::Later::Client.put_item(Shoryuken::Later.default_table, perform_at: delay.to_i,
|
37
33
|
shoryuken_queue: job.queue_name, shoryuken_class: JobWrapper.to_s,
|
38
34
|
shoryuken_args: JSON.dump(body: job.serialize, options: {}))
|
39
35
|
else
|
@@ -41,32 +37,6 @@ module ActiveJob
|
|
41
37
|
message_attributes: message_attributes)
|
42
38
|
end
|
43
39
|
end
|
44
|
-
|
45
|
-
|
46
|
-
private
|
47
|
-
|
48
|
-
def register_worker!(job)
|
49
|
-
Shoryuken.register_worker(job.queue_name, JobWrapper)
|
50
|
-
end
|
51
|
-
|
52
|
-
def message_attributes
|
53
|
-
@message_attributes ||= {
|
54
|
-
'shoryuken_class' => {
|
55
|
-
string_value: JobWrapper.to_s,
|
56
|
-
data_type: 'String'
|
57
|
-
}
|
58
|
-
}
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
class JobWrapper #:nodoc:
|
63
|
-
include Shoryuken::Worker
|
64
|
-
|
65
|
-
shoryuken_options body_parser: :json, auto_delete: true
|
66
|
-
|
67
|
-
def perform(sqs_msg, hash)
|
68
|
-
Base.execute hash
|
69
|
-
end
|
70
40
|
end
|
71
41
|
end
|
72
42
|
end
|
data/lib/shoryuken/later/cli.rb
CHANGED
@@ -7,7 +7,6 @@ require 'singleton'
|
|
7
7
|
require 'optparse'
|
8
8
|
require 'erb'
|
9
9
|
require 'shoryuken/later'
|
10
|
-
require 'timers'
|
11
10
|
|
12
11
|
module Shoryuken
|
13
12
|
module Later
|
@@ -15,10 +14,12 @@ module Shoryuken
|
|
15
14
|
include Shoryuken::Util
|
16
15
|
include Singleton
|
17
16
|
|
17
|
+
attr_accessor :launcher
|
18
|
+
|
18
19
|
def run(args)
|
19
20
|
self_read, self_write = IO.pipe
|
20
21
|
|
21
|
-
%w[INT TERM USR1 USR2].each do |sig|
|
22
|
+
%w[INT TERM USR1 USR2 TTIN].each do |sig|
|
22
23
|
trap sig do
|
23
24
|
self_write.puts(sig)
|
24
25
|
end
|
@@ -33,49 +34,38 @@ module Shoryuken
|
|
33
34
|
validate!
|
34
35
|
daemonize
|
35
36
|
write_pid
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
poll_tables
|
48
|
-
@timers.every(Shoryuken::Later.poll_delay){ poll_tables }
|
49
|
-
|
50
|
-
# Loop watching for signals and firing off of timers
|
51
|
-
while @timers
|
52
|
-
interval = @timers.wait_interval
|
53
|
-
readable, writable = IO.select([self_read], nil, nil, interval)
|
54
|
-
if readable
|
55
|
-
handle_signal readable.first.gets.strip
|
56
|
-
else
|
57
|
-
@timers.fire
|
58
|
-
end
|
59
|
-
end
|
60
|
-
rescue Interrupt
|
61
|
-
@timers.cancel
|
62
|
-
exit 0
|
37
|
+
load_celluloid
|
38
|
+
|
39
|
+
require 'shoryuken/later/launcher'
|
40
|
+
@launcher = Shoryuken::Later::Launcher.new
|
41
|
+
|
42
|
+
begin
|
43
|
+
launcher.run
|
44
|
+
|
45
|
+
while readable_io = IO.select([self_read])
|
46
|
+
signal = readable_io.first[0].gets.strip
|
47
|
+
handle_signal(signal)
|
63
48
|
end
|
49
|
+
rescue Interrupt
|
50
|
+
launcher.stop(shutdown: true)
|
51
|
+
exit 0
|
64
52
|
end
|
65
53
|
end
|
66
|
-
|
67
|
-
protected
|
68
|
-
|
69
|
-
def poll_tables
|
70
|
-
logger.debug "Polling schedule tables"
|
71
|
-
@pollers.each do |poller|
|
72
|
-
poller.poll
|
73
|
-
end
|
74
|
-
logger.debug "Polling done"
|
75
|
-
end
|
76
54
|
|
77
55
|
private
|
78
56
|
|
57
|
+
def load_celluloid
|
58
|
+
raise "Celluloid cannot be required until here, or it will break Shoryuken::Later's daemonization" if defined?(::Celluloid) && Shoryuken::Later.options[:daemon]
|
59
|
+
|
60
|
+
# Celluloid can't be loaded until after we've daemonized
|
61
|
+
# because it spins up threads and creates locks which get
|
62
|
+
# into a very bad state if forked.
|
63
|
+
require 'celluloid/autostart'
|
64
|
+
Celluloid.logger = (Shoryuken::Later.options[:verbose] ? Shoryuken::Later.logger : nil)
|
65
|
+
|
66
|
+
require 'shoryuken/later/manager'
|
67
|
+
end
|
68
|
+
|
79
69
|
def load_rails
|
80
70
|
# Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
|
81
71
|
|
@@ -89,7 +79,6 @@ module Shoryuken
|
|
89
79
|
::Rails::Application.initializer "shoryuken-later.eager_load" do
|
90
80
|
::Rails.application.config.eager_load = true
|
91
81
|
end
|
92
|
-
require 'shoryuken/later/active_job_adapter' if defined?(::ActiveJob)
|
93
82
|
require File.expand_path("config/environment.rb")
|
94
83
|
end
|
95
84
|
|
@@ -128,7 +117,7 @@ module Shoryuken
|
|
128
117
|
end
|
129
118
|
|
130
119
|
def write_pid
|
131
|
-
if path = Shoryuken::Later.options[:
|
120
|
+
if path = Shoryuken::Later.options[:pidfile]
|
132
121
|
File.open(path, 'w') do |f|
|
133
122
|
f.puts Process.pid
|
134
123
|
end
|
@@ -136,7 +125,7 @@ module Shoryuken
|
|
136
125
|
end
|
137
126
|
|
138
127
|
def parse_options(argv)
|
139
|
-
opts = {
|
128
|
+
opts = {}
|
140
129
|
|
141
130
|
@parser = OptionParser.new do |o|
|
142
131
|
o.on '-d', '--daemon', 'Daemonize process' do |arg|
|
@@ -164,7 +153,7 @@ module Shoryuken
|
|
164
153
|
end
|
165
154
|
|
166
155
|
o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
|
167
|
-
opts[:
|
156
|
+
opts[:pidfile] = arg
|
168
157
|
end
|
169
158
|
|
170
159
|
o.on '-v', '--verbose', 'Print more verbose output' do |arg|
|
@@ -192,10 +181,28 @@ module Shoryuken
|
|
192
181
|
case sig
|
193
182
|
when 'USR1'
|
194
183
|
logger.info "Received USR1, will soft shutdown down"
|
195
|
-
|
196
|
-
|
184
|
+
|
185
|
+
launcher.stop
|
186
|
+
|
187
|
+
exit 0
|
188
|
+
when 'TTIN'
|
189
|
+
Thread.list.each do |thread|
|
190
|
+
logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
|
191
|
+
if thread.backtrace
|
192
|
+
logger.info thread.backtrace.join("\n")
|
193
|
+
else
|
194
|
+
logger.info "<no backtrace available>"
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
idle = launcher.manager.instance_variable_get(:@idle).size
|
199
|
+
busy = launcher.manager.instance_variable_get(:@busy).size
|
200
|
+
tables = launcher.manager.instance_variable_get(:@tables)
|
201
|
+
|
202
|
+
logger.info "Idle: #{idle}, Busy: #{busy}, Polled Tables: #{tables.join(', ')}"
|
197
203
|
else
|
198
204
|
logger.info "Received #{sig}, will shutdown down"
|
205
|
+
|
199
206
|
raise Interrupt
|
200
207
|
end
|
201
208
|
end
|
@@ -208,10 +215,8 @@ module Shoryuken
|
|
208
215
|
|
209
216
|
config = options[:config_file] ? parse_config(options[:config_file]).deep_symbolize_keys : {}
|
210
217
|
|
211
|
-
Shoryuken::Later.options[:later].merge!(config.delete(:later) || {})
|
212
218
|
Shoryuken::Later.options.merge!(config)
|
213
219
|
|
214
|
-
Shoryuken::Later.options[:later].merge!(options.delete(:later) || {})
|
215
220
|
Shoryuken::Later.options.merge!(options)
|
216
221
|
|
217
222
|
# Tables from command line options take precedence...
|
File without changes
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# All of this has been "borrowed" from Shoryuken.
|
2
|
+
|
3
|
+
# @see Shoryuken::Launcher
|
4
|
+
module Shoryuken
|
5
|
+
module Later
|
6
|
+
class Launcher
|
7
|
+
include Celluloid
|
8
|
+
include Shoryuken::Util
|
9
|
+
|
10
|
+
trap_exit :actor_died
|
11
|
+
|
12
|
+
attr_accessor :manager
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
@manager = Shoryuken::Later::Manager.new_link
|
16
|
+
|
17
|
+
@done = false
|
18
|
+
end
|
19
|
+
|
20
|
+
def stop(options = {})
|
21
|
+
watchdog('Later::Launcher#stop') do
|
22
|
+
@done = true
|
23
|
+
|
24
|
+
manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken::Later.options[:timeout])
|
25
|
+
manager.wait(:shutdown)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def run
|
30
|
+
watchdog('Later::Launcher#run') do
|
31
|
+
manager.async.start
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def actor_died(actor, reason)
|
36
|
+
return if @done
|
37
|
+
logger.warn 'Shoryuken::Later died due to the following error, cannot recover, process exiting'
|
38
|
+
exit 1
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,138 @@
|
|
1
|
+
# Most of this has been "borrowed" from Shoryuken, but then repurposed for periodic polling.
|
2
|
+
|
3
|
+
# @see Shoryuken::Manager
|
4
|
+
require 'set'
|
5
|
+
require 'shoryuken/later/poller'
|
6
|
+
|
7
|
+
module Shoryuken
|
8
|
+
module Later
|
9
|
+
class Manager
|
10
|
+
include Celluloid
|
11
|
+
include Shoryuken::Util
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@tables = Shoryuken::Later.tables.dup.uniq
|
15
|
+
|
16
|
+
@done = false
|
17
|
+
|
18
|
+
@idle = Set.new([])
|
19
|
+
@busy = Set.new([])
|
20
|
+
@timers = {}
|
21
|
+
|
22
|
+
@tables.each{|table| Poller.supervise_as :"poller-#{table}", current_actor, table }
|
23
|
+
end
|
24
|
+
|
25
|
+
def start
|
26
|
+
logger.info 'Starting'
|
27
|
+
|
28
|
+
# Start a poller for every table being polled.
|
29
|
+
@tables.each do |table|
|
30
|
+
dispatch table
|
31
|
+
|
32
|
+
# Save the timer so it can be cancelled at shutdown.
|
33
|
+
@timers[table] = every(Shoryuken::Later.poll_delay) { dispatch table }
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def stop(options = {})
|
38
|
+
watchdog('Later::Manager#stop died') do
|
39
|
+
@done = true
|
40
|
+
|
41
|
+
@timers.each_value{|timer| timer.cancel if timer }
|
42
|
+
@timers.clear
|
43
|
+
|
44
|
+
logger.info { "Shutting down #{@idle.size} idle poller(s)" }
|
45
|
+
|
46
|
+
@idle.each do |name|
|
47
|
+
poller = Actor[name] and poller.alive? and poller.terminate
|
48
|
+
end
|
49
|
+
@idle.clear
|
50
|
+
|
51
|
+
if @busy.empty?
|
52
|
+
return after(0) { signal(:shutdown) }
|
53
|
+
end
|
54
|
+
|
55
|
+
if options[:shutdown]
|
56
|
+
hard_shutdown_in(options[:timeout])
|
57
|
+
else
|
58
|
+
soft_shutdown(options[:timeout])
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def poller_done(table, poller)
|
64
|
+
watchdog('Later::Manager#poller_done died') do
|
65
|
+
logger.debug { "Poller done for '#{table}'" }
|
66
|
+
|
67
|
+
name = :"poller-#{table}"
|
68
|
+
@busy.delete name
|
69
|
+
|
70
|
+
if stopped?
|
71
|
+
poller.terminate if poller.alive?
|
72
|
+
else
|
73
|
+
@idle << name
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def poller_ready(table, poller)
|
79
|
+
watchdog('Later::Manager#poller_ready died') do
|
80
|
+
logger.debug { "Poller for '#{table}' ready" }
|
81
|
+
|
82
|
+
name = :"poller-#{table}"
|
83
|
+
@busy.delete name
|
84
|
+
@idle << name
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def stopped?
|
89
|
+
@done
|
90
|
+
end
|
91
|
+
|
92
|
+
private
|
93
|
+
|
94
|
+
def dispatch(table)
|
95
|
+
name = :"poller-#{table}"
|
96
|
+
|
97
|
+
# Only start polling if the poller is idle.
|
98
|
+
if ! stopped? && @idle.include?(name)
|
99
|
+
@idle.delete(name)
|
100
|
+
@busy << name
|
101
|
+
|
102
|
+
Actor[name].async.poll
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def soft_shutdown(delay)
|
107
|
+
logger.info { "Waiting for #{@busy.size} busy pollers" }
|
108
|
+
|
109
|
+
if @busy.size > 0
|
110
|
+
after(delay) { soft_shutdown(delay) }
|
111
|
+
else
|
112
|
+
after(0) { signal(:shutdown) }
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def hard_shutdown_in(delay)
|
117
|
+
logger.info { "Waiting for #{@busy.size} busy pollers" }
|
118
|
+
logger.info { "Pausing up to #{delay} seconds to allow pollers to finish..." }
|
119
|
+
|
120
|
+
after(delay) do
|
121
|
+
watchdog("Later::Manager#hard_shutdown_in died") do
|
122
|
+
if @busy.size > 0
|
123
|
+
logger.info { "Hard shutting down #{@busy.size} busy pollers" }
|
124
|
+
|
125
|
+
@busy.each do |busy|
|
126
|
+
if poller = Actor[busy]
|
127
|
+
t = poller.bare_object.actual_work_thread
|
128
|
+
t.raise Shutdown if poller.alive?
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
after(0) { signal(:shutdown) }
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
@@ -3,12 +3,16 @@ require 'json'
|
|
3
3
|
module Shoryuken
|
4
4
|
module Later
|
5
5
|
class Poller
|
6
|
+
include Celluloid
|
6
7
|
include Shoryuken::Util
|
7
8
|
|
8
9
|
attr_reader :table_name
|
9
10
|
|
10
|
-
def initialize(table_name)
|
11
|
+
def initialize(manager, table_name)
|
12
|
+
@manager = manager
|
11
13
|
@table_name = table_name
|
14
|
+
|
15
|
+
@manager.async.poller_ready(@table_name, self)
|
12
16
|
end
|
13
17
|
|
14
18
|
def poll
|
@@ -21,10 +25,12 @@ module Shoryuken
|
|
21
25
|
while item = next_item
|
22
26
|
id = item.attributes['id']
|
23
27
|
logger.info "Found message #{id} from '#{@table_name}'"
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
+
defer do
|
29
|
+
if sent_msg = process_item(item)
|
30
|
+
logger.debug { "Enqueued message #{id} from '#{@table_name}' as #{sent_msg.id}" }
|
31
|
+
else
|
32
|
+
logger.debug { "Skipping already queued message #{id} from '#{@table_name}'" }
|
33
|
+
end
|
28
34
|
end
|
29
35
|
end
|
30
36
|
|
@@ -33,6 +39,8 @@ module Shoryuken
|
|
33
39
|
logger.error "Error fetching message: #{ex}"
|
34
40
|
logger.error ex.backtrace.first
|
35
41
|
end
|
42
|
+
|
43
|
+
@manager.async.poller_done(@table_name, self)
|
36
44
|
end
|
37
45
|
end
|
38
46
|
|
File without changes
|
data/shoryuken-later.gemspec
CHANGED
@@ -20,13 +20,13 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
-
spec.required_ruby_version = '>=
|
23
|
+
spec.required_ruby_version = '>= 2.0.0'
|
24
24
|
|
25
25
|
spec.add_development_dependency "bundler", '>= 1.3.5'
|
26
26
|
spec.add_development_dependency "rake", '~> 10.0'
|
27
27
|
spec.add_development_dependency "rspec", '~> 3.0', '< 3.1'
|
28
28
|
|
29
29
|
spec.add_dependency "aws-sdk-v1"
|
30
|
-
spec.add_dependency "
|
31
|
-
spec.add_dependency "shoryuken", "
|
30
|
+
spec.add_dependency "celluloid", "~> 0.16.0"
|
31
|
+
spec.add_dependency "shoryuken", "~> 0.0.5"
|
32
32
|
end
|
File without changes
|
@@ -1,7 +1,9 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'shoryuken/later/poller'
|
3
|
+
require 'shoryuken/later/manager'
|
3
4
|
|
4
5
|
describe Shoryuken::Later::Poller do
|
6
|
+
let(:manager) { double Shoryuken::Later::Manager, poller_ready: nil, poller_done: nil }
|
5
7
|
let(:ddb_table) { double 'DynamoDb Table' }
|
6
8
|
let(:ddb_items) { double 'Table Items' }
|
7
9
|
let(:table) { 'shoryuken_later' }
|
@@ -15,26 +17,38 @@ describe Shoryuken::Later::Poller do
|
|
15
17
|
end
|
16
18
|
|
17
19
|
before do
|
20
|
+
allow(manager).to receive(:async).and_return(manager)
|
18
21
|
allow(Shoryuken::Later::Client).to receive(:tables).with(table).and_return(ddb_table)
|
19
22
|
end
|
20
23
|
|
21
24
|
subject do
|
22
|
-
described_class.new(table)
|
25
|
+
described_class.new(manager, table)
|
23
26
|
end
|
24
27
|
|
28
|
+
describe '#initialize' do
|
29
|
+
it 'informs the manager that the poller is ready' do
|
30
|
+
expect(manager).to receive(:poller_ready).once
|
31
|
+
|
32
|
+
subject.inspect
|
33
|
+
subject.inspect
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
25
37
|
describe '#poll' do
|
26
38
|
it 'pulls items from #next_item, and processes with #process_item' do
|
27
39
|
items = [ddb_item]
|
28
40
|
expect_any_instance_of(described_class).to receive(:next_item).twice { items.pop }
|
29
41
|
expect_any_instance_of(described_class).to receive(:process_item).once.with(ddb_item)
|
42
|
+
expect(manager).to receive(:poller_done).once
|
30
43
|
|
31
44
|
subject.poll
|
32
45
|
end
|
33
46
|
|
34
|
-
it '
|
47
|
+
it 'informs the manager after polling is done' do
|
35
48
|
items = []
|
36
49
|
expect_any_instance_of(described_class).to receive(:next_item).once { items.pop }
|
37
50
|
expect_any_instance_of(described_class).not_to receive(:process_item)
|
51
|
+
expect(manager).to receive(:poller_done).once
|
38
52
|
|
39
53
|
subject.poll
|
40
54
|
end
|
File without changes
|
data/spec/spec_helper.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'bundler/setup'
|
2
2
|
Bundler.setup
|
3
3
|
|
4
|
+
require 'celluloid'
|
4
5
|
require 'shoryuken-later'
|
5
6
|
require 'json'
|
6
7
|
|
@@ -15,6 +16,7 @@ if File.exists? options_file
|
|
15
16
|
end
|
16
17
|
|
17
18
|
Shoryuken.logger.level = Logger::UNKNOWN
|
19
|
+
Celluloid.logger.level = Logger::UNKNOWN
|
18
20
|
|
19
21
|
# For Ruby 1.9
|
20
22
|
module Kernel
|
@@ -66,7 +68,7 @@ RSpec.configure do |config|
|
|
66
68
|
TestWorker.get_shoryuken_options['queue'] = 'shoryuken_later'
|
67
69
|
TestWorker.get_shoryuken_options['schedule_table'] = 'shoryuken_later'
|
68
70
|
|
69
|
-
Shoryuken.
|
71
|
+
Shoryuken.worker_registry.clear
|
70
72
|
Shoryuken.register_worker('shoryuken_later', TestWorker)
|
71
73
|
end
|
72
74
|
end
|
metadata
CHANGED
@@ -1,27 +1,27 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken-later
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Joe Khoobyar
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-03-
|
11
|
+
date: 2015-03-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: 1.3.5
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 1.3.5
|
27
27
|
- !ruby/object:Gem::Dependency
|
@@ -62,46 +62,46 @@ dependencies:
|
|
62
62
|
name: aws-sdk-v1
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
64
64
|
requirements:
|
65
|
-
- -
|
65
|
+
- - '>='
|
66
66
|
- !ruby/object:Gem::Version
|
67
67
|
version: '0'
|
68
68
|
type: :runtime
|
69
69
|
prerelease: false
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
|
-
- -
|
72
|
+
- - '>='
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '0'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
|
-
name:
|
76
|
+
name: celluloid
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
78
78
|
requirements:
|
79
79
|
- - ~>
|
80
80
|
- !ruby/object:Gem::Version
|
81
|
-
version:
|
81
|
+
version: 0.16.0
|
82
82
|
type: :runtime
|
83
83
|
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
86
|
- - ~>
|
87
87
|
- !ruby/object:Gem::Version
|
88
|
-
version:
|
88
|
+
version: 0.16.0
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
90
|
name: shoryuken
|
91
91
|
requirement: !ruby/object:Gem::Requirement
|
92
92
|
requirements:
|
93
|
-
- -
|
93
|
+
- - ~>
|
94
94
|
- !ruby/object:Gem::Version
|
95
|
-
version: 0.0.
|
95
|
+
version: 0.0.5
|
96
96
|
type: :runtime
|
97
97
|
prerelease: false
|
98
98
|
version_requirements: !ruby/object:Gem::Requirement
|
99
99
|
requirements:
|
100
|
-
- -
|
100
|
+
- - ~>
|
101
101
|
- !ruby/object:Gem::Version
|
102
|
-
version: 0.0.
|
103
|
-
description:
|
104
|
-
|
102
|
+
version: 0.0.5
|
103
|
+
description: "\n This gem provides a scheduling plugin (using Dynamo DB) for Shoryuken,
|
104
|
+
as well as an ActiveJob adapter\n "
|
105
105
|
email:
|
106
106
|
- joe@khoobyar.name
|
107
107
|
executables:
|
@@ -120,6 +120,8 @@ files:
|
|
120
120
|
- lib/shoryuken/later/active_job_adapter.rb
|
121
121
|
- lib/shoryuken/later/cli.rb
|
122
122
|
- lib/shoryuken/later/client.rb
|
123
|
+
- lib/shoryuken/later/launcher.rb
|
124
|
+
- lib/shoryuken/later/manager.rb
|
123
125
|
- lib/shoryuken/later/poller.rb
|
124
126
|
- lib/shoryuken/later/version.rb
|
125
127
|
- lib/shoryuken/later/worker.rb
|
@@ -138,17 +140,17 @@ require_paths:
|
|
138
140
|
- lib
|
139
141
|
required_ruby_version: !ruby/object:Gem::Requirement
|
140
142
|
requirements:
|
141
|
-
- -
|
143
|
+
- - '>='
|
142
144
|
- !ruby/object:Gem::Version
|
143
|
-
version:
|
145
|
+
version: 2.0.0
|
144
146
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
145
147
|
requirements:
|
146
|
-
- -
|
148
|
+
- - '>='
|
147
149
|
- !ruby/object:Gem::Version
|
148
150
|
version: '0'
|
149
151
|
requirements: []
|
150
152
|
rubyforge_project:
|
151
|
-
rubygems_version: 2.
|
153
|
+
rubygems_version: 2.4.5
|
152
154
|
signing_key:
|
153
155
|
specification_version: 4
|
154
156
|
summary: A scheduling plugin (using Dynamo DB) for Shoryuken
|
@@ -157,3 +159,4 @@ test_files:
|
|
157
159
|
- spec/shoryuken/later/poller_spec.rb
|
158
160
|
- spec/shoryuken/worker_spec.rb
|
159
161
|
- spec/spec_helper.rb
|
162
|
+
has_rdoc:
|