shoryuken 2.1.1 → 3.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +5 -8
- data/.rubocop.yml +8 -2
- data/.travis.yml +6 -0
- data/CHANGELOG.md +167 -0
- data/Gemfile +2 -0
- data/README.md +16 -155
- data/Rakefile +0 -1
- data/bin/cli/base.rb +40 -0
- data/bin/cli/sqs.rb +204 -0
- data/bin/shoryuken +55 -9
- data/examples/default_worker.rb +1 -1
- data/lib/shoryuken/client.rb +3 -15
- data/lib/shoryuken/default_worker_registry.rb +10 -6
- data/lib/shoryuken/environment_loader.rb +60 -57
- data/lib/shoryuken/fetcher.rb +21 -51
- data/lib/shoryuken/launcher.rb +77 -23
- data/lib/shoryuken/logging.rb +0 -5
- data/lib/shoryuken/manager.rb +54 -205
- data/lib/shoryuken/message.rb +4 -13
- data/lib/shoryuken/middleware/chain.rb +1 -18
- data/lib/shoryuken/middleware/server/auto_delete.rb +3 -8
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +16 -17
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +37 -22
- data/lib/shoryuken/middleware/server/timing.rb +2 -2
- data/lib/shoryuken/options.rb +196 -0
- data/lib/shoryuken/polling/base.rb +67 -0
- data/lib/shoryuken/polling/strict_priority.rb +77 -0
- data/lib/shoryuken/polling/weighted_round_robin.rb +66 -0
- data/lib/shoryuken/processor.rb +27 -25
- data/lib/shoryuken/queue.rb +28 -9
- data/lib/shoryuken/runner.rb +131 -0
- data/lib/shoryuken/util.rb +1 -9
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +9 -1
- data/lib/shoryuken.rb +47 -157
- data/shoryuken.gemspec +7 -7
- data/spec/integration/launcher_spec.rb +15 -8
- data/spec/shoryuken/client_spec.rb +2 -45
- data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
- data/spec/shoryuken/environment_loader_spec.rb +54 -0
- data/spec/shoryuken/fetcher_spec.rb +27 -46
- data/spec/shoryuken/manager_spec.rb +80 -93
- data/spec/shoryuken/middleware/chain_spec.rb +0 -24
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
- data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +7 -3
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +56 -33
- data/spec/shoryuken/middleware/server/timing_spec.rb +5 -3
- data/spec/shoryuken/options_spec.rb +100 -0
- data/spec/shoryuken/polling/strict_priority_spec.rb +140 -0
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +99 -0
- data/spec/shoryuken/processor_spec.rb +20 -37
- data/spec/shoryuken/queue_spec.rb +72 -26
- data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +11 -26
- data/spec/shoryuken_spec.rb +1 -48
- data/spec/spec_helper.rb +14 -20
- data/test_workers/endless_interruptive_worker.rb +41 -0
- data/test_workers/endless_uninterruptive_worker.rb +44 -0
- metadata +39 -34
- data/lib/shoryuken/aws_config.rb +0 -64
- data/lib/shoryuken/cli.rb +0 -215
- data/lib/shoryuken/sns_arn.rb +0 -27
- data/lib/shoryuken/topic.rb +0 -17
- data/spec/shoryuken/sns_arn_spec.rb +0 -42
- data/spec/shoryuken/topic_spec.rb +0 -32
- data/spec/shoryuken_endpoint.yml +0 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 942ccdeaca0a94e766b82fa40da9ed091b703ea2
|
|
4
|
+
data.tar.gz: f94a089b56ed37b5db2752ccf418c667fae31b6d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 67c92d3af6c4063f42afe07a3f56c32b847bd1bc76d80f4f9f46cc2aa48d3e7bf82213f4d98fe229265ce9b1387a3649652863f019e73aeb0d809b1a03c5ca4d
|
|
7
|
+
data.tar.gz: 1215eca9e669975edb70fa43855122e6f25c6e54452771ba26435f7e8df37684be20516037e1419a3e7d93a14773231387f471cc2ff8a03eb838f1e52f2c144d
|
data/.codeclimate.yml
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
engines:
|
|
3
|
-
|
|
4
|
-
enabled: true
|
|
5
|
-
bundler-audit:
|
|
3
|
+
reek:
|
|
6
4
|
enabled: true
|
|
7
5
|
duplication:
|
|
8
6
|
enabled: true
|
|
@@ -11,13 +9,12 @@ engines:
|
|
|
11
9
|
- ruby
|
|
12
10
|
fixme:
|
|
13
11
|
enabled: true
|
|
14
|
-
reek:
|
|
15
|
-
enabled: true
|
|
16
12
|
rubocop:
|
|
17
13
|
enabled: true
|
|
14
|
+
config:
|
|
15
|
+
file: .rubocop.yml
|
|
18
16
|
ratings:
|
|
19
17
|
paths:
|
|
20
18
|
- "**.rb"
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
# - "*/spec/**/*"
|
|
19
|
+
exclude_paths:
|
|
20
|
+
- spec/
|
data/.rubocop.yml
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
AllCops:
|
|
2
|
+
TargetRubyVersion: 2.0
|
|
3
3
|
|
|
4
4
|
Style/SignalException:
|
|
5
5
|
Enabled: false
|
|
6
6
|
|
|
7
|
+
Style/DoubleNegation:
|
|
8
|
+
Enabled: false
|
|
9
|
+
|
|
7
10
|
Style/SpaceAroundEqualsInParameterDefault:
|
|
8
11
|
Enabled: false
|
|
9
12
|
|
|
@@ -28,6 +31,9 @@ Metrics/ClassLength:
|
|
|
28
31
|
Metrics/ParameterLists:
|
|
29
32
|
Enabled: false
|
|
30
33
|
|
|
34
|
+
Metrics/LineLength:
|
|
35
|
+
Max: 130
|
|
36
|
+
|
|
31
37
|
Metrics/MethodLength :
|
|
32
38
|
Enabled: false
|
|
33
39
|
|
data/.travis.yml
CHANGED
|
@@ -3,6 +3,8 @@ rvm:
|
|
|
3
3
|
- 2.0.0
|
|
4
4
|
- 2.1.0
|
|
5
5
|
- 2.2.0
|
|
6
|
+
- 2.3.3
|
|
7
|
+
- 2.4.1
|
|
6
8
|
|
|
7
9
|
notifications:
|
|
8
10
|
email:
|
|
@@ -14,3 +16,7 @@ before_install:
|
|
|
14
16
|
- gem update bundler
|
|
15
17
|
after_success:
|
|
16
18
|
- bundle exec codeclimate-test-reporter
|
|
19
|
+
|
|
20
|
+
addons:
|
|
21
|
+
code_climate:
|
|
22
|
+
repo_token: 7709fd21981bb9d2658647a66d959415a1029a83f1c199573828797944f26c52
|
data/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,175 @@
|
|
|
1
|
+
## [v3.1.3] - 2017-07-11
|
|
2
|
+
|
|
3
|
+
- Add queue prefixing support for groups
|
|
4
|
+
- [#405](https://github.com/phstc/shoryuken/pull/405)
|
|
5
|
+
|
|
6
|
+
- Remove dead code
|
|
7
|
+
- [#402](https://github.com/phstc/shoryuken/pull/402)
|
|
8
|
+
|
|
9
|
+
## [v3.1.2] - 2017-07-06
|
|
10
|
+
|
|
11
|
+
- Fix stack level too deep on Ubuntu
|
|
12
|
+
- [#400](https://github.com/phstc/shoryuken/pull/400)
|
|
13
|
+
|
|
14
|
+
## [v3.1.1] - 2017-07-05
|
|
15
|
+
|
|
16
|
+
- Reduce log verbosity introduced in 3.1.0
|
|
17
|
+
- [#397](https://github.com/phstc/shoryuken/pull/397)
|
|
18
|
+
|
|
19
|
+
- Try to prevent stack level too deep on Ubuntu
|
|
20
|
+
- [#396](https://github.com/phstc/shoryuken/pull/396)
|
|
21
|
+
|
|
22
|
+
## [v3.1.0] - 2017-07-02
|
|
23
|
+
|
|
24
|
+
- Add shoryuken sqs delete command
|
|
25
|
+
- [#395](https://github.com/phstc/shoryuken/pull/395)
|
|
26
|
+
|
|
27
|
+
- Add processing groups support; Concurrency per queue support
|
|
28
|
+
- [#389](https://github.com/phstc/shoryuken/pull/389)
|
|
29
|
+
|
|
30
|
+
- Terminate Shoryuken if the fetcher crashes
|
|
31
|
+
- [#389](https://github.com/phstc/shoryuken/pull/389)
|
|
32
|
+
|
|
33
|
+
## [v3.0.11] - 2017-06-24
|
|
34
|
+
|
|
35
|
+
- Add shoryuken sqs create command
|
|
36
|
+
- [#388](https://github.com/phstc/shoryuken/pull/388)
|
|
37
|
+
|
|
38
|
+
## [v3.0.10] - 2017-06-24
|
|
39
|
+
|
|
40
|
+
- Allow aws sdk v3
|
|
41
|
+
- [#381](https://github.com/phstc/shoryuken/pull/381)
|
|
42
|
+
|
|
43
|
+
- Allow configuring Rails via the config file
|
|
44
|
+
- [#387](https://github.com/phstc/shoryuken/pull/387)
|
|
45
|
+
|
|
46
|
+
## [v3.0.9] - 2017-06-05
|
|
47
|
+
|
|
48
|
+
- Allow configuring queue URLs instead of names
|
|
49
|
+
- [#378](https://github.com/phstc/shoryuken/pull/378)
|
|
50
|
+
|
|
51
|
+
## [v3.0.8] - 2017-06-02
|
|
52
|
+
|
|
53
|
+
- Fix miss handling empty batch fetches
|
|
54
|
+
- [#376](https://github.com/phstc/shoryuken/pull/376)
|
|
55
|
+
|
|
56
|
+
- Various minor styling changes :lipstick:
|
|
57
|
+
- [#373](https://github.com/phstc/shoryuken/pull/373)
|
|
58
|
+
|
|
59
|
+
- Logout when batch delete returns any failure
|
|
60
|
+
- [#371](https://github.com/phstc/shoryuken/pull/371)
|
|
61
|
+
|
|
62
|
+
## [v3.0.7] - 2017-05-18
|
|
63
|
+
|
|
64
|
+
- Trigger events for dispatch
|
|
65
|
+
- [#362](https://github.com/phstc/shoryuken/pull/362)
|
|
66
|
+
|
|
67
|
+
- Log (warn) exponential backoff tries
|
|
68
|
+
- [#365](https://github.com/phstc/shoryuken/pull/365)
|
|
69
|
+
|
|
70
|
+
- Fix displaying of long queue names in `shoryuken sqs ls`
|
|
71
|
+
- [#366](https://github.com/phstc/shoryuken/pull/366)
|
|
72
|
+
|
|
73
|
+
## [v3.0.6] - 2017-04-11
|
|
74
|
+
|
|
75
|
+
- Fix delay option type
|
|
76
|
+
- [#356](https://github.com/phstc/shoryuken/pull/356)
|
|
77
|
+
|
|
78
|
+
## [v3.0.5] - 2017-04-09
|
|
79
|
+
|
|
80
|
+
- Pause endless dispatcher to avoid CPU overload
|
|
81
|
+
- [#354](https://github.com/phstc/shoryuken/pull/354)
|
|
82
|
+
|
|
83
|
+
- Auto log processor errors
|
|
84
|
+
- [#355](https://github.com/phstc/shoryuken/pull/355)
|
|
85
|
+
|
|
86
|
+
- Add a delay as a CLI param
|
|
87
|
+
- [#350](https://github.com/phstc/shoryuken/pull/350)
|
|
88
|
+
|
|
89
|
+
- Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
|
|
90
|
+
- [#344](https://github.com/phstc/shoryuken/pull/344)
|
|
91
|
+
|
|
92
|
+
## [v3.0.4] - 2017-03-24
|
|
93
|
+
|
|
94
|
+
- Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
|
|
95
|
+
- [#344](https://github.com/phstc/shoryuken/pull/344)
|
|
96
|
+
|
|
97
|
+
- Fix "Thread exhaustion" error. This issue was most noticed when using long polling. @waynerobinson :beers: for pairing up on this.
|
|
98
|
+
- [#345](https://github.com/phstc/shoryuken/pull/345)
|
|
99
|
+
|
|
100
|
+
## [v3.0.3] - 2017-03-19
|
|
101
|
+
|
|
102
|
+
- Update `sqs` CLI commands to use `get_queue_url` when appropriated
|
|
103
|
+
- [#341](https://github.com/phstc/shoryuken/pull/341)
|
|
104
|
+
|
|
105
|
+
## [v3.0.2] - 2017-03-19
|
|
106
|
+
|
|
107
|
+
- Fix custom SQS client initialization
|
|
108
|
+
- [#335](https://github.com/phstc/shoryuken/pull/335)
|
|
109
|
+
|
|
110
|
+
## [v3.0.1] - 2017-03-13
|
|
111
|
+
|
|
112
|
+
- Fix commands sqs mv and dump `options.delete` checker
|
|
113
|
+
- [#332](https://github.com/phstc/shoryuken/pull/332)
|
|
114
|
+
|
|
115
|
+
## [v3.0.0] - 2017-03-12
|
|
116
|
+
|
|
117
|
+
- Replace Celluloid with Concurrent Ruby
|
|
118
|
+
- [#291](https://github.com/phstc/shoryuken/pull/291)
|
|
119
|
+
|
|
120
|
+
- Remove AWS configuration from Shoryuken. Now AWS should be configured from outside. Check [this](https://github.com/phstc/shoryuken/wiki/Configure-the-AWS-Client) for more details
|
|
121
|
+
- [#317](https://github.com/phstc/shoryuken/pull/317)
|
|
122
|
+
|
|
123
|
+
- Remove deprecation warnings
|
|
124
|
+
- [#326](https://github.com/phstc/shoryuken/pull/326)
|
|
125
|
+
|
|
126
|
+
- Allow dynamic adding queues
|
|
127
|
+
- [#322](https://github.com/phstc/shoryuken/pull/322)
|
|
128
|
+
|
|
129
|
+
- Support retry_intervals passed in as a lambda. Auto coerce intervals into integer
|
|
130
|
+
- [#329](https://github.com/phstc/shoryuken/pull/329)
|
|
131
|
+
|
|
132
|
+
- Add SQS commands `shoryuken help sqs`, such as `ls`, `mv`, `dump` and `requeue`
|
|
133
|
+
- [#330](https://github.com/phstc/shoryuken/pull/330)
|
|
134
|
+
|
|
135
|
+
## [v2.1.3] - 2017-01-27
|
|
136
|
+
|
|
137
|
+
- Show a warn message when batch isn't supported
|
|
138
|
+
- [#302](https://github.com/phstc/shoryuken/pull/302)
|
|
139
|
+
|
|
140
|
+
- Require Celluloid ~> 17
|
|
141
|
+
- [#305](https://github.com/phstc/shoryuken/pull/305)
|
|
142
|
+
|
|
143
|
+
- Fix excessive logging when 0 messages found
|
|
144
|
+
- [#307](https://github.com/phstc/shoryuken/pull/307)
|
|
145
|
+
|
|
146
|
+
## [v2.1.2] - 2016-12-22
|
|
147
|
+
|
|
148
|
+
- Fix loading `logfile` from shoryuken.yml
|
|
149
|
+
- [#296](https://github.com/phstc/shoryuken/pull/296)
|
|
150
|
+
|
|
151
|
+
- Add support for Strict priority polling (pending documentation)
|
|
152
|
+
- [#288](https://github.com/phstc/shoryuken/pull/288)
|
|
153
|
+
|
|
154
|
+
- Add `test_workers` for end-to-end testing supporting
|
|
155
|
+
- [#286](https://github.com/phstc/shoryuken/pull/286)
|
|
156
|
+
|
|
157
|
+
- Update README documenting `configure_client` and `configure_server`
|
|
158
|
+
- [#283](https://github.com/phstc/shoryuken/pull/283)
|
|
159
|
+
|
|
160
|
+
- Fix memory leak caused by async tracking busy threads
|
|
161
|
+
- [#289](https://github.com/phstc/shoryuken/pull/289)
|
|
162
|
+
|
|
163
|
+
- Refactor fetcher, polling strategy and manager
|
|
164
|
+
- [#284](https://github.com/phstc/shoryuken/pull/284)
|
|
165
|
+
|
|
1
166
|
## [v2.1.1] - 2016-12-05
|
|
167
|
+
|
|
2
168
|
- Fix aws deprecation warning message
|
|
3
169
|
- [#279](https://github.com/phstc/shoryuken/pull/279)
|
|
4
170
|
|
|
5
171
|
## [v2.1.0] - 2016-12-03
|
|
172
|
+
|
|
6
173
|
- Fix celluloid "running in BACKPORTED mode" warning
|
|
7
174
|
- [#260](https://github.com/phstc/shoryuken/pull/260)
|
|
8
175
|
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,46 +1,27 @@
|
|
|
1
1
|
# Shoryuken
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
|
|
4
4
|
|
|
5
|
-
Shoryuken _sho-ryu-ken_ is a super-efficient [
|
|
5
|
+
Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
|
|
6
6
|
|
|
7
7
|
[](https://travis-ci.org/phstc/shoryuken)
|
|
8
8
|
[](https://codeclimate.com/github/phstc/shoryuken)
|
|
9
9
|
|
|
10
10
|
## Key features
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
queues:
|
|
22
|
-
- [high_priority, 6]
|
|
23
|
-
- [default, 2]
|
|
24
|
-
- [low_priority, 1]
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
And supposing all the queues are full of messages, the configuration above will make Shoryuken to process `high_priority` 3 times more than `default` and 6 times more than `low_priority`,
|
|
28
|
-
splitting the work among the `concurrency: 25` available processors.
|
|
29
|
-
|
|
30
|
-
If `high_priority` gets empty, Shoryuken will keep using the 25 processors, but only to process `default` (2 times more than `low_priority`) and `low_priority`.
|
|
31
|
-
|
|
32
|
-
If `high_priority` receives a new message, Shoryuken will smoothly increase back the `high_priority` weight one by one until it reaches the weight of 6 again, which is the maximum configured for `high_priority`.
|
|
33
|
-
|
|
34
|
-
If all queues get empty, all processors will be changed to the waiting state and the queues will be checked every `delay: 25`. If any queue receives a new message, Shoryuken will start processing again. [Check the delay option documentation for more information](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#delay).
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
### Fetch in batches
|
|
38
|
-
|
|
39
|
-
To be even more performant and cost effective, Shoryuken fetches SQS messages in batches, so a single SQS request can fetch up to 10 messages.
|
|
12
|
+
- [Rails Active Job](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job)
|
|
13
|
+
- [Queue Load balancing](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#load-balancing)
|
|
14
|
+
- [Concurrency per queue](https://github.com/phstc/shoryuken/wiki/Processing-Groups)
|
|
15
|
+
- [Long Polling](https://github.com/phstc/shoryuken/wiki/Long-Polling)
|
|
16
|
+
- [Batch processing](https://github.com/phstc/shoryuken/wiki/Worker-options#batch)
|
|
17
|
+
- [Auto extend visibility timeout](https://github.com/phstc/shoryuken/wiki/Worker-options#auto_visibility_timeout)
|
|
18
|
+
- [Exponential backoff](https://github.com/phstc/shoryuken/wiki/Worker-options#retry_intervals)
|
|
19
|
+
- [Middleware support](https://github.com/phstc/shoryuken/wiki/Middleware)
|
|
20
|
+
- Amazon SQS CLI. See `shoryuken help sqs`
|
|
40
21
|
|
|
41
22
|
## Requirements
|
|
42
23
|
|
|
43
|
-
Ruby 2.0 or greater.
|
|
24
|
+
Ruby 2.0 or greater.
|
|
44
25
|
|
|
45
26
|
## Installation
|
|
46
27
|
|
|
@@ -50,139 +31,19 @@ Add this line to your application's Gemfile:
|
|
|
50
31
|
gem 'shoryuken'
|
|
51
32
|
```
|
|
52
33
|
|
|
53
|
-
Or to get the latest updates:
|
|
54
|
-
|
|
55
|
-
```ruby
|
|
56
|
-
gem 'shoryuken', github: 'phstc/shoryuken', branch: 'master'
|
|
57
|
-
```
|
|
58
|
-
|
|
59
34
|
And then execute:
|
|
60
35
|
|
|
61
|
-
$ bundle
|
|
62
|
-
|
|
63
|
-
Or install it yourself as:
|
|
64
|
-
|
|
65
|
-
$ gem install shoryuken
|
|
66
|
-
|
|
67
|
-
## Usage
|
|
68
|
-
|
|
69
|
-
### Worker class
|
|
70
|
-
|
|
71
|
-
```ruby
|
|
72
|
-
class MyWorker
|
|
73
|
-
include Shoryuken::Worker
|
|
74
|
-
|
|
75
|
-
shoryuken_options queue: 'default', auto_delete: true
|
|
76
|
-
# shoryuken_options queue: ->{ "#{ENV['environment']}_default" }
|
|
77
|
-
|
|
78
|
-
# shoryuken_options body_parser: :json
|
|
79
|
-
# shoryuken_options body_parser: ->(sqs_msg){ REXML::Document.new(sqs_msg.body) }
|
|
80
|
-
# shoryuken_options body_parser: JSON
|
|
81
|
-
|
|
82
|
-
def perform(sqs_msg, body)
|
|
83
|
-
puts body
|
|
84
|
-
end
|
|
85
|
-
end
|
|
86
|
-
```
|
|
87
|
-
|
|
88
|
-
[Check the Worker options documention](https://github.com/phstc/shoryuken/wiki/Worker-options).
|
|
89
|
-
|
|
90
|
-
### Sending a message
|
|
91
|
-
|
|
92
|
-
[Check the Sending a message documentation](https://github.com/phstc/shoryuken/wiki/Sending-a-message)
|
|
93
|
-
|
|
94
|
-
### Middleware
|
|
95
|
-
|
|
96
|
-
```ruby
|
|
97
|
-
class MyMiddleware
|
|
98
|
-
def call(worker_instance, queue, sqs_msg, body)
|
|
99
|
-
puts 'Before work'
|
|
100
|
-
yield
|
|
101
|
-
puts 'After work'
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
```
|
|
105
|
-
|
|
106
|
-
[Check the Middleware documentation](https://github.com/phstc/shoryuken/wiki/Middleware).
|
|
107
|
-
|
|
108
|
-
### Configuration (worker side)
|
|
109
|
-
|
|
110
|
-
Sample configuration file `shoryuken.yml`.
|
|
111
|
-
|
|
112
|
-
```yaml
|
|
113
|
-
aws:
|
|
114
|
-
access_key_id: ... # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
|
|
115
|
-
secret_access_key: ... # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
|
|
116
|
-
region: us-east-1 # or <%= ENV['AWS_REGION'] %>
|
|
117
|
-
receive_message: # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
|
|
118
|
-
# wait_time_seconds: N # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
|
|
119
|
-
attribute_names:
|
|
120
|
-
- ApproximateReceiveCount
|
|
121
|
-
- SentTimestamp
|
|
122
|
-
concurrency: 25 # The number of allocated threads to process messages. Default 25
|
|
123
|
-
delay: 25 # The delay in seconds to pause a queue when it's empty. Default 0
|
|
124
|
-
queues:
|
|
125
|
-
- [high_priority, 6]
|
|
126
|
-
- [default, 2]
|
|
127
|
-
- [low_priority, 1]
|
|
128
|
-
```
|
|
129
|
-
|
|
130
|
-
The ```aws``` section is used to configure both the Aws objects used by Shoryuken internally, and also to set up some Shoryuken-specific config. The Shoryuken-specific keys are listed below, and you can expect any other key defined in that block to be passed on untouched to ```Aws::SQS::Client#initialize```:
|
|
131
|
-
|
|
132
|
-
- ```account_id``` is used when generating SNS ARNs
|
|
133
|
-
- ```sns_endpoint``` can be used to explicitly override the SNS endpoint
|
|
134
|
-
- ```sqs_endpoint``` can be used to explicitly override the SQS endpoint
|
|
135
|
-
- ```receive_message``` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
|
|
136
|
-
|
|
137
|
-
The ```sns_endpoint``` and ```sqs_endpoint``` Shoryuken-specific options will also fallback to the environment variables ```AWS_SNS_ENDPOINT``` and ```AWS_SQS_ENDPOINT``` respectively, if they are set.
|
|
138
|
-
|
|
139
|
-
### Configuration (producer side)
|
|
140
|
-
|
|
141
|
-
'Producer' processes need permissions to put messages into SQS. There are a few ways:
|
|
142
|
-
|
|
143
|
-
* Ensure the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` env vars are set.
|
|
144
|
-
* Create a `~/.aws/credentials` file.
|
|
145
|
-
* Set `Aws.config[:credentials]` from Ruby code (e.g. in a Rails initializer)
|
|
146
|
-
* Use the Instance Profiles feature. The IAM role of the targeted machine must have an adequate SQS Policy.
|
|
147
|
-
|
|
148
|
-
Note that storing your credentials into Amazon instances represents a security risk. Instance Profiles tends to be the best choice.
|
|
149
|
-
|
|
150
|
-
You can read about these in more detail [here](http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html).
|
|
151
|
-
|
|
152
|
-
### Rails Integration
|
|
153
|
-
|
|
154
|
-
[Check the Rails Integration Active Job documention](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job).
|
|
155
|
-
|
|
156
|
-
### Start Shoryuken
|
|
157
|
-
|
|
158
36
|
```shell
|
|
159
|
-
bundle
|
|
37
|
+
$ bundle
|
|
160
38
|
```
|
|
161
39
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
```bash
|
|
165
|
-
shoryuken --help
|
|
166
|
-
|
|
167
|
-
shoryuken [options]
|
|
168
|
-
-c, --concurrency INT Processor threads to use
|
|
169
|
-
-d, --daemon Daemonize process
|
|
170
|
-
-q, --queue QUEUE[,WEIGHT]... Queues to process with optional weights
|
|
171
|
-
-r, --require [PATH|DIR] Location of the worker
|
|
172
|
-
-C, --config PATH Path to YAML config file
|
|
173
|
-
-R, --rails Attempts to load the containing Rails project
|
|
174
|
-
-L, --logfile PATH Path to writable logfile
|
|
175
|
-
-P, --pidfile PATH Path to pidfile
|
|
176
|
-
-v, --verbose Print more verbose output
|
|
177
|
-
-V, --version Print version and exit
|
|
178
|
-
-h, --help Show help
|
|
179
|
-
...
|
|
180
|
-
```
|
|
40
|
+
## Usage
|
|
181
41
|
|
|
42
|
+
Check the [Getting Started](https://github.com/phstc/shoryuken/wiki/Getting-Started) page.
|
|
182
43
|
|
|
183
44
|
## More Information
|
|
184
45
|
|
|
185
|
-
For more information
|
|
46
|
+
For more information check the [wiki page](https://github.com/phstc/shoryuken/wiki).
|
|
186
47
|
|
|
187
48
|
## Credits
|
|
188
49
|
|
data/Rakefile
CHANGED
data/bin/cli/base.rb
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# rubocop:disable Metrics/BlockLength
|
|
2
|
+
module Shoryuken
|
|
3
|
+
module CLI
|
|
4
|
+
class Base < Thor
|
|
5
|
+
no_commands do
|
|
6
|
+
def print_table(entries)
|
|
7
|
+
column_sizes = print_columns_size(entries)
|
|
8
|
+
|
|
9
|
+
entries.map do |entry|
|
|
10
|
+
puts entry.map.with_index { |e, i| print_format_column(e, column_sizes[i]) }.join
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def print_columns_size(entries)
|
|
15
|
+
column_sizes = Hash.new(0)
|
|
16
|
+
|
|
17
|
+
entries.each do |entry|
|
|
18
|
+
entry.each_with_index do |e, i|
|
|
19
|
+
e = e.to_s
|
|
20
|
+
column_sizes[i] = e.size if column_sizes[i] < e.size
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
column_sizes
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def print_format_column(column, size)
|
|
28
|
+
size_with_padding = size + 4
|
|
29
|
+
column = column.to_s.ljust(size_with_padding)
|
|
30
|
+
column
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def fail_task(msg, quit = true)
|
|
34
|
+
say "[FAIL] #{msg}", :red
|
|
35
|
+
exit(1) if quit
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|