shoryuken 2.0.11 → 3.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +20 -0
  3. data/.rubocop.yml +8 -2
  4. data/.travis.yml +7 -5
  5. data/CHANGELOG.md +127 -10
  6. data/Gemfile +1 -0
  7. data/README.md +20 -57
  8. data/Rakefile +0 -1
  9. data/bin/cli/base.rb +42 -0
  10. data/bin/cli/sqs.rb +187 -0
  11. data/bin/shoryuken +55 -9
  12. data/examples/default_worker.rb +1 -12
  13. data/lib/shoryuken/client.rb +3 -25
  14. data/lib/shoryuken/default_worker_registry.rb +9 -5
  15. data/lib/shoryuken/environment_loader.rb +29 -67
  16. data/lib/shoryuken/fetcher.rb +22 -53
  17. data/lib/shoryuken/launcher.rb +5 -29
  18. data/lib/shoryuken/manager.rb +77 -180
  19. data/lib/shoryuken/message.rb +4 -13
  20. data/lib/shoryuken/middleware/chain.rb +1 -18
  21. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +21 -18
  22. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +26 -19
  23. data/lib/shoryuken/polling.rb +204 -0
  24. data/lib/shoryuken/processor.rb +9 -14
  25. data/lib/shoryuken/queue.rb +36 -38
  26. data/lib/shoryuken/runner.rb +143 -0
  27. data/lib/shoryuken/util.rb +3 -9
  28. data/lib/shoryuken/version.rb +1 -1
  29. data/lib/shoryuken/worker.rb +1 -1
  30. data/lib/shoryuken.rb +78 -39
  31. data/shoryuken.gemspec +6 -6
  32. data/spec/integration/launcher_spec.rb +4 -3
  33. data/spec/shoryuken/client_spec.rb +2 -43
  34. data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
  35. data/spec/shoryuken/environment_loader_spec.rb +34 -0
  36. data/spec/shoryuken/fetcher_spec.rb +18 -52
  37. data/spec/shoryuken/manager_spec.rb +56 -97
  38. data/spec/shoryuken/middleware/chain_spec.rb +0 -24
  39. data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
  40. data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +7 -3
  41. data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +56 -33
  42. data/spec/shoryuken/polling_spec.rb +239 -0
  43. data/spec/shoryuken/processor_spec.rb +7 -5
  44. data/spec/shoryuken/queue_spec.rb +110 -63
  45. data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +10 -24
  46. data/spec/shoryuken_spec.rb +13 -1
  47. data/spec/spec_helper.rb +8 -20
  48. data/test_workers/endless_interruptive_worker.rb +41 -0
  49. data/test_workers/endless_uninterruptive_worker.rb +44 -0
  50. metadata +34 -35
  51. data/.hound.yml +0 -6
  52. data/lib/shoryuken/cli.rb +0 -210
  53. data/lib/shoryuken/sns_arn.rb +0 -27
  54. data/lib/shoryuken/topic.rb +0 -17
  55. data/spec/shoryuken/sns_arn_spec.rb +0 -42
  56. data/spec/shoryuken/topic_spec.rb +0 -32
  57. data/spec/shoryuken_endpoint.yml +0 -6
  58. /data/{LICENSE.txt → LICENSE} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9859d893c7542fbdac0224f68e55a8b49f0fd984
4
- data.tar.gz: d3e5419d74c6766c95c4b8ad20333bdced3cfd57
3
+ metadata.gz: ed324ac96e13143c24ee4cd81623c80c3558f1bf
4
+ data.tar.gz: 1f88dd595425bd1c9185a1ce33ab9c8ad4945bcd
5
5
  SHA512:
6
- metadata.gz: cb5725e4def20f4c82a785db1c8a055ba59892979608ce95855888bfbd321c5310f491284a2d6ef319e7bc1b3c1f186d91e376cc6b35faef600c26d4fa453325
7
- data.tar.gz: 15274a15cd3dc7f067d4ae54a40921b90b703e09e2002ed32049ce41f6dd1261629de3b3fa842656d574455f01b13e4558536997c93d4f5ebd05880f1c80b9e2
6
+ metadata.gz: 809cbd0b7dde05a27458d4b44fa3452efb0542a9fda8e2ed06b6bcd1e2ea443b67eaca4e38aa958b6252b40db6ffbda3974afee38f3e018d1f37f6ed4d650aa8
7
+ data.tar.gz: 04b8353ecbb0d0a5458446a821e86e808c8106aa6e5805d769351a191a34cf5a018a7502edc5cdf99afd1937194b79338d31b08ed6e4b0102c757511b4338dfc
data/.codeclimate.yml ADDED
@@ -0,0 +1,20 @@
1
+ ---
2
+ engines:
3
+ reek:
4
+ enabled: true
5
+ duplication:
6
+ enabled: true
7
+ config:
8
+ languages:
9
+ - ruby
10
+ fixme:
11
+ enabled: true
12
+ rubocop:
13
+ enabled: true
14
+ config:
15
+ file: .rubocop.yml
16
+ ratings:
17
+ paths:
18
+ - "**.rb"
19
+ exclude_paths:
20
+ - spec/
data/.rubocop.yml CHANGED
@@ -1,9 +1,12 @@
1
- LineLength:
2
- Max: 120
1
+ AllCops:
2
+ TargetRubyVersion: 2.0
3
3
 
4
4
  Style/SignalException:
5
5
  Enabled: false
6
6
 
7
+ Style/DoubleNegation:
8
+ Enabled: false
9
+
7
10
  Style/SpaceAroundEqualsInParameterDefault:
8
11
  Enabled: false
9
12
 
@@ -28,6 +31,9 @@ Metrics/ClassLength:
28
31
  Metrics/ParameterLists:
29
32
  Enabled: false
30
33
 
34
+ Metrics/LineLength:
35
+ Max: 120
36
+
31
37
  Metrics/MethodLength :
32
38
  Enabled: false
33
39
 
data/.travis.yml CHANGED
@@ -1,13 +1,9 @@
1
1
  language: ruby
2
2
  rvm:
3
- # - 1.9.3
4
- # - 1.9.2
5
3
  - 2.0.0
6
4
  - 2.1.0
7
5
  - 2.2.0
8
- # - ruby-head
9
- # - jruby-19mode
10
- # - jruby-head
6
+ - 2.3.3
11
7
 
12
8
  notifications:
13
9
  email:
@@ -17,3 +13,9 @@ notifications:
17
13
  script: SPEC_ALL=true bundle exec rspec spec
18
14
  before_install:
19
15
  - gem update bundler
16
+ after_success:
17
+ - bundle exec codeclimate-test-reporter
18
+
19
+ addons:
20
+ code_climate:
21
+ repo_token: 7709fd21981bb9d2658647a66d959415a1029a83f1c199573828797944f26c52
data/CHANGELOG.md CHANGED
@@ -1,40 +1,157 @@
1
+ ## [v3.0.6] - 2017-04-11
2
+ - Fix delay option type
3
+ - [#356](https://github.com/phstc/shoryuken/pull/356)
4
+
5
+ ## [v3.0.5] - 2017-04-09
6
+ - Pause endless dispatcher to avoid CPU overload
7
+ - [#354](https://github.com/phstc/shoryuken/pull/354)
8
+
9
+ - Auto log processor errors
10
+ - [#355](https://github.com/phstc/shoryuken/pull/355)
11
+
12
+ - Add a delay as a CLI param
13
+ - [#350](https://github.com/phstc/shoryuken/pull/350)
14
+
15
+ - Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
16
+ - [#344](https://github.com/phstc/shoryuken/pull/344)
17
+ ## [v3.0.4] - 2017-03-24
18
+ - Add `sqs purge` command. See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_PurgeQueue.html
19
+ - [#344](https://github.com/phstc/shoryuken/pull/344)
20
+
21
+ - Fix "Thread exhaustion" error. This issue was most noticed when using long polling. @waynerobinson :beers: for pairing up on this.
22
+ - [#345](https://github.com/phstc/shoryuken/pull/345)
23
+
24
+ ## [v3.0.3] - 2017-03-19
25
+ - Update `sqs` CLI commands to use `get_queue_url` when appropriated
26
+ - [#341](https://github.com/phstc/shoryuken/pull/341)
27
+
28
+ ## [v3.0.2] - 2017-03-19
29
+ - Fix custom SQS client initialization
30
+ - [#335](https://github.com/phstc/shoryuken/pull/335)
31
+
32
+ ## [v3.0.1] - 2017-03-13
33
+ - Fix commands sqs mv and dump `options.delete` checker
34
+ - [#332](https://github.com/phstc/shoryuken/pull/332)
35
+
36
+ ## [v3.0.0] - 2017-03-12
37
+ - Replace Celluloid with Concurrent Ruby
38
+ - [#291](https://github.com/phstc/shoryuken/pull/291)
39
+
40
+ - Remove AWS configuration from Shoryuken. Now AWS should be configured from outside. Check [this](https://github.com/phstc/shoryuken/wiki/Configure-the-AWS-Client) for more details
41
+ - [#317](https://github.com/phstc/shoryuken/pull/317)
42
+
43
+ - Remove deprecation warnings
44
+ - [#326](https://github.com/phstc/shoryuken/pull/326)
45
+
46
+ - Allow dynamic adding queues
47
+ - [#322](https://github.com/phstc/shoryuken/pull/322)
48
+
49
+ - Support retry_intervals passed in as a lambda. Auto coerce intervals into integer
50
+ - [#329](https://github.com/phstc/shoryuken/pull/329)
51
+
52
+ - Add SQS commands `shoryuken help sqs`, such as `ls`, `mv`, `dump` and `requeue`
53
+ - [#330](https://github.com/phstc/shoryuken/pull/330)
54
+
55
+ ## [v2.1.3] - 2017-01-27
56
+ - Show a warn message when batch isn't supported
57
+ - [#302](https://github.com/phstc/shoryuken/pull/302)
58
+
59
+ - Require Celluloid ~> 17
60
+ - [#305](https://github.com/phstc/shoryuken/pull/305)
61
+
62
+ - Fix excessive logging when 0 messages found
63
+ - [#307](https://github.com/phstc/shoryuken/pull/307)
64
+
65
+ ## [v2.1.2] - 2016-12-22
66
+ - Fix loading `logfile` from shoryuken.yml
67
+ - [#296](https://github.com/phstc/shoryuken/pull/296)
68
+
69
+ - Add support for Strict priority polling (pending documentation)
70
+ - [#288](https://github.com/phstc/shoryuken/pull/288)
71
+
72
+ - Add `test_workers` for end-to-end testing supporting
73
+ - [#286](https://github.com/phstc/shoryuken/pull/286)
74
+
75
+ - Update README documenting `configure_client` and `configure_server`
76
+ - [#283](https://github.com/phstc/shoryuken/pull/283)
77
+
78
+ - Fix memory leak caused by async tracking busy threads
79
+ - [#289](https://github.com/phstc/shoryuken/pull/289)
80
+
81
+ - Refactor fetcher, polling strategy and manager
82
+ - [#284](https://github.com/phstc/shoryuken/pull/284)
83
+
84
+ ## [v2.1.1] - 2016-12-05
85
+ - Fix aws deprecation warning message
86
+ - [#279](https://github.com/phstc/shoryuken/pull/279)
87
+
88
+ ## [v2.1.0] - 2016-12-03
89
+ - Fix celluloid "running in BACKPORTED mode" warning
90
+ - [#260](https://github.com/phstc/shoryuken/pull/260)
91
+
92
+ - Allow setting the aws configuration in 'Shoryuken.configure_server'
93
+ - [#252](https://github.com/phstc/shoryuken/pull/252)
94
+
95
+ - Allow requiring a file or dir a through `-r`
96
+ - [#248](https://github.com/phstc/shoryuken/pull/248)
97
+
98
+ - Reduce info log verbosity
99
+ - [#243](https://github.com/phstc/shoryuken/pull/243)
100
+
101
+ - Fix auto extender when using ActiveJob
102
+ - [#3213](https://github.com/phstc/shoryuken/pull/213)
103
+
104
+ - Add FIFO queue support
105
+ - [#272](https://github.com/phstc/shoryuken/issues/272)
106
+
107
+ - Deprecates initialize_aws
108
+ - [#269](https://github.com/phstc/shoryuken/pull/269)
109
+
110
+ - [Other miscellaneous updates](https://github.com/phstc/shoryuken/compare/v2.0.11...v2.1.0)
111
+
112
+ ## [v2.0.11] - 2016-07-02
113
+
114
+ - Same as 2.0.10. Unfortunately 2.0.10 was removed `yanked` by mistake from RubyGems.
115
+ - [#b255bc3](https://github.com/phstc/shoryuken/commit/b255bc3)
116
+
1
117
  ## [v2.0.10] - 2016-06-09
2
118
 
3
119
  - Fix manager #225
4
- - [#226] https://github.com/phstc/shoryuken/pull/226
120
+ - [#226](https://github.com/phstc/shoryuken/pull/226)
5
121
 
6
122
  ## [v2.0.9] - 2016-06-08
7
123
 
8
124
  - Fix daemonization broken in #219
9
- - [#224] https://github.com/phstc/shoryuken/pull/224
125
+ - [#224](https://github.com/phstc/shoryuken/pull/224)
10
126
 
11
127
  ## [v2.0.8] - 2016-06-07
12
128
 
13
129
  - Fix daemonization
14
- - [#223] https://github.com/phstc/shoryuken/pull/223
130
+ - [#223](https://github.com/phstc/shoryuken/pull/223)
15
131
 
16
132
  ## [v2.0.7] - 2016-06-06
17
133
 
18
134
  - Daemonize before loading environment
19
- - [#219] https://github.com/phstc/shoryuken/pull/219
135
+ - [#219](https://github.com/phstc/shoryuken/pull/219)
20
136
 
21
137
  - Fix initialization when using rails
22
- - [#197] https://github.com/phstc/shoryuken/pull/197
138
+ - [#197](https://github.com/phstc/shoryuken/pull/197)
23
139
 
24
140
  - Improve message fetching
25
- - https://github.com/phstc/shoryuken/pull/214 and https://github.com/phstc/shoryuken/commit/f4640d97950c1783a061195855d93994725ed64a
141
+ - [#214](https://github.com/phstc/shoryuken/pull/214)
142
+ - [#f4640d9](https://github.com/phstc/shoryuken/commit/f4640d9)
26
143
 
27
144
  - Fix hard shutdown if there are some busy workers when signal received
28
- - [#215] https://github.com/phstc/shoryuken/pull/215
145
+ - [#215](https://github.com/phstc/shoryuken/pull/215)
29
146
 
30
147
  - Fix `rake console` task
31
- - [#208] https://github.com/phstc/shoryuken/pull/208
148
+ - [#208](https://github.com/phstc/shoryuken/pull/208)
32
149
 
33
150
  - Isolate `MessageVisibilityExtender` as new middleware
34
- - [#199] https://github.com/phstc/shoryuken/pull/190
151
+ - [#199](https://github.com/phstc/shoryuken/pull/190)
35
152
 
36
153
  - Fail on non-existent queues
37
- - [#196] https://github.com/phstc/shoryuken/pull/196
154
+ - [#196](https://github.com/phstc/shoryuken/pull/196)
38
155
 
39
156
  ## [v2.0.6] - 2016-04-18
40
157
 
data/Gemfile CHANGED
@@ -5,5 +5,6 @@ gemspec
5
5
 
6
6
  group :test do
7
7
  gem 'codeclimate-test-reporter', require: nil
8
+ gem 'simplecov'
8
9
  gem 'multi_xml'
9
10
  end
data/README.md CHANGED
@@ -20,18 +20,18 @@ concurrency: 25
20
20
  delay: 25
21
21
  queues:
22
22
  - [high_priority, 6]
23
- - [default, 2]
23
+ - [normal_priority, 2]
24
24
  - [low_priority, 1]
25
25
  ```
26
26
 
27
- And supposing all the queues are full of messages, the configuration above will make Shoryuken to process `high_priority` 3 times more than `default` and 6 times more than `low_priority`,
28
- splitting the work among the `concurrency: 25` available processors.
27
+ And supposing all the queues are full of messages, the configuration above will make Shoryuken to process `high_priority` 3 times more than `normal_priority` and 6 times more than `low_priority`,
28
+ splitting the work load among all available processors `concurrency: 25` .
29
29
 
30
- If `high_priority` gets empty, Shoryuken will keep using the 25 processors, but only to process `default` (2 times more than `low_priority`) and `low_priority`.
30
+ If `high_priority` gets empty, Shoryuken will keep using the 25 processors, but only to process `normal_priority` and `low_priority`.
31
31
 
32
- If `high_priority` receives a new message, Shoryuken will smoothly increase back the `high_priority` weight one by one until it reaches the weight of 6 again, which is the maximum configured for `high_priority`.
32
+ If `high_priority` receives a new message, Shoryuken will smoothly increase back its weight one by one until it reaches the weight of 6 again.
33
33
 
34
- If all queues get empty, all processors will be changed to the waiting state and the queues will be checked every `delay: 25`. If any queue receives a new message, Shoryuken will start processing again. [Check the delay option documentation for more information](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#delay).
34
+ [If a queue gets empty, Shoryuken will pause checking it for `delay: 25`](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#delay).
35
35
 
36
36
 
37
37
  ### Fetch in batches
@@ -40,7 +40,7 @@ To be even more performant and cost effective, Shoryuken fetches SQS messages in
40
40
 
41
41
  ## Requirements
42
42
 
43
- Ruby 2.0 or greater. Ruby 1.9 is no longer supported.
43
+ Ruby 2.0 or greater.
44
44
 
45
45
  ## Installation
46
46
 
@@ -105,49 +105,22 @@ end
105
105
 
106
106
  [Check the Middleware documentation](https://github.com/phstc/shoryuken/wiki/Middleware).
107
107
 
108
- ### Configuration (worker side)
108
+ ### Shoryuken Configuration
109
109
 
110
110
  Sample configuration file `shoryuken.yml`.
111
111
 
112
112
  ```yaml
113
- aws:
114
- access_key_id: ... # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
115
- secret_access_key: ... # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
116
- region: us-east-1 # or <%= ENV['AWS_REGION'] %>
117
- receive_message: # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
118
- # wait_time_seconds: N # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
119
- attribute_names:
120
- - ApproximateReceiveCount
121
- - SentTimestamp
122
113
  concurrency: 25 # The number of allocated threads to process messages. Default 25
123
114
  delay: 25 # The delay in seconds to pause a queue when it's empty. Default 0
124
115
  queues:
125
116
  - [high_priority, 6]
126
- - [default, 2]
117
+ - [normal_priority, 2]
127
118
  - [low_priority, 1]
128
119
  ```
129
120
 
130
- The ```aws``` section is used to configure both the Aws objects used by Shoryuken internally, and also to set up some Shoryuken-specific config. The Shoryuken-specific keys are listed below, and you can expect any other key defined in that block to be passed on untouched to ```Aws::SQS::Client#initialize```:
121
+ #### AWS Configuration
131
122
 
132
- - ```account_id``` is used when generating SNS ARNs
133
- - ```sns_endpoint``` can be used to explicitly override the SNS endpoint
134
- - ```sqs_endpoint``` can be used to explicitly override the SQS endpoint
135
- - ```receive_message``` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
136
-
137
- The ```sns_endpoint``` and ```sqs_endpoint``` Shoryuken-specific options will also fallback to the environment variables ```AWS_SNS_ENDPOINT``` and ```AWS_SQS_ENDPOINT``` respectively, if they are set.
138
-
139
- ### Configuration (producer side)
140
-
141
- 'Producer' processes need permissions to put messages into SQS. There are a few ways:
142
-
143
- * Ensure the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` env vars are set.
144
- * Create a `~/.aws/credentials` file.
145
- * Set `Aws.config[:credentials]` from Ruby code (e.g. in a Rails initializer)
146
- * Use the Instance Profiles feature. The IAM role of the targeted machine must have an adequate SQS Policy.
147
-
148
- Note that storing your credentials into Amazon instances represents a security risk. Instance Profiles tends to be the best choice.
149
-
150
- You can read about these in more detail [here](http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html).
123
+ [Check the Configure AWS Client documentation](https://github.com/phstc/shoryuken/wiki/Configure-the-AWS-Client)
151
124
 
152
125
  ### Rails Integration
153
126
 
@@ -159,26 +132,16 @@ You can read about these in more detail [here](http://docs.aws.amazon.com/sdkfor
159
132
  bundle exec shoryuken -r worker.rb -C shoryuken.yml
160
133
  ```
161
134
 
162
- Other options:
163
-
164
- ```bash
165
- shoryuken --help
166
-
167
- shoryuken [options]
168
- -c, --concurrency INT Processor threads to use
169
- -d, --daemon Daemonize process
170
- -q, --queue QUEUE[,WEIGHT]... Queues to process with optional weights
171
- -r, --require [PATH|DIR] Location of the worker
172
- -C, --config PATH Path to YAML config file
173
- -R, --rails Attempts to load the containing Rails project
174
- -L, --logfile PATH Path to writable logfile
175
- -P, --pidfile PATH Path to pidfile
176
- -v, --verbose Print more verbose output
177
- -V, --version Print version and exit
178
- -h, --help Show help
179
- ...
180
- ```
135
+ For other options check `bundle exec shoryuken help start`
136
+
137
+ #### SQS commands
138
+
139
+ Check also some available SQS commands `bundle exec shoryuken help sqs`, such as:
181
140
 
141
+ - `ls` list queues
142
+ - `mv` move messages from one queue to another
143
+ - `dump` dump messages from a queue into a JSON lines file
144
+ - `requeue` requeue messages from a dump file
182
145
 
183
146
  ## More Information
184
147
 
data/Rakefile CHANGED
@@ -10,7 +10,6 @@ end
10
10
  desc 'Open Shoryuken pry console'
11
11
  task :console do
12
12
  require 'pry'
13
- require 'celluloid'
14
13
  require 'shoryuken'
15
14
 
16
15
  config_file = File.join File.expand_path('..', __FILE__), 'shoryuken.yml'
data/bin/cli/base.rb ADDED
@@ -0,0 +1,42 @@
1
+ # rubocop:disable Metrics/BlockLength
2
+ module Shoryuken
3
+ module CLI
4
+ class Base < Thor
5
+ no_commands do
6
+ def print_table(entries)
7
+ column_sizes = print_columns_size(entries)
8
+
9
+ entries.map do |entry|
10
+ puts entry.map.with_index { |e, i| print_format_column(e, column_sizes[i]) }.join
11
+ end
12
+ end
13
+
14
+ def print_columns_size(entries)
15
+ column_sizes = Hash.new(0)
16
+
17
+ entries.each do |entry|
18
+ entry.each_with_index do |e, i|
19
+ e = e.to_s
20
+ column_sizes[i] = e.size if column_sizes[i] < e.size
21
+ end
22
+ end
23
+
24
+ column_sizes
25
+ end
26
+
27
+ def print_format_column(column, size)
28
+ size = 40 if size > 40
29
+ size_with_padding = size + 4
30
+ column = column.to_s.ljust(size_with_padding)
31
+ column = "#{column[0...size - 2]}.." if column.size > size_with_padding
32
+ column
33
+ end
34
+
35
+ def fail_task(msg, quit = true)
36
+ say "[FAIL] #{msg}", :red
37
+ exit(1) if quit
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
data/bin/cli/sqs.rb ADDED
@@ -0,0 +1,187 @@
1
+ require 'date'
2
+
3
+ # rubocop:disable Metrics/AbcSize, Metrics/BlockLength
4
+ module Shoryuken
5
+ module CLI
6
+ class SQS < Base
7
+ namespace :sqs
8
+
9
+ no_commands do
10
+ def normalize_dump_message(message)
11
+ message[:id] = message.delete(:message_id)
12
+ message[:message_body] = message.delete(:body)
13
+ message.delete(:receipt_handle)
14
+ message.delete(:md5_of_body)
15
+ message.delete(:md5_of_message_attributes)
16
+ message
17
+ end
18
+
19
+ def sqs
20
+ @_sqs ||= Aws::SQS::Client.new
21
+ end
22
+
23
+ def find_queue_url(queue_name)
24
+ sqs.get_queue_url(queue_name: queue_name).queue_url
25
+ rescue Aws::SQS::Errors::NonExistentQueue
26
+ fail_task "The specified queue #{queue_name} does not exist"
27
+ end
28
+
29
+ def batch_delete(url, messages)
30
+ messages.to_a.flatten.each_slice(10) do |batch|
31
+ sqs.delete_message_batch(
32
+ queue_url: url,
33
+ entries: batch.map { |message| { id: message.message_id, receipt_handle: message.receipt_handle } }
34
+ ).failed.any? do |failure|
35
+ say "Could not delete #{failure.id}, code: #{failure.code}", :yellow
36
+ end
37
+ end
38
+ end
39
+
40
+ def batch_send(url, messages)
41
+ messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(10) do |batch|
42
+ sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
43
+ say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
44
+ end
45
+ end
46
+ end
47
+
48
+ def find_all(url, limit, &block)
49
+ count = 0
50
+ batch_size = limit > 10 ? 10 : limit
51
+
52
+ loop do
53
+ n = limit - count
54
+ batch_size = n if n < batch_size
55
+
56
+ messages = sqs.receive_message(
57
+ queue_url: url,
58
+ max_number_of_messages: batch_size,
59
+ message_attribute_names: ['All']
60
+ ).messages
61
+
62
+ messages.each { |m| yield m }
63
+
64
+ count += messages.size
65
+
66
+ break if count >= limit
67
+ break if messages.empty?
68
+ end
69
+
70
+ count
71
+ end
72
+
73
+ def list_and_print_queues(urls)
74
+ attrs = %w(QueueArn ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible LastModifiedTimestamp)
75
+
76
+ entries = urls.map { |u| sqs.get_queue_attributes(queue_url: u, attribute_names: attrs).attributes }.map do |q|
77
+ [
78
+ q['QueueArn'].split(':').last,
79
+ q['ApproximateNumberOfMessages'],
80
+ q['ApproximateNumberOfMessagesNotVisible'],
81
+ Time.at(q['LastModifiedTimestamp'].to_i)
82
+ ]
83
+ end
84
+
85
+ entries.unshift(['Queue', 'Messages Available', 'Messages Inflight', 'Last Modified'])
86
+
87
+ print_table(entries)
88
+ end
89
+
90
+ def dump_file(path, queue_name)
91
+ File.join(path, "#{queue_name}-#{Date.today}.jsonl")
92
+ end
93
+ end
94
+
95
+ desc 'ls [QUEUE-NAME-PREFIX]', 'Lists queues'
96
+ method_option :watch, aliases: '-w', type: :boolean, desc: 'watch queues'
97
+ method_option :watch_interval, type: :numeric, default: 10, desc: 'watch interval'
98
+ def ls(queue_name_prefix = '')
99
+ trap('SIGINT', 'EXIT') # expect ctrl-c from loop
100
+
101
+ urls = sqs.list_queues(queue_name_prefix: queue_name_prefix).queue_urls
102
+
103
+ loop do
104
+ list_and_print_queues(urls)
105
+
106
+ break unless options[:watch]
107
+
108
+ sleep options[:watch_interval]
109
+ puts
110
+ end
111
+ end
112
+
113
+ desc 'dump QUEUE-NAME', 'Dumps messages from a queue into a JSON lines file'
114
+ method_option :number, aliases: '-n', type: :numeric, default: Float::INFINITY, desc: 'number of messages to dump'
115
+ method_option :path, aliases: '-p', type: :string, default: './', desc: 'path to save the dump file'
116
+ method_option :delete, aliases: '-d', type: :boolean, default: true, desc: 'delete from the queue'
117
+ def dump(queue_name)
118
+ path = dump_file(options[:path], queue_name)
119
+
120
+ fail_task "File #{path} already exists" if File.exist?(path)
121
+
122
+ url = find_queue_url(queue_name)
123
+
124
+ messages = []
125
+
126
+ file = nil
127
+
128
+ count = find_all(url, options[:number]) do |m|
129
+ file ||= File.open(path, 'w')
130
+
131
+ file.puts(JSON.dump(m.to_h))
132
+
133
+ messages << m if options[:delete]
134
+ end
135
+
136
+ batch_delete(url, messages) if options[:delete]
137
+
138
+ if count.zero?
139
+ say "Queue #{queue_name} is empty", :yellow
140
+ else
141
+ say "Dump saved in #{path} with #{count} messages", :green
142
+ end
143
+ ensure
144
+ file.close if file
145
+ end
146
+
147
+ desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
148
+ def requeue(queue_name, path)
149
+ fail_task "Path #{path} not found" unless File.exist?(path)
150
+
151
+ messages = File.readlines(path).map { |line| JSON.parse(line, symbolize_names: true) }
152
+
153
+ batch_send(find_queue_url(queue_name), messages)
154
+
155
+ say "Requeued #{messages.size} messages from #{path} to #{queue_name}", :green
156
+ end
157
+
158
+ desc 'mv QUEUE-NAME-SOURCE QUEUE-NAME-TARGET', 'Moves messages from one queue (source) to another (target)'
159
+ method_option :number, aliases: '-n', type: :numeric, default: Float::INFINITY, desc: 'number of messages to move'
160
+ method_option :delete, aliases: '-d', type: :boolean, default: true, desc: 'delete from the queue'
161
+ def mv(queue_name_source, queue_name_target)
162
+ url_source = find_queue_url(queue_name_source)
163
+ messages = []
164
+
165
+ count = find_all(url_source, options[:number]) do |m|
166
+ messages << m
167
+ end
168
+
169
+ batch_send(find_queue_url(queue_name_target), messages.map(&:to_h))
170
+ batch_delete(url_source, messages) if options[:delete]
171
+
172
+ if count.zero?
173
+ say "Queue #{queue_name_source} is empty", :yellow
174
+ else
175
+ say "Moved #{count} messages from #{queue_name_source} to #{queue_name_target}", :green
176
+ end
177
+ end
178
+
179
+ desc 'purge QUEUE-NAME', 'Deletes the messages in a queue'
180
+ def purge(queue_name)
181
+ sqs.purge_queue(queue_url: find_queue_url(queue_name))
182
+
183
+ say "Purge request sent for #{queue_name}. The message deletion process takes up to 60 seconds", :yellow
184
+ end
185
+ end
186
+ end
187
+ end