shoryuken 5.0.5 → 6.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/.devcontainer/Dockerfile +17 -0
  3. data/.devcontainer/base.Dockerfile +43 -0
  4. data/.devcontainer/devcontainer.json +35 -0
  5. data/.github/dependabot.yml +6 -0
  6. data/.github/workflows/specs.yml +65 -0
  7. data/.github/workflows/stale.yml +20 -0
  8. data/.gitignore +1 -1
  9. data/.reek.yml +5 -0
  10. data/.rubocop.yml +1 -1
  11. data/Appraisals +42 -0
  12. data/CHANGELOG.md +114 -0
  13. data/Gemfile +8 -3
  14. data/README.md +41 -1
  15. data/Rakefile +15 -1
  16. data/bin/cli/sqs.rb +51 -6
  17. data/gemfiles/.gitignore +1 -0
  18. data/gemfiles/aws_sdk_core_2.gemfile +21 -0
  19. data/gemfiles/rails_4_2.gemfile +20 -0
  20. data/gemfiles/rails_5_2.gemfile +21 -0
  21. data/gemfiles/rails_6_0.gemfile +21 -0
  22. data/gemfiles/rails_6_1.gemfile +21 -0
  23. data/gemfiles/rails_7_0.gemfile +22 -0
  24. data/lib/shoryuken/default_exception_handler.rb +10 -0
  25. data/lib/shoryuken/environment_loader.rb +22 -4
  26. data/lib/shoryuken/extensions/active_job_adapter.rb +30 -20
  27. data/lib/shoryuken/extensions/active_job_extensions.rb +38 -0
  28. data/lib/shoryuken/launcher.rb +26 -3
  29. data/lib/shoryuken/logging.rb +2 -2
  30. data/lib/shoryuken/manager.rb +50 -14
  31. data/lib/shoryuken/message.rb +11 -28
  32. data/lib/shoryuken/options.rb +6 -3
  33. data/lib/shoryuken/polling/base.rb +2 -0
  34. data/lib/shoryuken/polling/strict_priority.rb +8 -0
  35. data/lib/shoryuken/polling/weighted_round_robin.rb +9 -0
  36. data/lib/shoryuken/processor.rb +1 -2
  37. data/lib/shoryuken/queue.rb +5 -3
  38. data/lib/shoryuken/runner.rb +4 -3
  39. data/lib/shoryuken/version.rb +1 -1
  40. data/lib/shoryuken.rb +8 -0
  41. data/shoryuken.gemspec +1 -2
  42. data/spec/integration/launcher_spec.rb +29 -2
  43. data/spec/shared_examples_for_active_job.rb +230 -11
  44. data/spec/shoryuken/default_exception_handler_spec.rb +71 -0
  45. data/spec/shoryuken/environment_loader_spec.rb +62 -9
  46. data/spec/shoryuken/extensions/active_job_adapter_spec.rb +1 -1
  47. data/spec/shoryuken/extensions/active_job_base_spec.rb +84 -0
  48. data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +4 -0
  49. data/spec/shoryuken/extensions/active_job_wrapper_spec.rb +20 -0
  50. data/spec/shoryuken/fetcher_spec.rb +12 -12
  51. data/spec/shoryuken/launcher_spec.rb +105 -0
  52. data/spec/shoryuken/manager_spec.rb +61 -1
  53. data/spec/shoryuken/polling/strict_priority_spec.rb +10 -0
  54. data/spec/shoryuken/polling/weighted_round_robin_spec.rb +35 -0
  55. data/spec/shoryuken/queue_spec.rb +10 -5
  56. data/spec/shoryuken/worker/default_executor_spec.rb +48 -48
  57. data/spec/shoryuken_spec.rb +9 -0
  58. data/spec/spec_helper.rb +7 -9
  59. metadata +32 -24
  60. data/.travis.yml +0 -30
  61. data/Gemfile.aws-sdk-core-v2 +0 -13
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c2a0f01f40da91819e9e8e0b127a2504fccdd927365866f7971a7df207cea287
4
- data.tar.gz: 3da85303fc49af5b0ad06888d667bde14ef12b79c595c98cdc3232d085d08350
3
+ metadata.gz: feeefb6a4aad595df6286c02928284e9403b0f0a81ba188f3fe399138dd8e32c
4
+ data.tar.gz: a6dde4ed4a888f005580202bfd416fe2a41610fbc3c666efb31210cc2e831487
5
5
  SHA512:
6
- metadata.gz: 8a94cf02d3c7c25bf8282097449b83f1f4f3b4816b0e7be396048585ccc5667e9dcbdfe839f593c0ed04b2304fc1c0ca2d5d9564c86b854de63de6dafddc5de9
7
- data.tar.gz: 20844e414f26d4a6a4e9d3db14bd4ea89bad4550486b91a1a1b8ba6eff2145e89fdf63007e356c707e54dbe8e0d1bb608844ec81db02516e84c4971633fd621d
6
+ metadata.gz: 39df6a6ff9f262e7e9337487d484433c2182b8c71eb6401bdf7e0798505993a8b1a80af00f034eccf3718fafc0ca616c83cf8c5937eebbf73333ea52578077aa
7
+ data.tar.gz: 0bec3c731f75b848a6b7ddcc8eaba03331f71e2eb98687c9955ff9fc9b83196044771d182d07bb470d4a7b16070c7d06fe988561955942c740dd0eb89b4d4356
@@ -0,0 +1,17 @@
1
+ # [Choice] Ruby version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.1, 3.0, 2, 2.7, 2.6, 3-bullseye, 3.1-bullseye, 3.0-bullseye, 2-bullseye, 2.7-bullseye, 2.6-bullseye, 3-buster, 3.1-buster, 3.0-buster, 2-buster, 2.7-buster, 2.6-buster
2
+ ARG VARIANT=2-bullseye
3
+ FROM mcr.microsoft.com/vscode/devcontainers/ruby:0-${VARIANT}
4
+
5
+ # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
6
+ ARG NODE_VERSION="none"
7
+ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
8
+
9
+ # [Optional] Uncomment this section to install additional OS packages.
10
+ # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
11
+ # && apt-get -y install --no-install-recommends <your-package-list-here>
12
+
13
+ # [Optional] Uncomment this line to install additional gems.
14
+ # RUN gem install <your-gem-names-here>
15
+
16
+ # [Optional] Uncomment this line to install global node packages.
17
+ # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
@@ -0,0 +1,43 @@
1
+ # [Choice] Ruby version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.1, 3.0, 2, 2.7, 2.6, 3-bullseye, 3.1-bullseye, 3.0-bullseye, 2-bullseye, 2.7-bullseye, 2.6-bullseye, 3-buster, 3.1-buster, 3.0-buster, 2-buster, 2.7-buster, 2.6-buster
2
+ ARG VARIANT=2-bullseye
3
+ FROM ruby:${VARIANT}
4
+
5
+ # Copy library scripts to execute
6
+ COPY library-scripts/*.sh library-scripts/*.env /tmp/library-scripts/
7
+
8
+ # [Option] Install zsh
9
+ ARG INSTALL_ZSH="true"
10
+ # [Option] Upgrade OS packages to their latest versions
11
+ ARG UPGRADE_PACKAGES="true"
12
+ # Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies.
13
+ ARG USERNAME=vscode
14
+ ARG USER_UID=1000
15
+ ARG USER_GID=$USER_UID
16
+ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
17
+ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
18
+ && apt-get purge -y imagemagick imagemagick-6-common \
19
+ # Install common packages, non-root user, rvm, core build tools
20
+ && bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \
21
+ && bash /tmp/library-scripts/ruby-debian.sh "none" "${USERNAME}" "true" "true" \
22
+ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
23
+
24
+ # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
25
+ ARG NODE_VERSION="none"
26
+ ENV NVM_DIR=/usr/local/share/nvm
27
+ ENV NVM_SYMLINK_CURRENT=true \
28
+ PATH=${NVM_DIR}/current/bin:${PATH}
29
+ RUN bash /tmp/library-scripts/node-debian.sh "${NVM_DIR}" "${NODE_VERSION}" "${USERNAME}" \
30
+ && apt-get clean -y && rm -rf /var/lib/apt/lists/*
31
+
32
+ # Remove library scripts for final image
33
+ RUN rm -rf /tmp/library-scripts
34
+
35
+ # [Optional] Uncomment this section to install additional OS packages.
36
+ # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
37
+ # && apt-get -y install --no-install-recommends <your-package-list-here>
38
+
39
+ # [Optional] Uncomment this line to install additional gems.
40
+ # RUN gem install <your-gem-names-here>
41
+
42
+ # [Optional] Uncomment this line to install global node packages.
43
+ # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
@@ -0,0 +1,35 @@
1
+ // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
2
+ // https://github.com/microsoft/vscode-dev-containers/tree/v0.224.3/containers/ruby
3
+ {
4
+ "name": "Ruby",
5
+ "build": {
6
+ "dockerfile": "Dockerfile",
7
+ "args": {
8
+ // Update 'VARIANT' to pick a Ruby version: 3, 3.1, 3.0, 2, 2.7, 2.6
9
+ // Append -bullseye or -buster to pin to an OS version.
10
+ // Use -bullseye variants on local on arm64/Apple Silicon.
11
+ "VARIANT": "3-bullseye",
12
+ // Options
13
+ "NODE_VERSION": "none"
14
+ }
15
+ },
16
+
17
+ // Set *default* container specific settings.json values on container create.
18
+ "settings": {},
19
+
20
+ // Add the IDs of extensions you want installed when the container is created.
21
+ "extensions": ["rebornix.Ruby"],
22
+
23
+ // Use 'forwardPorts' to make a list of ports inside the container available locally.
24
+ // "forwardPorts": [],
25
+
26
+ // Use 'postCreateCommand' to run commands after the container is created.
27
+ // "postCreateCommand": "ruby --version",
28
+
29
+ // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
30
+ "remoteUser": "vscode",
31
+ "features": {
32
+ "github-cli": "latest",
33
+ "aws-cli": "latest"
34
+ }
35
+ }
@@ -0,0 +1,6 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "github-actions"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "weekly"
@@ -0,0 +1,65 @@
1
+ name: Specs
2
+
3
+ on:
4
+ - push
5
+ - pull_request
6
+
7
+ jobs:
8
+ all_specs:
9
+ name: All Specs
10
+ strategy:
11
+ matrix:
12
+ ruby: ['2.4', '2.5', '2.6', '2.7', '3.0', '3.1', '3.2']
13
+ gemfile: ['Gemfile', 'gemfiles/aws_sdk_core_2.gemfile']
14
+ runs-on: ubuntu-20.04
15
+ services:
16
+ moto_sqs:
17
+ image: quay.io/cjlarose/moto-sqs-server:1.1.0
18
+ ports:
19
+ - 5000:5000
20
+ env:
21
+ BUNDLE_GEMFILE: ${{ matrix.gemfile }}
22
+ steps:
23
+ - name: Checkout code
24
+ uses: actions/checkout@v2
25
+ - uses: ruby/setup-ruby@v1
26
+ with:
27
+ ruby-version: ${{ matrix.ruby }}
28
+ bundler-cache: true
29
+ - name: Run specs
30
+ run: bundle exec rake spec
31
+ - name: Run integration specs
32
+ run: bundle exec rake spec:integration
33
+ rails_specs:
34
+ name: Rails Specs
35
+ strategy:
36
+ matrix:
37
+ rails: ['4.2', '5.2', '6.0', '6.1', '7.0']
38
+ include:
39
+ - rails: '4.2'
40
+ ruby: '2.2'
41
+ gemfile: gemfiles/rails_4_2.gemfile
42
+ - rails: '5.2'
43
+ ruby: '2.5'
44
+ gemfile: gemfiles/rails_5_2.gemfile
45
+ - rails: '6.0'
46
+ ruby: '2.6'
47
+ gemfile: gemfiles/rails_6_0.gemfile
48
+ - rails: '6.1'
49
+ ruby: '3.0'
50
+ gemfile: gemfiles/rails_6_1.gemfile
51
+ - rails: '7.0'
52
+ ruby: '3.1'
53
+ gemfile: gemfiles/rails_7_0.gemfile
54
+ runs-on: ubuntu-20.04
55
+ env:
56
+ BUNDLE_GEMFILE: ${{ matrix.gemfile }}
57
+ steps:
58
+ - name: Checkout code
59
+ uses: actions/checkout@v2
60
+ - uses: ruby/setup-ruby@v1
61
+ with:
62
+ ruby-version: ${{ matrix.ruby }}
63
+ bundler-cache: true
64
+ - name: Run Rails specs
65
+ run: bundle exec rake spec:rails
@@ -0,0 +1,20 @@
1
+ name: "Close stale issues and PRs"
2
+ on:
3
+ schedule:
4
+ - cron: "30 1 * * *" # At 01:30 - https://crontab.guru/#30_1_*_*_*
5
+ workflow_dispatch: {}
6
+ jobs:
7
+ stale:
8
+ runs-on: ubuntu-latest
9
+ permissions:
10
+ issues: write
11
+ pull-requests: write
12
+ steps:
13
+ - uses: actions/stale@v8
14
+ with:
15
+ stale-issue-message: This issue is now marked as stale because it hasn't seen activity for a while. Add a comment or it will be closed soon.
16
+ stale-pr-message: This PR is now marked as stale because it hasn't seen activity for a while. Add a comment or it will be closed soon.
17
+ close-issue-message: This issue was closed because it hasn't seen activity for a while.
18
+ close-pr-message: This PR was closed because it hasn't seen activity for a while.
19
+ days-before-stale: 60
20
+ days-before-close: 7
data/.gitignore CHANGED
@@ -25,4 +25,4 @@ shoryuken.yml
25
25
  *.log
26
26
  .env
27
27
  rubocop.html
28
- .byebug_history
28
+ .byebug_history
data/.reek.yml ADDED
@@ -0,0 +1,5 @@
1
+ ---
2
+ detectors:
3
+
4
+ UtilityFunction:
5
+ public_methods_only: true
data/.rubocop.yml CHANGED
@@ -23,7 +23,7 @@ Metrics/AbcSize:
23
23
  # because codeclimate already give that for us with more details
24
24
  Enabled: false
25
25
 
26
- Metrics/LineLength:
26
+ Layout/LineLength:
27
27
  Max: 125
28
28
 
29
29
  Style/Alias:
data/Appraisals ADDED
@@ -0,0 +1,42 @@
1
+ appraise 'aws_sdk_core_2' do
2
+ group :test do
3
+ gem 'aws-sdk-core', '~> 2'
4
+ gem 'webrick' # required for Ruby 3
5
+ remove_gem 'aws-sdk-sqs'
6
+ end
7
+ end
8
+
9
+ appraise 'rails_4_2' do
10
+ group :test do
11
+ gem 'activejob', '~> 4.2'
12
+ end
13
+
14
+ group :development do
15
+ gem 'appraisal', '~> 2.2'
16
+ remove_gem 'pry-byebug'
17
+ end
18
+ end
19
+
20
+ appraise 'rails_5_2' do
21
+ group :test do
22
+ gem 'activejob', '~> 5.2'
23
+ end
24
+ end
25
+
26
+ appraise 'rails_6_0' do
27
+ group :test do
28
+ gem 'activejob', '~> 6.0'
29
+ end
30
+ end
31
+
32
+ appraise 'rails_6_1' do
33
+ group :test do
34
+ gem 'activejob', '~> 6.1'
35
+ end
36
+ end
37
+
38
+ appraise 'rails_7_0' do
39
+ group :test do
40
+ gem 'activejob', '~> 7.0'
41
+ end
42
+ end
data/CHANGELOG.md CHANGED
@@ -1,3 +1,117 @@
1
+ ## [v6.1.1] - 2023-11-27
2
+
3
+ - Fix SQS API Changes causing nil returns instead of empty arrays
4
+ - [#754](https://github.com/ruby-shoryuken/shoryuken/pull/754)
5
+ - [#753](https://github.com/ruby-shoryuken/shoryuken/pull/753)
6
+
7
+ ## [v6.1.0] - 2023-11-01
8
+
9
+ - Add GitHub Codespaces
10
+ - [#698](https://github.com/ruby-shoryuken/shoryuken/pull/698)
11
+
12
+ - Fix spec for ruby 3.0
13
+ - [#727](https://github.com/ruby-shoryuken/shoryuken/pull/727)
14
+
15
+ - Upgrade test matrix. Add Ruby 3.1, Ruby 3.2 and Rails 7
16
+ - [#739](https://github.com/ruby-shoryuken/shoryuken/pull/739)
17
+
18
+ - Fire stopped event after executor is stopped
19
+ - [#741](https://github.com/ruby-shoryuken/shoryuken/pull/741)
20
+
21
+ - Allow setup custom exception handlers for failing jobs
22
+ - [#742](https://github.com/ruby-shoryuken/shoryuken/pull/742)
23
+
24
+ - Configure dependabot to update GH Actions
25
+ - [#745](https://github.com/ruby-shoryuken/shoryuken/pull/745)
26
+
27
+ - Stop the dispatching of new messages when a SIGTERM signal has been received
28
+ - [#750](https://github.com/ruby-shoryuken/shoryuken/pull/750)
29
+
30
+ ## [v6.0.0] - 2022-02-18
31
+
32
+ - Breaking changes: Initialize Rails before parsing config file
33
+ - [#686](https://github.com/ruby-shoryuken/shoryuken/pull/686)
34
+ - Previously, Shoryuken read its configuration from an optional YAML file, then allowed CLI arguments to override those, then initialized the Rails application (provided that `--rails` or `-R` was specified). This behavior meant that the config file did not have access to things like environment variables that were initialized by Rails (such as when using `dotenv`). With this change, Rails is initialized much earlier in the process. After Rails is initialized, the YAML configuration file is interpreted, and CLI arguments are finally interpreted last. Most applications will not need to undergo changes in order to upgrade, but the new load order could technically result in different behavior depending on the application's YAML configuration file or Rails initializers.
35
+
36
+ ## [v5.3.2] - 2022-01-19
37
+
38
+ - (Bugfix) Preserve queue weights when unpausing queues
39
+ - [#687](https://github.com/ruby-shoryuken/shoryuken/pull/687)
40
+
41
+ - Improve error message on startup when shoryuken has insufficient permissions to access a queue
42
+ - [#691](https://github.com/ruby-shoryuken/shoryuken/pull/691)
43
+
44
+ ## [v5.3.1] - 2022-01-07
45
+
46
+ - (Bugfix) Fix issue where, when using the TSTP or USR1 signals for soft shutdowns, it was possible for shoryuken to terminate without first attempting to handle all messages it fetched from SQS
47
+ - [#676](https://github.com/ruby-shoryuken/shoryuken/pull/676)
48
+
49
+ ## [v5.3.0] - 2021-10-31
50
+
51
+ - (Refactor) Use Forwardable within Message to avoid method boilerplate
52
+ - [#681](https://github.com/ruby-shoryuken/shoryuken/pull/681)
53
+
54
+ - Add basic health check API
55
+ - [#679](https://github.com/ruby-shoryuken/shoryuken/pull/679)
56
+
57
+ ## [v5.2.3] - 2021-07-29
58
+
59
+ - Fire new `:utilization_update` event any time a worker pool's utilization changes
60
+ - [#673](https://github.com/ruby-shoryuken/shoryuken/pull/673)
61
+
62
+ ## [v5.2.2] - 2021-06-22
63
+
64
+ - When using ActiveJob queue name prefixing, avoid applying prefix to queues configured with a URL or ARN
65
+ - [#667](https://github.com/ruby-shoryuken/shoryuken/pull/667)
66
+
67
+ ## [v5.2.1] - 2021-04-06
68
+
69
+ - Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands
70
+ - [#666](https://github.com/ruby-shoryuken/shoryuken/pull/666)
71
+
72
+ - Fix bug in `shoryuken sqs requeue` and `shoryuken sqs mv` where those commands would exceed the SQS `SendMessageBatch` maximum payload size
73
+ - [#663](https://github.com/ruby-shoryuken/shoryuken/issues/663)
74
+ - [#664](https://github.com/ruby-shoryuken/shoryuken/pull/664)
75
+
76
+ - Remove test stub for `Concurrent.global_io_executor`
77
+ - [#662](https://github.com/ruby-shoryuken/shoryuken/pull/662)
78
+
79
+ - Run integration tests on CI
80
+ - [#660](https://github.com/ruby-shoryuken/shoryuken/pull/660)
81
+
82
+ ## [v5.2.0] - 2021-02-26
83
+
84
+ - Set `executions` correctly for ActiveJob jobs
85
+ - [#657](https://github.com/phstc/shoryuken/pull/657)
86
+
87
+ ## [v5.1.1] - 2021-02-10
88
+
89
+ - Fix regression in Ruby 3.0 introduced in Shoryuken 5.1.0, where enqueueing jobs with ActiveJob to workers that used keyword arguments would fail
90
+ - [#654](https://github.com/phstc/shoryuken/pull/654)
91
+
92
+ ## [v5.1.0] - 2021-02-06
93
+
94
+ - Add support for specifying SQS SendMessage parameters with ActiveJob `.set`
95
+ - [#635](https://github.com/phstc/shoryuken/pull/635)
96
+ - [#648](https://github.com/phstc/shoryuken/pull/648)
97
+ - [#651](https://github.com/phstc/shoryuken/pull/651)
98
+
99
+ - Unpause FIFO queues on worker completion
100
+ - [#644](https://github.com/phstc/shoryuken/pull/644)
101
+
102
+ - Add multiple versions of Rails to test matrix
103
+ - [#647](https://github.com/phstc/shoryuken/pull/647)
104
+
105
+ - Migrate from Travis CI to Github Actions
106
+ - [#649](https://github.com/phstc/shoryuken/pull/649)
107
+ - [#650](https://github.com/phstc/shoryuken/pull/650)
108
+ - [#652](https://github.com/phstc/shoryuken/pull/652)
109
+
110
+ ## [v5.0.6] - 2020-12-30
111
+
112
+ - Load ShoryukenConcurrentSendAdapter when loading Rails
113
+ - [#642](https://github.com/phstc/shoryuken/pull/642)
114
+
1
115
  ## [v5.0.5] - 2020-06-07
2
116
 
3
117
  - Add ability to configure queue by ARN
data/Gemfile CHANGED
@@ -4,9 +4,12 @@ source 'https://rubygems.org'
4
4
  gemspec
5
5
 
6
6
  group :test do
7
- gem 'activejob', '~> 4'
7
+ gem 'activejob'
8
8
  gem 'aws-sdk-core', '~> 3'
9
- gem 'aws-sdk-sqs'
9
+ # Pin to 1.65.0 because of below issues:
10
+ # - https://github.com/ruby-shoryuken/shoryuken/pull/753#issuecomment-1822720647
11
+ # - https://github.com/getmoto/moto/issues/7054
12
+ gem 'aws-sdk-sqs', '1.65.0'
10
13
  gem 'codeclimate-test-reporter', require: nil
11
14
  gem 'httparty'
12
15
  gem 'multi_xml'
@@ -14,5 +17,7 @@ group :test do
14
17
  end
15
18
 
16
19
  group :development do
17
- gem 'rubocop'
20
+ gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
21
+ gem 'pry-byebug'
22
+ gem 'rubocop', '<= 1.12'
18
23
  end
data/README.md CHANGED
@@ -1,10 +1,12 @@
1
+ **I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? [Join our Slack](https://join.slack.com/t/shoryuken/shared_invite/zt-19xjq3iqc-KmoJ6eU6~qvZNqcLzIrjww)**
2
+
1
3
  # Shoryuken
2
4
 
3
5
  ![Shoryuken](shoryuken.jpg)
4
6
 
5
7
  Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
6
8
 
7
- [![Build Status](https://travis-ci.org/phstc/shoryuken.svg)](https://travis-ci.org/phstc/shoryuken)
9
+ [![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
8
10
  [![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
9
11
 
10
12
  ## Key features
@@ -64,3 +66,41 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
64
66
  3. Commit your changes (`git commit -am 'Add some feature'`)
65
67
  4. Push to the branch (`git push origin my-new-feature`)
66
68
  5. Create a new Pull Request
69
+
70
+ ### Testing
71
+
72
+ To run all unit specs against the latest dependency vesions, execute
73
+
74
+ ```sh
75
+ bundle exec rake spec
76
+ ```
77
+
78
+ To run all Rails-related specs against all supported versions of Rails, execute
79
+
80
+ ```sh
81
+ bundle exec appraisal rake spec:rails
82
+ ```
83
+
84
+ To run integration specs, start a mock SQS server on `localhost:5000`. One such option is [cjlarose/moto-sqs-server](https://github.com/cjlarose/moto-sqs-server). Then execute
85
+
86
+ ```sh
87
+ bundle exec rake spec:integration
88
+ ```
89
+
90
+ ### To release a new version
91
+
92
+ Compare latest tag with HEAD:
93
+
94
+ ```sh
95
+ git log $(git describe --tags --abbrev=0)..HEAD --oneline
96
+ ```
97
+
98
+ then update CHANGELOG.md.
99
+
100
+ Update version in `lib/shoryuken/version.rb` with the appropriate version number [SEMVER](https://semver.org/).
101
+
102
+ then run:
103
+
104
+ ```sh
105
+ bundle exec rake release
106
+ ```
data/Rakefile CHANGED
@@ -3,7 +3,21 @@ $stdout.sync = true
3
3
 
4
4
  begin
5
5
  require 'rspec/core/rake_task'
6
- RSpec::Core::RakeTask.new(:spec)
6
+ RSpec::Core::RakeTask.new(:spec) do |t|
7
+ t.exclude_pattern = 'spec/integration/**/*_spec.rb'
8
+ end
9
+
10
+ namespace :spec do
11
+ desc 'Run Rails specs only'
12
+ RSpec::Core::RakeTask.new(:rails) do |t|
13
+ t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
14
+ end
15
+
16
+ desc 'Run integration specs only'
17
+ RSpec::Core::RakeTask.new(:integration) do |t|
18
+ t.pattern = 'spec/integration/**/*_spec.rb'
19
+ end
20
+ end
7
21
  rescue LoadError
8
22
  end
9
23
 
data/bin/cli/sqs.rb CHANGED
@@ -4,6 +4,9 @@ require 'date'
4
4
  module Shoryuken
5
5
  module CLI
6
6
  class SQS < Base
7
+ # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
8
+ MAX_BATCH_SIZE = 256 * 1024
9
+
7
10
  namespace :sqs
8
11
  class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
9
12
 
@@ -51,14 +54,56 @@ module Shoryuken
51
54
  end
52
55
  end
53
56
 
54
- def batch_send(url, messages, messages_per_batch = 10)
55
- messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch|
56
- sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
57
- say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
57
+ def batch_send(url, messages, max_batch_size = 10)
58
+ messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
59
+ batch_send_normalized_messages url, messages, max_batch_size
60
+ end
61
+
62
+ def batch_send_normalized_messages(url, messages, max_batch_size)
63
+ # Repeatedly take the longest prefix of messages such that
64
+ # 1. The number of messages is less than or equal to max_batch_size
65
+ # 2. The total message payload size is less than or equal to the
66
+ # batch payload limit
67
+ while messages.size.positive?
68
+ batch_size = max_batch_size
69
+ loop do
70
+ batch = messages.take batch_size
71
+
72
+ unless batch.size == 1 || batch_payload_size(batch) <= MAX_BATCH_SIZE
73
+ batch_size = batch.size - 1
74
+ next
75
+ end
76
+
77
+ sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
78
+ say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
79
+ end
80
+ messages = messages.drop batch.size
81
+ break
58
82
  end
59
83
  end
60
84
  end
61
85
 
86
+ def batch_payload_size(messages)
87
+ messages.sum(&method(:message_size))
88
+ end
89
+
90
+ def message_size(message)
91
+ attribute_size = (message[:message_attributes] || []).sum do |name, value|
92
+ name_size = name.to_s.bytesize
93
+ data_type_size = value[:data_type].bytesize
94
+ value_size = if value[:string_value]
95
+ value[:string_value].bytesize
96
+ elsif value[:binary_value]
97
+ value[:binary_value].bytesize
98
+ end
99
+ name_size + data_type_size + value_size
100
+ end
101
+
102
+ body_size = message[:message_body].bytesize
103
+
104
+ attribute_size + body_size
105
+ end
106
+
62
107
  def find_all(url, limit)
63
108
  count = 0
64
109
  batch_size = limit > 10 ? 10 : limit
@@ -72,7 +117,7 @@ module Shoryuken
72
117
  max_number_of_messages: batch_size,
73
118
  attribute_names: ['All'],
74
119
  message_attribute_names: ['All']
75
- ).messages
120
+ ).messages || []
76
121
 
77
122
  messages.each { |m| yield m }
78
123
 
@@ -160,7 +205,7 @@ module Shoryuken
160
205
  end
161
206
 
162
207
  desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
163
- method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
208
+ method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
164
209
  def requeue(queue_name, path)
165
210
  fail_task "Path #{path} not found" unless File.exist?(path)
166
211
 
@@ -0,0 +1 @@
1
+ *.gemfile.lock
@@ -0,0 +1,21 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ group :test do
6
+ gem "activejob"
7
+ gem "aws-sdk-core", "~> 2"
8
+ gem "codeclimate-test-reporter", require: nil
9
+ gem "httparty"
10
+ gem "multi_xml"
11
+ gem "simplecov"
12
+ gem "webrick"
13
+ end
14
+
15
+ group :development do
16
+ gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
17
+ gem "pry-byebug"
18
+ gem "rubocop"
19
+ end
20
+
21
+ gemspec path: "../"
@@ -0,0 +1,20 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ group :test do
6
+ gem "activejob", "~> 4.2"
7
+ gem "aws-sdk-core", "~> 3"
8
+ gem "aws-sdk-sqs"
9
+ gem "codeclimate-test-reporter", require: nil
10
+ gem "httparty"
11
+ gem "multi_xml"
12
+ gem "simplecov"
13
+ end
14
+
15
+ group :development do
16
+ gem "appraisal", "~> 2.2"
17
+ gem "rubocop"
18
+ end
19
+
20
+ gemspec path: "../"
@@ -0,0 +1,21 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ group :test do
6
+ gem "activejob", "~> 5.2"
7
+ gem "aws-sdk-core", "~> 3"
8
+ gem "aws-sdk-sqs"
9
+ gem "codeclimate-test-reporter", require: nil
10
+ gem "httparty"
11
+ gem "multi_xml"
12
+ gem "simplecov"
13
+ end
14
+
15
+ group :development do
16
+ gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
17
+ gem "pry-byebug", "3.9.0"
18
+ gem "rubocop"
19
+ end
20
+
21
+ gemspec path: "../"