shoryuken 5.0.5 → 6.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.devcontainer/Dockerfile +17 -0
- data/.devcontainer/base.Dockerfile +43 -0
- data/.devcontainer/devcontainer.json +35 -0
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/specs.yml +65 -0
- data/.github/workflows/stale.yml +20 -0
- data/.gitignore +1 -1
- data/.reek.yml +5 -0
- data/.rubocop.yml +1 -1
- data/Appraisals +42 -0
- data/CHANGELOG.md +114 -0
- data/Gemfile +8 -3
- data/README.md +41 -1
- data/Rakefile +15 -1
- data/bin/cli/sqs.rb +51 -6
- data/gemfiles/.gitignore +1 -0
- data/gemfiles/aws_sdk_core_2.gemfile +21 -0
- data/gemfiles/rails_4_2.gemfile +20 -0
- data/gemfiles/rails_5_2.gemfile +21 -0
- data/gemfiles/rails_6_0.gemfile +21 -0
- data/gemfiles/rails_6_1.gemfile +21 -0
- data/gemfiles/rails_7_0.gemfile +22 -0
- data/lib/shoryuken/default_exception_handler.rb +10 -0
- data/lib/shoryuken/environment_loader.rb +22 -4
- data/lib/shoryuken/extensions/active_job_adapter.rb +30 -20
- data/lib/shoryuken/extensions/active_job_extensions.rb +38 -0
- data/lib/shoryuken/launcher.rb +26 -3
- data/lib/shoryuken/logging.rb +2 -2
- data/lib/shoryuken/manager.rb +50 -14
- data/lib/shoryuken/message.rb +11 -28
- data/lib/shoryuken/options.rb +6 -3
- data/lib/shoryuken/polling/base.rb +2 -0
- data/lib/shoryuken/polling/strict_priority.rb +8 -0
- data/lib/shoryuken/polling/weighted_round_robin.rb +9 -0
- data/lib/shoryuken/processor.rb +1 -2
- data/lib/shoryuken/queue.rb +5 -3
- data/lib/shoryuken/runner.rb +4 -3
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken.rb +8 -0
- data/shoryuken.gemspec +1 -2
- data/spec/integration/launcher_spec.rb +29 -2
- data/spec/shared_examples_for_active_job.rb +230 -11
- data/spec/shoryuken/default_exception_handler_spec.rb +71 -0
- data/spec/shoryuken/environment_loader_spec.rb +62 -9
- data/spec/shoryuken/extensions/active_job_adapter_spec.rb +1 -1
- data/spec/shoryuken/extensions/active_job_base_spec.rb +84 -0
- data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +4 -0
- data/spec/shoryuken/extensions/active_job_wrapper_spec.rb +20 -0
- data/spec/shoryuken/fetcher_spec.rb +12 -12
- data/spec/shoryuken/launcher_spec.rb +105 -0
- data/spec/shoryuken/manager_spec.rb +61 -1
- data/spec/shoryuken/polling/strict_priority_spec.rb +10 -0
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +35 -0
- data/spec/shoryuken/queue_spec.rb +10 -5
- data/spec/shoryuken/worker/default_executor_spec.rb +48 -48
- data/spec/shoryuken_spec.rb +9 -0
- data/spec/spec_helper.rb +7 -9
- metadata +32 -24
- data/.travis.yml +0 -30
- data/Gemfile.aws-sdk-core-v2 +0 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: feeefb6a4aad595df6286c02928284e9403b0f0a81ba188f3fe399138dd8e32c
|
4
|
+
data.tar.gz: a6dde4ed4a888f005580202bfd416fe2a41610fbc3c666efb31210cc2e831487
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 39df6a6ff9f262e7e9337487d484433c2182b8c71eb6401bdf7e0798505993a8b1a80af00f034eccf3718fafc0ca616c83cf8c5937eebbf73333ea52578077aa
|
7
|
+
data.tar.gz: 0bec3c731f75b848a6b7ddcc8eaba03331f71e2eb98687c9955ff9fc9b83196044771d182d07bb470d4a7b16070c7d06fe988561955942c740dd0eb89b4d4356
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# [Choice] Ruby version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.1, 3.0, 2, 2.7, 2.6, 3-bullseye, 3.1-bullseye, 3.0-bullseye, 2-bullseye, 2.7-bullseye, 2.6-bullseye, 3-buster, 3.1-buster, 3.0-buster, 2-buster, 2.7-buster, 2.6-buster
|
2
|
+
ARG VARIANT=2-bullseye
|
3
|
+
FROM mcr.microsoft.com/vscode/devcontainers/ruby:0-${VARIANT}
|
4
|
+
|
5
|
+
# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
|
6
|
+
ARG NODE_VERSION="none"
|
7
|
+
RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
|
8
|
+
|
9
|
+
# [Optional] Uncomment this section to install additional OS packages.
|
10
|
+
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
11
|
+
# && apt-get -y install --no-install-recommends <your-package-list-here>
|
12
|
+
|
13
|
+
# [Optional] Uncomment this line to install additional gems.
|
14
|
+
# RUN gem install <your-gem-names-here>
|
15
|
+
|
16
|
+
# [Optional] Uncomment this line to install global node packages.
|
17
|
+
# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# [Choice] Ruby version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.1, 3.0, 2, 2.7, 2.6, 3-bullseye, 3.1-bullseye, 3.0-bullseye, 2-bullseye, 2.7-bullseye, 2.6-bullseye, 3-buster, 3.1-buster, 3.0-buster, 2-buster, 2.7-buster, 2.6-buster
|
2
|
+
ARG VARIANT=2-bullseye
|
3
|
+
FROM ruby:${VARIANT}
|
4
|
+
|
5
|
+
# Copy library scripts to execute
|
6
|
+
COPY library-scripts/*.sh library-scripts/*.env /tmp/library-scripts/
|
7
|
+
|
8
|
+
# [Option] Install zsh
|
9
|
+
ARG INSTALL_ZSH="true"
|
10
|
+
# [Option] Upgrade OS packages to their latest versions
|
11
|
+
ARG UPGRADE_PACKAGES="true"
|
12
|
+
# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies.
|
13
|
+
ARG USERNAME=vscode
|
14
|
+
ARG USER_UID=1000
|
15
|
+
ARG USER_GID=$USER_UID
|
16
|
+
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
17
|
+
# Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
|
18
|
+
&& apt-get purge -y imagemagick imagemagick-6-common \
|
19
|
+
# Install common packages, non-root user, rvm, core build tools
|
20
|
+
&& bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \
|
21
|
+
&& bash /tmp/library-scripts/ruby-debian.sh "none" "${USERNAME}" "true" "true" \
|
22
|
+
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
|
23
|
+
|
24
|
+
# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
|
25
|
+
ARG NODE_VERSION="none"
|
26
|
+
ENV NVM_DIR=/usr/local/share/nvm
|
27
|
+
ENV NVM_SYMLINK_CURRENT=true \
|
28
|
+
PATH=${NVM_DIR}/current/bin:${PATH}
|
29
|
+
RUN bash /tmp/library-scripts/node-debian.sh "${NVM_DIR}" "${NODE_VERSION}" "${USERNAME}" \
|
30
|
+
&& apt-get clean -y && rm -rf /var/lib/apt/lists/*
|
31
|
+
|
32
|
+
# Remove library scripts for final image
|
33
|
+
RUN rm -rf /tmp/library-scripts
|
34
|
+
|
35
|
+
# [Optional] Uncomment this section to install additional OS packages.
|
36
|
+
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
|
37
|
+
# && apt-get -y install --no-install-recommends <your-package-list-here>
|
38
|
+
|
39
|
+
# [Optional] Uncomment this line to install additional gems.
|
40
|
+
# RUN gem install <your-gem-names-here>
|
41
|
+
|
42
|
+
# [Optional] Uncomment this line to install global node packages.
|
43
|
+
# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
|
@@ -0,0 +1,35 @@
|
|
1
|
+
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
|
2
|
+
// https://github.com/microsoft/vscode-dev-containers/tree/v0.224.3/containers/ruby
|
3
|
+
{
|
4
|
+
"name": "Ruby",
|
5
|
+
"build": {
|
6
|
+
"dockerfile": "Dockerfile",
|
7
|
+
"args": {
|
8
|
+
// Update 'VARIANT' to pick a Ruby version: 3, 3.1, 3.0, 2, 2.7, 2.6
|
9
|
+
// Append -bullseye or -buster to pin to an OS version.
|
10
|
+
// Use -bullseye variants on local on arm64/Apple Silicon.
|
11
|
+
"VARIANT": "3-bullseye",
|
12
|
+
// Options
|
13
|
+
"NODE_VERSION": "none"
|
14
|
+
}
|
15
|
+
},
|
16
|
+
|
17
|
+
// Set *default* container specific settings.json values on container create.
|
18
|
+
"settings": {},
|
19
|
+
|
20
|
+
// Add the IDs of extensions you want installed when the container is created.
|
21
|
+
"extensions": ["rebornix.Ruby"],
|
22
|
+
|
23
|
+
// Use 'forwardPorts' to make a list of ports inside the container available locally.
|
24
|
+
// "forwardPorts": [],
|
25
|
+
|
26
|
+
// Use 'postCreateCommand' to run commands after the container is created.
|
27
|
+
// "postCreateCommand": "ruby --version",
|
28
|
+
|
29
|
+
// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
|
30
|
+
"remoteUser": "vscode",
|
31
|
+
"features": {
|
32
|
+
"github-cli": "latest",
|
33
|
+
"aws-cli": "latest"
|
34
|
+
}
|
35
|
+
}
|
@@ -0,0 +1,65 @@
|
|
1
|
+
name: Specs
|
2
|
+
|
3
|
+
on:
|
4
|
+
- push
|
5
|
+
- pull_request
|
6
|
+
|
7
|
+
jobs:
|
8
|
+
all_specs:
|
9
|
+
name: All Specs
|
10
|
+
strategy:
|
11
|
+
matrix:
|
12
|
+
ruby: ['2.4', '2.5', '2.6', '2.7', '3.0', '3.1', '3.2']
|
13
|
+
gemfile: ['Gemfile', 'gemfiles/aws_sdk_core_2.gemfile']
|
14
|
+
runs-on: ubuntu-20.04
|
15
|
+
services:
|
16
|
+
moto_sqs:
|
17
|
+
image: quay.io/cjlarose/moto-sqs-server:1.1.0
|
18
|
+
ports:
|
19
|
+
- 5000:5000
|
20
|
+
env:
|
21
|
+
BUNDLE_GEMFILE: ${{ matrix.gemfile }}
|
22
|
+
steps:
|
23
|
+
- name: Checkout code
|
24
|
+
uses: actions/checkout@v2
|
25
|
+
- uses: ruby/setup-ruby@v1
|
26
|
+
with:
|
27
|
+
ruby-version: ${{ matrix.ruby }}
|
28
|
+
bundler-cache: true
|
29
|
+
- name: Run specs
|
30
|
+
run: bundle exec rake spec
|
31
|
+
- name: Run integration specs
|
32
|
+
run: bundle exec rake spec:integration
|
33
|
+
rails_specs:
|
34
|
+
name: Rails Specs
|
35
|
+
strategy:
|
36
|
+
matrix:
|
37
|
+
rails: ['4.2', '5.2', '6.0', '6.1', '7.0']
|
38
|
+
include:
|
39
|
+
- rails: '4.2'
|
40
|
+
ruby: '2.2'
|
41
|
+
gemfile: gemfiles/rails_4_2.gemfile
|
42
|
+
- rails: '5.2'
|
43
|
+
ruby: '2.5'
|
44
|
+
gemfile: gemfiles/rails_5_2.gemfile
|
45
|
+
- rails: '6.0'
|
46
|
+
ruby: '2.6'
|
47
|
+
gemfile: gemfiles/rails_6_0.gemfile
|
48
|
+
- rails: '6.1'
|
49
|
+
ruby: '3.0'
|
50
|
+
gemfile: gemfiles/rails_6_1.gemfile
|
51
|
+
- rails: '7.0'
|
52
|
+
ruby: '3.1'
|
53
|
+
gemfile: gemfiles/rails_7_0.gemfile
|
54
|
+
runs-on: ubuntu-20.04
|
55
|
+
env:
|
56
|
+
BUNDLE_GEMFILE: ${{ matrix.gemfile }}
|
57
|
+
steps:
|
58
|
+
- name: Checkout code
|
59
|
+
uses: actions/checkout@v2
|
60
|
+
- uses: ruby/setup-ruby@v1
|
61
|
+
with:
|
62
|
+
ruby-version: ${{ matrix.ruby }}
|
63
|
+
bundler-cache: true
|
64
|
+
- name: Run Rails specs
|
65
|
+
run: bundle exec rake spec:rails
|
@@ -0,0 +1,20 @@
|
|
1
|
+
name: "Close stale issues and PRs"
|
2
|
+
on:
|
3
|
+
schedule:
|
4
|
+
- cron: "30 1 * * *" # At 01:30 - https://crontab.guru/#30_1_*_*_*
|
5
|
+
workflow_dispatch: {}
|
6
|
+
jobs:
|
7
|
+
stale:
|
8
|
+
runs-on: ubuntu-latest
|
9
|
+
permissions:
|
10
|
+
issues: write
|
11
|
+
pull-requests: write
|
12
|
+
steps:
|
13
|
+
- uses: actions/stale@v8
|
14
|
+
with:
|
15
|
+
stale-issue-message: This issue is now marked as stale because it hasn't seen activity for a while. Add a comment or it will be closed soon.
|
16
|
+
stale-pr-message: This PR is now marked as stale because it hasn't seen activity for a while. Add a comment or it will be closed soon.
|
17
|
+
close-issue-message: This issue was closed because it hasn't seen activity for a while.
|
18
|
+
close-pr-message: This PR was closed because it hasn't seen activity for a while.
|
19
|
+
days-before-stale: 60
|
20
|
+
days-before-close: 7
|
data/.gitignore
CHANGED
data/.reek.yml
ADDED
data/.rubocop.yml
CHANGED
data/Appraisals
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
appraise 'aws_sdk_core_2' do
|
2
|
+
group :test do
|
3
|
+
gem 'aws-sdk-core', '~> 2'
|
4
|
+
gem 'webrick' # required for Ruby 3
|
5
|
+
remove_gem 'aws-sdk-sqs'
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
appraise 'rails_4_2' do
|
10
|
+
group :test do
|
11
|
+
gem 'activejob', '~> 4.2'
|
12
|
+
end
|
13
|
+
|
14
|
+
group :development do
|
15
|
+
gem 'appraisal', '~> 2.2'
|
16
|
+
remove_gem 'pry-byebug'
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
appraise 'rails_5_2' do
|
21
|
+
group :test do
|
22
|
+
gem 'activejob', '~> 5.2'
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
appraise 'rails_6_0' do
|
27
|
+
group :test do
|
28
|
+
gem 'activejob', '~> 6.0'
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
appraise 'rails_6_1' do
|
33
|
+
group :test do
|
34
|
+
gem 'activejob', '~> 6.1'
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
appraise 'rails_7_0' do
|
39
|
+
group :test do
|
40
|
+
gem 'activejob', '~> 7.0'
|
41
|
+
end
|
42
|
+
end
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,117 @@
|
|
1
|
+
## [v6.1.1] - 2023-11-27
|
2
|
+
|
3
|
+
- Fix SQS API Changes causing nil returns instead of empty arrays
|
4
|
+
- [#754](https://github.com/ruby-shoryuken/shoryuken/pull/754)
|
5
|
+
- [#753](https://github.com/ruby-shoryuken/shoryuken/pull/753)
|
6
|
+
|
7
|
+
## [v6.1.0] - 2023-11-01
|
8
|
+
|
9
|
+
- Add GitHub Codespaces
|
10
|
+
- [#698](https://github.com/ruby-shoryuken/shoryuken/pull/698)
|
11
|
+
|
12
|
+
- Fix spec for ruby 3.0
|
13
|
+
- [#727](https://github.com/ruby-shoryuken/shoryuken/pull/727)
|
14
|
+
|
15
|
+
- Upgrade test matrix. Add Ruby 3.1, Ruby 3.2 and Rails 7
|
16
|
+
- [#739](https://github.com/ruby-shoryuken/shoryuken/pull/739)
|
17
|
+
|
18
|
+
- Fire stopped event after executor is stopped
|
19
|
+
- [#741](https://github.com/ruby-shoryuken/shoryuken/pull/741)
|
20
|
+
|
21
|
+
- Allow setup custom exception handlers for failing jobs
|
22
|
+
- [#742](https://github.com/ruby-shoryuken/shoryuken/pull/742)
|
23
|
+
|
24
|
+
- Configure dependabot to update GH Actions
|
25
|
+
- [#745](https://github.com/ruby-shoryuken/shoryuken/pull/745)
|
26
|
+
|
27
|
+
- Stop the dispatching of new messages when a SIGTERM signal has been received
|
28
|
+
- [#750](https://github.com/ruby-shoryuken/shoryuken/pull/750)
|
29
|
+
|
30
|
+
## [v6.0.0] - 2022-02-18
|
31
|
+
|
32
|
+
- Breaking changes: Initialize Rails before parsing config file
|
33
|
+
- [#686](https://github.com/ruby-shoryuken/shoryuken/pull/686)
|
34
|
+
- Previously, Shoryuken read its configuration from an optional YAML file, then allowed CLI arguments to override those, then initialized the Rails application (provided that `--rails` or `-R` was specified). This behavior meant that the config file did not have access to things like environment variables that were initialized by Rails (such as when using `dotenv`). With this change, Rails is initialized much earlier in the process. After Rails is initialized, the YAML configuration file is interpreted, and CLI arguments are finally interpreted last. Most applications will not need to undergo changes in order to upgrade, but the new load order could technically result in different behavior depending on the application's YAML configuration file or Rails initializers.
|
35
|
+
|
36
|
+
## [v5.3.2] - 2022-01-19
|
37
|
+
|
38
|
+
- (Bugfix) Preserve queue weights when unpausing queues
|
39
|
+
- [#687](https://github.com/ruby-shoryuken/shoryuken/pull/687)
|
40
|
+
|
41
|
+
- Improve error message on startup when shoryuken has insufficient permissions to access a queue
|
42
|
+
- [#691](https://github.com/ruby-shoryuken/shoryuken/pull/691)
|
43
|
+
|
44
|
+
## [v5.3.1] - 2022-01-07
|
45
|
+
|
46
|
+
- (Bugfix) Fix issue where, when using the TSTP or USR1 signals for soft shutdowns, it was possible for shoryuken to terminate without first attempting to handle all messages it fetched from SQS
|
47
|
+
- [#676](https://github.com/ruby-shoryuken/shoryuken/pull/676)
|
48
|
+
|
49
|
+
## [v5.3.0] - 2021-10-31
|
50
|
+
|
51
|
+
- (Refactor) Use Forwardable within Message to avoid method boilerplate
|
52
|
+
- [#681](https://github.com/ruby-shoryuken/shoryuken/pull/681)
|
53
|
+
|
54
|
+
- Add basic health check API
|
55
|
+
- [#679](https://github.com/ruby-shoryuken/shoryuken/pull/679)
|
56
|
+
|
57
|
+
## [v5.2.3] - 2021-07-29
|
58
|
+
|
59
|
+
- Fire new `:utilization_update` event any time a worker pool's utilization changes
|
60
|
+
- [#673](https://github.com/ruby-shoryuken/shoryuken/pull/673)
|
61
|
+
|
62
|
+
## [v5.2.2] - 2021-06-22
|
63
|
+
|
64
|
+
- When using ActiveJob queue name prefixing, avoid applying prefix to queues configured with a URL or ARN
|
65
|
+
- [#667](https://github.com/ruby-shoryuken/shoryuken/pull/667)
|
66
|
+
|
67
|
+
## [v5.2.1] - 2021-04-06
|
68
|
+
|
69
|
+
- Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands
|
70
|
+
- [#666](https://github.com/ruby-shoryuken/shoryuken/pull/666)
|
71
|
+
|
72
|
+
- Fix bug in `shoryuken sqs requeue` and `shoryuken sqs mv` where those commands would exceed the SQS `SendMessageBatch` maximum payload size
|
73
|
+
- [#663](https://github.com/ruby-shoryuken/shoryuken/issues/663)
|
74
|
+
- [#664](https://github.com/ruby-shoryuken/shoryuken/pull/664)
|
75
|
+
|
76
|
+
- Remove test stub for `Concurrent.global_io_executor`
|
77
|
+
- [#662](https://github.com/ruby-shoryuken/shoryuken/pull/662)
|
78
|
+
|
79
|
+
- Run integration tests on CI
|
80
|
+
- [#660](https://github.com/ruby-shoryuken/shoryuken/pull/660)
|
81
|
+
|
82
|
+
## [v5.2.0] - 2021-02-26
|
83
|
+
|
84
|
+
- Set `executions` correctly for ActiveJob jobs
|
85
|
+
- [#657](https://github.com/phstc/shoryuken/pull/657)
|
86
|
+
|
87
|
+
## [v5.1.1] - 2021-02-10
|
88
|
+
|
89
|
+
- Fix regression in Ruby 3.0 introduced in Shoryuken 5.1.0, where enqueueing jobs with ActiveJob to workers that used keyword arguments would fail
|
90
|
+
- [#654](https://github.com/phstc/shoryuken/pull/654)
|
91
|
+
|
92
|
+
## [v5.1.0] - 2021-02-06
|
93
|
+
|
94
|
+
- Add support for specifying SQS SendMessage parameters with ActiveJob `.set`
|
95
|
+
- [#635](https://github.com/phstc/shoryuken/pull/635)
|
96
|
+
- [#648](https://github.com/phstc/shoryuken/pull/648)
|
97
|
+
- [#651](https://github.com/phstc/shoryuken/pull/651)
|
98
|
+
|
99
|
+
- Unpause FIFO queues on worker completion
|
100
|
+
- [#644](https://github.com/phstc/shoryuken/pull/644)
|
101
|
+
|
102
|
+
- Add multiple versions of Rails to test matrix
|
103
|
+
- [#647](https://github.com/phstc/shoryuken/pull/647)
|
104
|
+
|
105
|
+
- Migrate from Travis CI to Github Actions
|
106
|
+
- [#649](https://github.com/phstc/shoryuken/pull/649)
|
107
|
+
- [#650](https://github.com/phstc/shoryuken/pull/650)
|
108
|
+
- [#652](https://github.com/phstc/shoryuken/pull/652)
|
109
|
+
|
110
|
+
## [v5.0.6] - 2020-12-30
|
111
|
+
|
112
|
+
- Load ShoryukenConcurrentSendAdapter when loading Rails
|
113
|
+
- [#642](https://github.com/phstc/shoryuken/pull/642)
|
114
|
+
|
1
115
|
## [v5.0.5] - 2020-06-07
|
2
116
|
|
3
117
|
- Add ability to configure queue by ARN
|
data/Gemfile
CHANGED
@@ -4,9 +4,12 @@ source 'https://rubygems.org'
|
|
4
4
|
gemspec
|
5
5
|
|
6
6
|
group :test do
|
7
|
-
gem 'activejob'
|
7
|
+
gem 'activejob'
|
8
8
|
gem 'aws-sdk-core', '~> 3'
|
9
|
-
|
9
|
+
# Pin to 1.65.0 because of below issues:
|
10
|
+
# - https://github.com/ruby-shoryuken/shoryuken/pull/753#issuecomment-1822720647
|
11
|
+
# - https://github.com/getmoto/moto/issues/7054
|
12
|
+
gem 'aws-sdk-sqs', '1.65.0'
|
10
13
|
gem 'codeclimate-test-reporter', require: nil
|
11
14
|
gem 'httparty'
|
12
15
|
gem 'multi_xml'
|
@@ -14,5 +17,7 @@ group :test do
|
|
14
17
|
end
|
15
18
|
|
16
19
|
group :development do
|
17
|
-
gem '
|
20
|
+
gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
|
21
|
+
gem 'pry-byebug'
|
22
|
+
gem 'rubocop', '<= 1.12'
|
18
23
|
end
|
data/README.md
CHANGED
@@ -1,10 +1,12 @@
|
|
1
|
+
**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? [Join our Slack](https://join.slack.com/t/shoryuken/shared_invite/zt-19xjq3iqc-KmoJ6eU6~qvZNqcLzIrjww)**
|
2
|
+
|
1
3
|
# Shoryuken
|
2
4
|
|
3
5
|
![Shoryuken](shoryuken.jpg)
|
4
6
|
|
5
7
|
Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
|
6
8
|
|
7
|
-
[![Build Status](https://
|
9
|
+
[![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
|
8
10
|
[![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
|
9
11
|
|
10
12
|
## Key features
|
@@ -64,3 +66,41 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
|
|
64
66
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
65
67
|
4. Push to the branch (`git push origin my-new-feature`)
|
66
68
|
5. Create a new Pull Request
|
69
|
+
|
70
|
+
### Testing
|
71
|
+
|
72
|
+
To run all unit specs against the latest dependency vesions, execute
|
73
|
+
|
74
|
+
```sh
|
75
|
+
bundle exec rake spec
|
76
|
+
```
|
77
|
+
|
78
|
+
To run all Rails-related specs against all supported versions of Rails, execute
|
79
|
+
|
80
|
+
```sh
|
81
|
+
bundle exec appraisal rake spec:rails
|
82
|
+
```
|
83
|
+
|
84
|
+
To run integration specs, start a mock SQS server on `localhost:5000`. One such option is [cjlarose/moto-sqs-server](https://github.com/cjlarose/moto-sqs-server). Then execute
|
85
|
+
|
86
|
+
```sh
|
87
|
+
bundle exec rake spec:integration
|
88
|
+
```
|
89
|
+
|
90
|
+
### To release a new version
|
91
|
+
|
92
|
+
Compare latest tag with HEAD:
|
93
|
+
|
94
|
+
```sh
|
95
|
+
git log $(git describe --tags --abbrev=0)..HEAD --oneline
|
96
|
+
```
|
97
|
+
|
98
|
+
then update CHANGELOG.md.
|
99
|
+
|
100
|
+
Update version in `lib/shoryuken/version.rb` with the appropriate version number [SEMVER](https://semver.org/).
|
101
|
+
|
102
|
+
then run:
|
103
|
+
|
104
|
+
```sh
|
105
|
+
bundle exec rake release
|
106
|
+
```
|
data/Rakefile
CHANGED
@@ -3,7 +3,21 @@ $stdout.sync = true
|
|
3
3
|
|
4
4
|
begin
|
5
5
|
require 'rspec/core/rake_task'
|
6
|
-
RSpec::Core::RakeTask.new(:spec)
|
6
|
+
RSpec::Core::RakeTask.new(:spec) do |t|
|
7
|
+
t.exclude_pattern = 'spec/integration/**/*_spec.rb'
|
8
|
+
end
|
9
|
+
|
10
|
+
namespace :spec do
|
11
|
+
desc 'Run Rails specs only'
|
12
|
+
RSpec::Core::RakeTask.new(:rails) do |t|
|
13
|
+
t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'Run integration specs only'
|
17
|
+
RSpec::Core::RakeTask.new(:integration) do |t|
|
18
|
+
t.pattern = 'spec/integration/**/*_spec.rb'
|
19
|
+
end
|
20
|
+
end
|
7
21
|
rescue LoadError
|
8
22
|
end
|
9
23
|
|
data/bin/cli/sqs.rb
CHANGED
@@ -4,6 +4,9 @@ require 'date'
|
|
4
4
|
module Shoryuken
|
5
5
|
module CLI
|
6
6
|
class SQS < Base
|
7
|
+
# See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
|
8
|
+
MAX_BATCH_SIZE = 256 * 1024
|
9
|
+
|
7
10
|
namespace :sqs
|
8
11
|
class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
|
9
12
|
|
@@ -51,14 +54,56 @@ module Shoryuken
|
|
51
54
|
end
|
52
55
|
end
|
53
56
|
|
54
|
-
def batch_send(url, messages,
|
55
|
-
messages.to_a.flatten.map(&method(:normalize_dump_message))
|
56
|
-
|
57
|
-
|
57
|
+
def batch_send(url, messages, max_batch_size = 10)
|
58
|
+
messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
|
59
|
+
batch_send_normalized_messages url, messages, max_batch_size
|
60
|
+
end
|
61
|
+
|
62
|
+
def batch_send_normalized_messages(url, messages, max_batch_size)
|
63
|
+
# Repeatedly take the longest prefix of messages such that
|
64
|
+
# 1. The number of messages is less than or equal to max_batch_size
|
65
|
+
# 2. The total message payload size is less than or equal to the
|
66
|
+
# batch payload limit
|
67
|
+
while messages.size.positive?
|
68
|
+
batch_size = max_batch_size
|
69
|
+
loop do
|
70
|
+
batch = messages.take batch_size
|
71
|
+
|
72
|
+
unless batch.size == 1 || batch_payload_size(batch) <= MAX_BATCH_SIZE
|
73
|
+
batch_size = batch.size - 1
|
74
|
+
next
|
75
|
+
end
|
76
|
+
|
77
|
+
sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
|
78
|
+
say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
|
79
|
+
end
|
80
|
+
messages = messages.drop batch.size
|
81
|
+
break
|
58
82
|
end
|
59
83
|
end
|
60
84
|
end
|
61
85
|
|
86
|
+
def batch_payload_size(messages)
|
87
|
+
messages.sum(&method(:message_size))
|
88
|
+
end
|
89
|
+
|
90
|
+
def message_size(message)
|
91
|
+
attribute_size = (message[:message_attributes] || []).sum do |name, value|
|
92
|
+
name_size = name.to_s.bytesize
|
93
|
+
data_type_size = value[:data_type].bytesize
|
94
|
+
value_size = if value[:string_value]
|
95
|
+
value[:string_value].bytesize
|
96
|
+
elsif value[:binary_value]
|
97
|
+
value[:binary_value].bytesize
|
98
|
+
end
|
99
|
+
name_size + data_type_size + value_size
|
100
|
+
end
|
101
|
+
|
102
|
+
body_size = message[:message_body].bytesize
|
103
|
+
|
104
|
+
attribute_size + body_size
|
105
|
+
end
|
106
|
+
|
62
107
|
def find_all(url, limit)
|
63
108
|
count = 0
|
64
109
|
batch_size = limit > 10 ? 10 : limit
|
@@ -72,7 +117,7 @@ module Shoryuken
|
|
72
117
|
max_number_of_messages: batch_size,
|
73
118
|
attribute_names: ['All'],
|
74
119
|
message_attribute_names: ['All']
|
75
|
-
).messages
|
120
|
+
).messages || []
|
76
121
|
|
77
122
|
messages.each { |m| yield m }
|
78
123
|
|
@@ -160,7 +205,7 @@ module Shoryuken
|
|
160
205
|
end
|
161
206
|
|
162
207
|
desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
|
163
|
-
method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
|
208
|
+
method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
|
164
209
|
def requeue(queue_name, path)
|
165
210
|
fail_task "Path #{path} not found" unless File.exist?(path)
|
166
211
|
|
data/gemfiles/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
*.gemfile.lock
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem "activejob"
|
7
|
+
gem "aws-sdk-core", "~> 2"
|
8
|
+
gem "codeclimate-test-reporter", require: nil
|
9
|
+
gem "httparty"
|
10
|
+
gem "multi_xml"
|
11
|
+
gem "simplecov"
|
12
|
+
gem "webrick"
|
13
|
+
end
|
14
|
+
|
15
|
+
group :development do
|
16
|
+
gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
|
17
|
+
gem "pry-byebug"
|
18
|
+
gem "rubocop"
|
19
|
+
end
|
20
|
+
|
21
|
+
gemspec path: "../"
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem "activejob", "~> 4.2"
|
7
|
+
gem "aws-sdk-core", "~> 3"
|
8
|
+
gem "aws-sdk-sqs"
|
9
|
+
gem "codeclimate-test-reporter", require: nil
|
10
|
+
gem "httparty"
|
11
|
+
gem "multi_xml"
|
12
|
+
gem "simplecov"
|
13
|
+
end
|
14
|
+
|
15
|
+
group :development do
|
16
|
+
gem "appraisal", "~> 2.2"
|
17
|
+
gem "rubocop"
|
18
|
+
end
|
19
|
+
|
20
|
+
gemspec path: "../"
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem "activejob", "~> 5.2"
|
7
|
+
gem "aws-sdk-core", "~> 3"
|
8
|
+
gem "aws-sdk-sqs"
|
9
|
+
gem "codeclimate-test-reporter", require: nil
|
10
|
+
gem "httparty"
|
11
|
+
gem "multi_xml"
|
12
|
+
gem "simplecov"
|
13
|
+
end
|
14
|
+
|
15
|
+
group :development do
|
16
|
+
gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
|
17
|
+
gem "pry-byebug", "3.9.0"
|
18
|
+
gem "rubocop"
|
19
|
+
end
|
20
|
+
|
21
|
+
gemspec path: "../"
|