gitlab-sidekiq-fetcher 0.8.0 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c7be23d59956ffa44288a1c870bcca66fd0119682f810325d71a3ebaa8b76e80
4
- data.tar.gz: 013a7124f61044572ad93335e95c18357c60804dd89024d987485b2d87775787
3
+ metadata.gz: 7f264e7d628cba58e996b5e2428e81a6e2d6a4921738bd5ade6fd9c3637fa8ba
4
+ data.tar.gz: 588f7a6e9a5d24d229bc6dbc7d0a6ca4cb9ec96a517bd9015659ed5b3c67eb8e
5
5
  SHA512:
6
- metadata.gz: 92653bc5f9b5729f4dd50a8243a869c20d9621a1a9d25c46d729e735895e0f2d4d940c5a766803f1a5fd908ab0d9340f27d255dbd99f31bab4923e2f539c1882
7
- data.tar.gz: d763b8b0ee3c2522752130fac86b83e67e8513faf919dd361aad9896aac684809650959b1627dd38e0440343c063f95b807b67fc31a10217ce1f15c428759803
6
+ metadata.gz: 955716fcccc6be01e941e1e366268c78d8179577d007e324bb68a56c7741ee6045e3fc7e66dab1b58d794f5c22ffc2c2af400c95dbe61ad697e8e5685a753eae
7
+ data.tar.gz: df0fd2c4c72a8401e2f0ac7f8eb3e7f5f47d1fa75da4904e0a0b67c4d80a57a47334e4bdf72728e830f756bbbb6b004412067ad36e8a66a9605471ddc010898f
data/.gitlab-ci.yml CHANGED
@@ -1,4 +1,8 @@
1
- image: "ruby:2.5"
1
+ variables:
2
+ RUBY_VERSION: "2.7"
3
+
4
+ default:
5
+ image: ruby:${RUBY_VERSION}
2
6
 
3
7
  before_script:
4
8
  - ruby -v
@@ -21,6 +25,10 @@ rspec:
21
25
  when: always
22
26
  paths:
23
27
  - coverage/
28
+ parallel:
29
+ matrix:
30
+ - RUBY_VERSION: "2.7"
31
+ - RUBY_VERSION: "3.0"
24
32
 
25
33
  .integration:
26
34
  stage: test
@@ -29,6 +37,10 @@ rspec:
29
37
  - bundle exec ruby reliability_test.rb
30
38
  services:
31
39
  - redis:alpine
40
+ parallel:
41
+ matrix:
42
+ - RUBY_VERSION: "2.7"
43
+ - RUBY_VERSION: "3.0"
32
44
 
33
45
  integration_semi:
34
46
  extends: .integration
@@ -53,6 +65,10 @@ kill_interruption:
53
65
  - bundle exec ruby test_kill_signal.rb
54
66
  services:
55
67
  - redis:alpine
68
+ parallel:
69
+ matrix:
70
+ - RUBY_VERSION: "2.7"
71
+ - RUBY_VERSION: "3.0"
56
72
 
57
73
  term_interruption:
58
74
  stage: test
@@ -61,6 +77,10 @@ term_interruption:
61
77
  - bundle exec ruby test_term_signal.rb
62
78
  services:
63
79
  - redis:alpine
80
+ parallel:
81
+ matrix:
82
+ - RUBY_VERSION: "2.7"
83
+ - RUBY_VERSION: "3.0"
64
84
 
65
85
  # rubocop:
66
86
  # script:
data/CONTRIBUTING.md ADDED
@@ -0,0 +1,41 @@
1
+ ## Developer Certificate of Origin and License
2
+
3
+ By contributing to GitLab B.V., you accept and agree to the following terms and
4
+ conditions for your present and future contributions submitted to GitLab B.V.
5
+ Except for the license granted herein to GitLab B.V. and recipients of software
6
+ distributed by GitLab B.V., you reserve all right, title, and interest in and to
7
+ your Contributions.
8
+
9
+ All contributions are subject to the Developer Certificate of Origin and license set out at [docs.gitlab.com/ce/legal/developer_certificate_of_origin](https://docs.gitlab.com/ce/legal/developer_certificate_of_origin).
10
+
11
+ _This notice should stay as the first item in the CONTRIBUTING.md file._
12
+
13
+ ## Code of conduct
14
+
15
+ As contributors and maintainers of this project, we pledge to respect all people
16
+ who contribute through reporting issues, posting feature requests, updating
17
+ documentation, submitting pull requests or patches, and other activities.
18
+
19
+ We are committed to making participation in this project a harassment-free
20
+ experience for everyone, regardless of level of experience, gender, gender
21
+ identity and expression, sexual orientation, disability, personal appearance,
22
+ body size, race, ethnicity, age, or religion.
23
+
24
+ Examples of unacceptable behavior by participants include the use of sexual
25
+ language or imagery, derogatory comments or personal attacks, trolling, public
26
+ or private harassment, insults, or other unprofessional conduct.
27
+
28
+ Project maintainers have the right and responsibility to remove, edit, or reject
29
+ comments, commits, code, wiki edits, issues, and other contributions that are
30
+ not aligned to this Code of Conduct. Project maintainers who do not follow the
31
+ Code of Conduct may be removed from the project team.
32
+
33
+ This code of conduct applies both within project spaces and in public spaces
34
+ when an individual is representing the project or its community.
35
+
36
+ Instances of abusive, harassing, or otherwise unacceptable behavior can be
37
+ reported by emailing contact@gitlab.com.
38
+
39
+ This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org), version 1.1.0,
40
+ available at [https://contributor-covenant.org/version/1/1/0/](https://contributor-covenant.org/version/1/1/0/).
41
+
data/Gemfile CHANGED
@@ -4,9 +4,11 @@ source "https://rubygems.org"
4
4
 
5
5
  git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
6
6
 
7
+ gemspec
8
+
7
9
  group :test do
8
10
  gem "rspec", '~> 3'
9
11
  gem "pry"
10
- gem "sidekiq", '~> 6.1'
11
12
  gem 'simplecov', require: false
13
+ gem 'stub_env', '~> 1.0'
12
14
  end
data/Gemfile.lock CHANGED
@@ -1,17 +1,24 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ gitlab-sidekiq-fetcher (0.8.0)
5
+ json (>= 2.5)
6
+ sidekiq (~> 6.1)
7
+
1
8
  GEM
2
9
  remote: https://rubygems.org/
3
10
  specs:
4
11
  coderay (1.1.2)
5
- connection_pool (2.2.3)
12
+ connection_pool (2.3.0)
6
13
  diff-lcs (1.3)
7
14
  docile (1.3.1)
8
- json (2.1.0)
15
+ json (2.5.1)
9
16
  method_source (0.9.0)
10
17
  pry (0.11.3)
11
18
  coderay (~> 1.1.0)
12
19
  method_source (~> 0.9.0)
13
- rack (2.2.3)
14
- redis (4.2.1)
20
+ rack (2.2.4)
21
+ redis (4.8.0)
15
22
  rspec (3.8.0)
16
23
  rspec-core (~> 3.8.0)
17
24
  rspec-expectations (~> 3.8.0)
@@ -25,24 +32,27 @@ GEM
25
32
  diff-lcs (>= 1.2.0, < 2.0)
26
33
  rspec-support (~> 3.8.0)
27
34
  rspec-support (3.8.0)
28
- sidekiq (6.1.0)
29
- connection_pool (>= 2.2.2)
35
+ sidekiq (6.5.7)
36
+ connection_pool (>= 2.2.5)
30
37
  rack (~> 2.0)
31
- redis (>= 4.2.0)
38
+ redis (>= 4.5.0, < 5)
32
39
  simplecov (0.16.1)
33
40
  docile (~> 1.1)
34
41
  json (>= 1.8, < 3)
35
42
  simplecov-html (~> 0.10.0)
36
43
  simplecov-html (0.10.2)
44
+ stub_env (1.0.4)
45
+ rspec (>= 2.0, < 4.0)
37
46
 
38
47
  PLATFORMS
39
48
  ruby
40
49
 
41
50
  DEPENDENCIES
51
+ gitlab-sidekiq-fetcher!
42
52
  pry
43
53
  rspec (~> 3)
44
- sidekiq (~> 6.1)
45
54
  simplecov
55
+ stub_env (~> 1.0)
46
56
 
47
57
  BUNDLED WITH
48
- 1.17.2
58
+ 2.3.24
data/README.md CHANGED
@@ -46,11 +46,11 @@ Sidekiq.configure_server do |config|
46
46
  end
47
47
  ```
48
48
 
49
- There is an additional parameter `config.options[:semi_reliable_fetch]` you can use to switch between two strategies:
49
+ There is an additional parameter `config[:semi_reliable_fetch]` you can use to switch between two strategies:
50
50
 
51
51
  ```ruby
52
52
  Sidekiq.configure_server do |config|
53
- config.options[:semi_reliable_fetch] = true # Default value is false
53
+ config[:semi_reliable_fetch] = true # Default value is false
54
54
 
55
55
  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
56
56
  end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'gitlab-sidekiq-fetcher'
3
- s.version = '0.8.0'
3
+ s.version = '0.9.0'
4
4
  s.authors = ['TEA', 'GitLab']
5
5
  s.email = 'valery@gitlab.com'
6
6
  s.license = 'LGPL-3.0'
@@ -11,4 +11,5 @@ Gem::Specification.new do |s|
11
11
  s.files = `git ls-files`.split($\)
12
12
  s.test_files = []
13
13
  s.add_dependency 'sidekiq', '~> 6.1'
14
+ s.add_runtime_dependency 'json', '>= 2.5'
14
15
  end
@@ -45,13 +45,15 @@ module Sidekiq
45
45
  end
46
46
 
47
47
  def self.setup_reliable_fetch!(config)
48
- fetch_strategy = if config.options[:semi_reliable_fetch]
48
+ config = config.options unless config.respond_to?(:[])
49
+
50
+ fetch_strategy = if config[:semi_reliable_fetch]
49
51
  Sidekiq::SemiReliableFetch
50
52
  else
51
53
  Sidekiq::ReliableFetch
52
54
  end
53
55
 
54
- config.options[:fetch] = fetch_strategy.new(config.options)
56
+ config[:fetch] = fetch_strategy.new(config)
55
57
 
56
58
  Sidekiq.logger.info('GitLab reliable fetch activated!')
57
59
 
@@ -113,6 +115,8 @@ module Sidekiq
113
115
  def initialize(options)
114
116
  raise ArgumentError, 'missing queue list' unless options[:queues]
115
117
 
118
+ @config = options
119
+ @interrupted_set = Sidekiq::InterruptedSet.new
116
120
  @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
117
121
  @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
118
122
  @last_try_to_take_lease_at = 0
@@ -225,7 +229,7 @@ module Sidekiq
225
229
  rescue NameError
226
230
  end
227
231
 
228
- max_retries_after_interruption ||= Sidekiq.options[:max_retries_after_interruption]
232
+ max_retries_after_interruption ||= @config[:max_retries_after_interruption]
229
233
  max_retries_after_interruption ||= DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION
230
234
  max_retries_after_interruption
231
235
  end
@@ -238,7 +242,7 @@ module Sidekiq
238
242
  )
239
243
 
240
244
  job = Sidekiq.dump_json(msg)
241
- Sidekiq::InterruptedSet.new.put(job, connection: multi_connection)
245
+ @interrupted_set.put(job, connection: multi_connection)
242
246
  end
243
247
 
244
248
  # Yield block with an existing connection or creates another one
@@ -37,11 +37,15 @@ module Sidekiq
37
37
  end
38
38
 
39
39
  def self.max_jobs
40
- Sidekiq.options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
40
+ options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
41
41
  end
42
42
 
43
43
  def self.timeout
44
- Sidekiq.options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
44
+ options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
45
+ end
46
+
47
+ def self.options
48
+ Sidekiq.respond_to?(:[]) ? Sidekiq : Sidekiq.options
45
49
  end
46
50
  end
47
51
  end
@@ -5,14 +5,14 @@ module Sidekiq
5
5
  # We want the fetch operation to timeout every few seconds so the thread
6
6
  # can check if the process is shutting down. This constant is only used
7
7
  # for semi-reliable fetch.
8
- SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
8
+ DEFAULT_SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
9
9
 
10
10
  def initialize(options)
11
11
  super
12
12
 
13
13
  if strictly_ordered_queues
14
14
  @queues = @queues.uniq
15
- @queues << SEMI_RELIABLE_FETCH_TIMEOUT
15
+ @queues << { timeout: semi_reliable_fetch_timeout }
16
16
  end
17
17
  end
18
18
 
@@ -36,9 +36,13 @@ module Sidekiq
36
36
  @queues
37
37
  else
38
38
  queues = @queues.shuffle.uniq
39
- queues << SEMI_RELIABLE_FETCH_TIMEOUT
39
+ queues << { timeout: semi_reliable_fetch_timeout }
40
40
  queues
41
41
  end
42
42
  end
43
+
44
+ def semi_reliable_fetch_timeout
45
+ @semi_reliable_fetch_timeout ||= ENV['SIDEKIQ_SEMI_RELIABLE_FETCH_TIMEOUT']&.to_i || DEFAULT_SEMI_RELIABLE_FETCH_TIMEOUT
46
+ end
43
47
  end
44
48
  end
@@ -65,7 +65,7 @@ describe Sidekiq::BaseReliableFetch do
65
65
  end
66
66
 
67
67
  it 'does not put jobs into interrupted queue if it is disabled' do
68
- Sidekiq.options[:max_retries_after_interruption] = -1
68
+ options[:max_retries_after_interruption] = -1
69
69
 
70
70
  uow = described_class::UnitOfWork
71
71
  interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
@@ -75,8 +75,6 @@ describe Sidekiq::BaseReliableFetch do
75
75
  expect(queue1.size).to eq 2
76
76
  expect(queue2.size).to eq 1
77
77
  expect(Sidekiq::InterruptedSet.new.size).to eq 0
78
-
79
- Sidekiq.options[:max_retries_after_interruption] = 3
80
78
  end
81
79
  end
82
80
 
@@ -5,4 +5,39 @@ require 'sidekiq/semi_reliable_fetch'
5
5
 
6
6
  describe Sidekiq::SemiReliableFetch do
7
7
  include_examples 'a Sidekiq fetcher'
8
+
9
+ describe '#retrieve_work' do
10
+ context 'timeout config' do
11
+ let(:queues) { ['stuff_to_do'] }
12
+ let(:fetcher) { described_class.new(queues: queues) }
13
+
14
+ before do
15
+ stub_env('SIDEKIQ_SEMI_RELIABLE_FETCH_TIMEOUT', timeout)
16
+ end
17
+
18
+ context 'when the timeout is not configured' do
19
+ let(:timeout) { nil }
20
+
21
+ it 'brpops with the default timeout timeout' do
22
+ Sidekiq.redis do |connection|
23
+ expect(connection).to receive(:brpop).with("queue:stuff_to_do", { timeout: 2 }).once.and_call_original
24
+
25
+ fetcher.retrieve_work
26
+ end
27
+ end
28
+ end
29
+
30
+ context 'when the timeout is set in the env' do
31
+ let(:timeout) { '5' }
32
+
33
+ it 'brpops with the default timeout timeout' do
34
+ Sidekiq.redis do |connection|
35
+ expect(connection).to receive(:brpop).with("queue:stuff_to_do", { timeout: 5 }).once.and_call_original
36
+
37
+ fetcher.retrieve_work
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
8
43
  end
data/spec/spec_helper.rb CHANGED
@@ -1,8 +1,8 @@
1
1
  require 'sidekiq'
2
- require 'sidekiq/util'
3
2
  require 'sidekiq/api'
4
3
  require 'pry'
5
4
  require 'simplecov'
5
+ require 'stub_env'
6
6
 
7
7
  SimpleCov.start
8
8
 
@@ -29,6 +29,7 @@ Sidekiq.logger.level = Logger::ERROR
29
29
  #
30
30
  # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
31
31
  RSpec.configure do |config|
32
+ config.include StubEnv::Helpers
32
33
  # rspec-expectations config goes here. You can use an alternate
33
34
  # assertion/expectation library such as wrong or the stdlib/minitest
34
35
  # assertions if you prefer.
data/tests/README.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # How to run reliability tests
2
2
 
3
3
  ```
4
- cd reliability_test
4
+ cd tests/reliability
5
5
  bundle exec ruby reliability_test.rb
6
6
  ```
7
7
 
@@ -7,13 +7,13 @@ TEST_CLEANUP_INTERVAL = 20
7
7
  TEST_LEASE_INTERVAL = 5
8
8
 
9
9
  Sidekiq.configure_server do |config|
10
- config.options[:semi_reliable_fetch] = true
10
+ config[:semi_reliable_fetch] = true
11
11
 
12
12
  # We need to override these parameters to not wait too long
13
13
  # The default values are good for production use only
14
14
  # These will be ignored for :basic
15
- config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
16
- config.options[:lease_interval] = TEST_LEASE_INTERVAL
15
+ config[:cleanup_interval] = TEST_CLEANUP_INTERVAL
16
+ config[:lease_interval] = TEST_LEASE_INTERVAL
17
17
 
18
18
  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
19
19
  end
@@ -16,13 +16,13 @@ WAIT_CLEANUP = TEST_CLEANUP_INTERVAL +
16
16
 
17
17
  Sidekiq.configure_server do |config|
18
18
  if %i[semi reliable].include?(JOB_FETCHER)
19
- config.options[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
19
+ config[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
20
20
 
21
21
  # We need to override these parameters to not wait too long
22
22
  # The default values are good for production use only
23
23
  # These will be ignored for :basic
24
- config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
25
- config.options[:lease_interval] = TEST_LEASE_INTERVAL
24
+ config[:cleanup_interval] = TEST_CLEANUP_INTERVAL
25
+ config[:lease_interval] = TEST_LEASE_INTERVAL
26
26
 
27
27
  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
28
28
  end
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'sidekiq'
4
- require 'sidekiq/util'
5
4
  require 'sidekiq/cli'
6
5
  require_relative 'config'
7
6
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-sidekiq-fetcher
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - TEA
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2021-03-02 00:00:00.000000000 Z
12
+ date: 2022-11-03 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: sidekiq
@@ -25,6 +25,20 @@ dependencies:
25
25
  - - "~>"
26
26
  - !ruby/object:Gem::Version
27
27
  version: '6.1'
28
+ - !ruby/object:Gem::Dependency
29
+ name: json
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - ">="
33
+ - !ruby/object:Gem::Version
34
+ version: '2.5'
35
+ type: :runtime
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - ">="
40
+ - !ruby/object:Gem::Version
41
+ version: '2.5'
28
42
  description: Redis reliable queue pattern implemented in Sidekiq
29
43
  email: valery@gitlab.com
30
44
  executables: []
@@ -34,6 +48,7 @@ files:
34
48
  - ".gitignore"
35
49
  - ".gitlab-ci.yml"
36
50
  - ".rspec"
51
+ - CONTRIBUTING.md
37
52
  - Gemfile
38
53
  - Gemfile.lock
39
54
  - LICENSE
@@ -78,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
78
93
  - !ruby/object:Gem::Version
79
94
  version: '0'
80
95
  requirements: []
81
- rubygems_version: 3.1.4
96
+ rubygems_version: 3.2.22
82
97
  signing_key:
83
98
  specification_version: 4
84
99
  summary: Reliable fetch extension for Sidekiq