gitlab-sidekiq-fetcher 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7be23d59956ffa44288a1c870bcca66fd0119682f810325d71a3ebaa8b76e80
-  data.tar.gz: 013a7124f61044572ad93335e95c18357c60804dd89024d987485b2d87775787
+  metadata.gz: 7f264e7d628cba58e996b5e2428e81a6e2d6a4921738bd5ade6fd9c3637fa8ba
+  data.tar.gz: 588f7a6e9a5d24d229bc6dbc7d0a6ca4cb9ec96a517bd9015659ed5b3c67eb8e
 SHA512:
-  metadata.gz: 92653bc5f9b5729f4dd50a8243a869c20d9621a1a9d25c46d729e735895e0f2d4d940c5a766803f1a5fd908ab0d9340f27d255dbd99f31bab4923e2f539c1882
-  data.tar.gz: d763b8b0ee3c2522752130fac86b83e67e8513faf919dd361aad9896aac684809650959b1627dd38e0440343c063f95b807b67fc31a10217ce1f15c428759803
+  metadata.gz: 955716fcccc6be01e941e1e366268c78d8179577d007e324bb68a56c7741ee6045e3fc7e66dab1b58d794f5c22ffc2c2af400c95dbe61ad697e8e5685a753eae
+  data.tar.gz: df0fd2c4c72a8401e2f0ac7f8eb3e7f5f47d1fa75da4904e0a0b67c4d80a57a47334e4bdf72728e830f756bbbb6b004412067ad36e8a66a9605471ddc010898f

data/.gitlab-ci.yml

@@ -1,4 +1,8 @@
-image: "ruby:2.5"
+variables:
+  RUBY_VERSION: "2.7"
+
+default:
+  image: ruby:${RUBY_VERSION}
 
 before_script:
   - ruby -v
@@ -21,6 +25,10 @@ rspec:
     when: always
     paths:
       - coverage/
+  parallel:
+    matrix:
+      - RUBY_VERSION: "2.7"
+      - RUBY_VERSION: "3.0"
 
 .integration:
   stage: test
@@ -29,6 +37,10 @@ rspec:
     - bundle exec ruby reliability_test.rb
   services:
     - redis:alpine
+  parallel:
+    matrix:
+      - RUBY_VERSION: "2.7"
+      - RUBY_VERSION: "3.0"
 
 integration_semi:
   extends: .integration
@@ -53,6 +65,10 @@ kill_interruption:
     - bundle exec ruby test_kill_signal.rb
   services:
     - redis:alpine
+  parallel:
+    matrix:
+      - RUBY_VERSION: "2.7"
+      - RUBY_VERSION: "3.0"
 
 term_interruption:
   stage: test
@@ -61,6 +77,10 @@ term_interruption:
     - bundle exec ruby test_term_signal.rb
   services:
     - redis:alpine
+  parallel:
+    matrix:
+      - RUBY_VERSION: "2.7"
+      - RUBY_VERSION: "3.0"
 
 # rubocop:
 #   script:

data/CONTRIBUTING.md

@@ -0,0 +1,41 @@
+## Developer Certificate of Origin and License
+
+By contributing to GitLab B.V., you accept and agree to the following terms and
+conditions for your present and future contributions submitted to GitLab B.V.
+Except for the license granted herein to GitLab B.V. and recipients of software
+distributed by GitLab B.V., you reserve all right, title, and interest in and to
+your Contributions.
+
+All contributions are subject to the Developer Certificate of Origin and license set out at [docs.gitlab.com/ce/legal/developer_certificate_of_origin](https://docs.gitlab.com/ce/legal/developer_certificate_of_origin).
+
+_This notice should stay as the first item in the CONTRIBUTING.md file._
+
+## Code of conduct
+
+As contributors and maintainers of this project, we pledge to respect all people
+who contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual
+language or imagery, derogatory comments or personal attacks, trolling, public
+or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct. Project maintainers who do not follow the
+Code of Conduct may be removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior can be
+reported by emailing contact@gitlab.com.
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org), version 1.1.0,
+available at [https://contributor-covenant.org/version/1/1/0/](https://contributor-covenant.org/version/1/1/0/).
+

data/Gemfile

@@ -4,9 +4,11 @@ source "https://rubygems.org"
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
+gemspec
+
 group :test do
   gem "rspec", '~> 3'
   gem "pry"
-  gem "sidekiq", '~> 6.1'
   gem 'simplecov', require: false
+  gem 'stub_env', '~> 1.0'
 end

data/Gemfile.lock

@@ -1,17 +1,24 @@
+PATH
+  remote: .
+  specs:
+    gitlab-sidekiq-fetcher (0.8.0)
+      json (>= 2.5)
+      sidekiq (~> 6.1)
+
 GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    connection_pool (2.2.3)
+    connection_pool (2.3.0)
     diff-lcs (1.3)
     docile (1.3.1)
-    json (2.1.0)
+    json (2.5.1)
     method_source (0.9.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rack (2.2.3)
-    redis (4.2.1)
+    rack (2.2.4)
+    redis (4.8.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -25,24 +32,27 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    sidekiq (6.1.0)
-      connection_pool (>= 2.2.2)
+    sidekiq (6.5.7)
+      connection_pool (>= 2.2.5)
       rack (~> 2.0)
-      redis (>= 4.2.0)
+      redis (>= 4.5.0, < 5)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
+    stub_env (1.0.4)
+      rspec (>= 2.0, < 4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  gitlab-sidekiq-fetcher!
   pry
   rspec (~> 3)
-  sidekiq (~> 6.1)
   simplecov
+  stub_env (~> 1.0)
 
 BUNDLED WITH
-   1.17.2
+   2.3.24

data/README.md

@@ -46,11 +46,11 @@ Sidekiq.configure_server do |config|
 end
 ```
 
-There is an additional parameter `config.options[:semi_reliable_fetch]` you can use to switch between two strategies:
+There is an additional parameter `config[:semi_reliable_fetch]` you can use to switch between two strategies:
 
 ```ruby
 Sidekiq.configure_server do |config|
-  config.options[:semi_reliable_fetch] = true # Default value is false
+  config[:semi_reliable_fetch] = true # Default value is false
 
   Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
 end

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'gitlab-sidekiq-fetcher'
-  s.version       = '0.8.0'
+  s.version       = '0.9.0'
   s.authors       = ['TEA', 'GitLab']
   s.email         = 'valery@gitlab.com'
   s.license       = 'LGPL-3.0'
@@ -11,4 +11,5 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files`.split($\)
   s.test_files    = []
   s.add_dependency 'sidekiq', '~> 6.1'
+  s.add_runtime_dependency 'json', '>= 2.5'
 end

data/lib/sidekiq/base_reliable_fetch.rb

@@ -45,13 +45,15 @@ module Sidekiq
     end
 
     def self.setup_reliable_fetch!(config)
-      fetch_strategy = if config.options[:semi_reliable_fetch]
+      config = config.options unless config.respond_to?(:[])
+
+      fetch_strategy = if config[:semi_reliable_fetch]
                          Sidekiq::SemiReliableFetch
                        else
                          Sidekiq::ReliableFetch
                        end
 
-      config.options[:fetch] = fetch_strategy.new(config.options)
+      config[:fetch] = fetch_strategy.new(config)
 
       Sidekiq.logger.info('GitLab reliable fetch activated!')
 
@@ -113,6 +115,8 @@ module Sidekiq
     def initialize(options)
       raise ArgumentError, 'missing queue list' unless options[:queues]
 
+      @config = options
+      @interrupted_set = Sidekiq::InterruptedSet.new
       @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
       @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
       @last_try_to_take_lease_at = 0
@@ -225,7 +229,7 @@ module Sidekiq
       rescue NameError
       end
 
-      max_retries_after_interruption ||= Sidekiq.options[:max_retries_after_interruption]
+      max_retries_after_interruption ||= @config[:max_retries_after_interruption]
       max_retries_after_interruption ||= DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION
       max_retries_after_interruption
     end
@@ -238,7 +242,7 @@ module Sidekiq
       )
 
       job = Sidekiq.dump_json(msg)
-      Sidekiq::InterruptedSet.new.put(job, connection: multi_connection)
+      @interrupted_set.put(job, connection: multi_connection)
     end
 
     # Yield block with an existing connection or creates another one

data/lib/sidekiq/interrupted_set.rb

@@ -37,11 +37,15 @@ module Sidekiq
     end
 
     def self.max_jobs
-      Sidekiq.options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
+      options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
     end
 
     def self.timeout
-      Sidekiq.options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
+      options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
+    end
+
+    def self.options
+      Sidekiq.respond_to?(:[]) ? Sidekiq : Sidekiq.options
     end
   end
 end

data/lib/sidekiq/semi_reliable_fetch.rb

@@ -5,14 +5,14 @@ module Sidekiq
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down. This constant is only used
     # for semi-reliable fetch.
-    SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
+    DEFAULT_SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
 
     def initialize(options)
       super
 
       if strictly_ordered_queues
         @queues = @queues.uniq
-        @queues << SEMI_RELIABLE_FETCH_TIMEOUT
+        @queues << { timeout: semi_reliable_fetch_timeout }
       end
     end
 
@@ -36,9 +36,13 @@ module Sidekiq
         @queues
       else
         queues = @queues.shuffle.uniq
-        queues << SEMI_RELIABLE_FETCH_TIMEOUT
+        queues << { timeout: semi_reliable_fetch_timeout }
         queues
       end
     end
+
+    def semi_reliable_fetch_timeout
+      @semi_reliable_fetch_timeout ||= ENV['SIDEKIQ_SEMI_RELIABLE_FETCH_TIMEOUT']&.to_i || DEFAULT_SEMI_RELIABLE_FETCH_TIMEOUT
+    end
   end
 end

data/spec/base_reliable_fetch_spec.rb

@@ -65,7 +65,7 @@ describe Sidekiq::BaseReliableFetch do
     end
 
     it 'does not put jobs into interrupted queue if it is disabled' do
-      Sidekiq.options[:max_retries_after_interruption] = -1
+      options[:max_retries_after_interruption] = -1
 
       uow = described_class::UnitOfWork
       interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
@@ -75,8 +75,6 @@ describe Sidekiq::BaseReliableFetch do
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
       expect(Sidekiq::InterruptedSet.new.size).to eq 0
-
-      Sidekiq.options[:max_retries_after_interruption] = 3
     end
   end
 

data/spec/semi_reliable_fetch_spec.rb

@@ -5,4 +5,39 @@ require 'sidekiq/semi_reliable_fetch'
 
 describe Sidekiq::SemiReliableFetch do
   include_examples 'a Sidekiq fetcher'
+
+  describe '#retrieve_work' do
+    context 'timeout config' do
+      let(:queues) { ['stuff_to_do'] }
+      let(:fetcher) { described_class.new(queues: queues) }
+
+      before do
+        stub_env('SIDEKIQ_SEMI_RELIABLE_FETCH_TIMEOUT', timeout)
+      end
+
+      context 'when the timeout is not configured' do
+        let(:timeout) { nil }
+
+        it 'brpops with the default timeout timeout' do
+          Sidekiq.redis do |connection|
+            expect(connection).to receive(:brpop).with("queue:stuff_to_do", { timeout: 2 }).once.and_call_original
+
+            fetcher.retrieve_work
+          end
+        end
+      end
+
+      context 'when the timeout is set in the env' do
+        let(:timeout) { '5' }
+
+        it 'brpops with the default timeout timeout' do
+          Sidekiq.redis do |connection|
+            expect(connection).to receive(:brpop).with("queue:stuff_to_do", { timeout: 5 }).once.and_call_original
+
+            fetcher.retrieve_work
+          end
+        end
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -1,8 +1,8 @@
 require 'sidekiq'
-require 'sidekiq/util'
 require 'sidekiq/api'
 require 'pry'
 require 'simplecov'
+require 'stub_env'
 
 SimpleCov.start
 
@@ -29,6 +29,7 @@ Sidekiq.logger.level = Logger::ERROR
 #
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 RSpec.configure do |config|
+  config.include StubEnv::Helpers
   # rspec-expectations config goes here. You can use an alternate
   # assertion/expectation library such as wrong or the stdlib/minitest
   # assertions if you prefer.

data/tests/README.md

@@ -1,7 +1,7 @@
 # How to run reliability tests
 
 ```
-cd reliability_test
+cd tests/reliability
 bundle exec ruby reliability_test.rb
 ```
 

data/tests/interruption/config.rb

@@ -7,13 +7,13 @@ TEST_CLEANUP_INTERVAL = 20
 TEST_LEASE_INTERVAL = 5
 
 Sidekiq.configure_server do |config|
-  config.options[:semi_reliable_fetch] = true
+  config[:semi_reliable_fetch] = true
 
   # We need to override these parameters to not wait too long
   # The default values are good for production use only
   # These will be ignored for :basic
-  config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
-  config.options[:lease_interval] = TEST_LEASE_INTERVAL
+  config[:cleanup_interval] = TEST_CLEANUP_INTERVAL
+  config[:lease_interval] = TEST_LEASE_INTERVAL
 
   Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
 end

data/tests/reliability/config.rb

@@ -16,13 +16,13 @@ WAIT_CLEANUP = TEST_CLEANUP_INTERVAL +
 
 Sidekiq.configure_server do |config|
   if %i[semi reliable].include?(JOB_FETCHER)
-    config.options[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
+    config[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
 
     # We need to override these parameters to not wait too long
     # The default values are good for production use only
     # These will be ignored for :basic
-    config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
-    config.options[:lease_interval] = TEST_LEASE_INTERVAL
+    config[:cleanup_interval] = TEST_CLEANUP_INTERVAL
+    config[:lease_interval] = TEST_LEASE_INTERVAL
 
     Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
   end

data/tests/reliability/reliability_test.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'sidekiq'
-require 'sidekiq/util'
 require 'sidekiq/cli'
 require_relative 'config'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - TEA
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-02 00:00:00.000000000 Z
+date: 2022-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -25,6 +25,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.5'
 description: Redis reliable queue pattern implemented in Sidekiq
 email: valery@gitlab.com
 executables: []
@@ -34,6 +48,7 @@ files:
 - ".gitignore"
 - ".gitlab-ci.yml"
 - ".rspec"
+- CONTRIBUTING.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -78,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.22
 signing_key:
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq