gitlab-sidekiq-fetcher 0.4.0 → 0.5.0.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc2e47cf7679deb6a6d526e199a09c20a671e3b55bad22d7c05ce17405eb6103
-  data.tar.gz: e71949587df8a635223ca8fa36339949df771f493e7edda0a4d9c34198600fb5
+  metadata.gz: 7da36ba54ba6a0e97cef5da210f02bdde86dc06e22960d37573e51e20046dd40
+  data.tar.gz: bd3bf13edc789374109a2c18b6797b44c1440bc2665938c654e3a627bbc5bc23
 SHA512:
-  metadata.gz: ea7d6b7283354053a4f9fc24f419ab56da97efff3ce3ac3a2a10517ab2a1bd184a44b0ffb9d39b1dbdf27b23032c58b1139a18d8f2ec1bf5357deed65187e3ed
-  data.tar.gz: ae4c78eca271dc63abf98bc112a582a2f56fc4e1e7be843189c30193ff0dabd99232f5a019b9b69d45cd1f08ca7bd88afe0c4341e0256c88d352441474a08400
+  metadata.gz: 852f01191a052384cfe4215766c6b27dc932ced77782a92c23f73aba091b520e83d293d57b407e45ed5d141661664a9f77b88211b50e9bed52cb15a011b0347a
+  data.tar.gz: 59ee3493c9ddcb3e145ce77259b85ed508f8e57ca97878c12f5427e25952d2a4418860cd1d90562d39aa693dd630359e88ac78fd882aaceb9c5c374b6ca7509a

data/.gitlab-ci.yml

@@ -3,7 +3,7 @@ image: "ruby:2.5"
 before_script:
   - ruby -v
   - which ruby
-  - gem install bundler --no-ri --no-rdoc
+  - gem install bundler
   - bundle install --jobs $(nproc)  "${FLAGS[@]}"
 
 variables:
@@ -25,7 +25,7 @@ rspec:
 .integration:
   stage: test
   script:
-    - cd test
+    - cd tests/reliability_test
     - bundle exec ruby reliability_test.rb
   services:
     - redis:alpine
@@ -47,6 +47,14 @@ integration_basic:
   variables:
     JOB_FETCHER: basic
 
+retry_test:
+  stage: test
+  script:
+    - cd tests/retry_test
+    - bundle exec ruby retry_test.rb
+  services:
+    - redis:alpine
+
 
 # rubocop:
 #   script:

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'gitlab-sidekiq-fetcher'
-  s.version     = '0.4.0'
+  s.version     = '0.5.0-alpha'
   s.authors     = ['TEA', 'GitLab']
   s.email       = 'valery@gitlab.com'
   s.license     = 'LGPL-3.0'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'sidekiq/job_retry'
+
 module Sidekiq
   class BaseReliableFetch
     DEFAULT_CLEANUP_INTERVAL = 60 * 60  # 1 hour
@@ -137,14 +139,58 @@ module Sidekiq
       Sidekiq.redis do |conn|
         count = 0
 
-        while conn.rpoplpush(working_queue, original_queue) do
-          count += 1
+        while job = conn.rpop(working_queue)
+          msg = begin
+                  Sidekiq.load_json(job)
+                rescue => e
+                  Sidekiq.logger.info("Skipped job: #{job} as we couldn't parse it")
+                  next
+                end
+
+          msg['retry_count'] = msg['retry_count'].to_i + 1
+
+          if retries_exhausted?(msg)
+            send_to_morgue(msg)
+          else
+            job = Sidekiq.dump_json(msg)
+
+            conn.lpush(original_queue, job)
+
+            count += 1
+          end
         end
 
         Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
       end
     end
 
+    def retries_exhausted?(msg)
+      max_retries_default = Sidekiq.options.fetch(:max_retries, Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS)
+
+      max_retry_attempts = retry_attempts_from(msg['retry'], max_retries_default)
+
+      msg['retry_count'] >= max_retry_attempts
+    end
+
+    def retry_attempts_from(msg_retry, default)
+      if msg_retry.is_a?(Integer)
+        msg_retry
+      else
+        default
+      end
+    end
+
+    def send_to_morgue(msg)
+      Sidekiq.logger.warn(
+        class: msg['class'],
+        jid: msg['jid'],
+        message: %(Reliable Fetcher: adding dead #{msg['class']} job #{msg['jid']})
+      )
+
+      payload = Sidekiq.dump_json(msg)
+      Sidekiq::DeadSet.new.kill(payload, notify_failure: false)
+    end
+
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
     def clean_working_queues!

data/spec/base_reliable_fetch_spec.rb

@@ -8,11 +8,12 @@ describe Sidekiq::BaseReliableFetch do
   before { Sidekiq.redis(&:flushdb) }
 
   describe 'UnitOfWork' do
+    let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
     let(:fetcher) { Sidekiq::ReliableFetch.new(queues: ['foo']) }
 
     describe '#requeue' do
       it 'requeues job' do
-        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
+        Sidekiq.redis { |conn| conn.rpush('queue:foo', job) }
 
         uow = fetcher.retrieve_work
 
@@ -25,7 +26,7 @@ describe Sidekiq::BaseReliableFetch do
 
     describe '#acknowledge' do
       it 'acknowledges job' do
-        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
+        Sidekiq.redis { |conn| conn.rpush('queue:foo', job) }
 
         uow = fetcher.retrieve_work
 

data/spec/fetch_shared_examples.rb

@@ -4,33 +4,38 @@ shared_examples 'a Sidekiq fetcher' do
   before { Sidekiq.redis(&:flushdb) }
 
   describe '#retrieve_work' do
+    let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
     let(:fetcher) { described_class.new(queues: ['assigned']) }
 
     it 'retrieves the job and puts it to working queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:assigned', 'msg') }
+      Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
 
       uow = fetcher.retrieve_work
 
       expect(working_queue_size('assigned')).to eq 1
       expect(uow.queue_name).to eq 'assigned'
-      expect(uow.job).to eq 'msg'
+      expect(uow.job).to eq job
       expect(Sidekiq::Queue.new('assigned').size).to eq 0
     end
 
     it 'does not retrieve a job from foreign queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', 'msg') }
+      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', job) }
 
       expect(fetcher.retrieve_work).to be_nil
     end
 
-    it 'requeues jobs from dead working queue' do
+    it 'requeues jobs from dead working queue with incremented retry_count' do
       Sidekiq.redis do |conn|
-        conn.rpush(other_process_working_queue_name('assigned'), 'msg')
+        conn.rpush(other_process_working_queue_name('assigned'), job)
       end
 
+      expected_job = Sidekiq.load_json(job)
+      expected_job['retry_count'] = 1
+      expected_job = Sidekiq.dump_json(expected_job)
+
       uow = fetcher.retrieve_work
 
-      expect(uow.job).to eq 'msg'
+      expect(uow.job).to eq expected_job
 
       Sidekiq.redis do |conn|
         expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
@@ -41,7 +46,7 @@ shared_examples 'a Sidekiq fetcher' do
       working_queue = live_other_process_working_queue_name('assigned')
 
       Sidekiq.redis do |conn|
-        conn.rpush(working_queue, 'msg')
+        conn.rpush(working_queue, job)
       end
 
       uow = fetcher.retrieve_work

data/spec/reliable_fetch_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'fetch_shared_examples'
+require 'sidekiq/base_reliable_fetch'
 require 'sidekiq/reliable_fetch'
 
 describe Sidekiq::ReliableFetch do

data/spec/semi_reliable_fetch_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'fetch_shared_examples'
+require 'sidekiq/base_reliable_fetch'
 require 'sidekiq/semi_reliable_fetch'
 
 describe Sidekiq::SemiReliableFetch do

data/tests/README.md

@@ -0,0 +1,32 @@
+# How to run reliability tests
+
+```
+cd reliability_test
+bundle exec ruby reliability_test.rb
+```
+
+You can adjust some parameters of the test in the `config.rb`.
+
+JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
+
+You need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
+`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).
+
+
+## How it works
+
+This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
+number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
+
+# How to run retry tests
+
+```
+cd retry_test
+bundle exec ruby retry_test.rb
+```
+
+It requires Redis to be running on 6379 port.
+
+## How it works
+
+It spawns Sidekiq workers then creates a job that will kill itself after a moment. The  reliable fetcher will bring it back. The purpose is to verify that job is run no more then `retry` parameter says even when job was killed.

data/{test → tests/reliability_test}/config.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../lib/sidekiq/base_reliable_fetch'
-require_relative '../lib/sidekiq/reliable_fetch'
-require_relative '../lib/sidekiq/semi_reliable_fetch'
+require_relative '../../lib/sidekiq-reliable-fetch'
 require_relative 'worker'
 
 REDIS_FINISHED_LIST = 'reliable-fetcher-finished-jids'

data/{test → tests/reliability_test}/reliability_test.rb

@@ -89,7 +89,7 @@ Sidekiq.redis(&:flushdb)
 jobs = []
 
 NUMBER_OF_JOBS.times do
-  jobs << TestWorker.perform_async
+  jobs << ReliabilityTestWorker.perform_async
 end
 
 puts "Queued #{NUMBER_OF_JOBS} jobs"

data/{test → tests/reliability_test}/worker.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class TestWorker
+class ReliabilityTestWorker
   include Sidekiq::Worker
 
   def perform

data/tests/retry_test/config.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative '../../lib/sidekiq-reliable-fetch'
+require_relative 'worker'
+
+TEST_CLEANUP_INTERVAL = 20
+TEST_LEASE_INTERVAL = 5
+
+Sidekiq.configure_server do |config|
+  config.options[:semi_reliable_fetch] = true
+
+  # We need to override these parameters to not wait too long
+  # The default values are good for production use only
+  # These will be ignored for :basic
+  config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
+  config.options[:lease_interval] = TEST_LEASE_INTERVAL
+
+  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+end

data/tests/retry_test/retry_test.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require 'sidekiq/util'
+require 'sidekiq/cli'
+require_relative 'config'
+require_relative 'simple_assert'
+
+NUM_WORKERS = RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN + 1
+
+Sidekiq.redis(&:flushdb)
+
+def spawn_workers
+  pids = []
+
+  NUM_WORKERS.times do
+    pids << spawn('sidekiq -r ./config.rb')
+  end
+
+  pids
+end
+
+pids = spawn_workers
+
+jid = RetryTestWorker.perform_async
+
+sleep 300
+
+Sidekiq.redis do |redis|
+  times_has_been_run = redis.get('times_has_been_run').to_i
+  assert "The job has been run", times_has_been_run, 2
+end
+
+assert "Found dead jobs", Sidekiq::DeadSet.new.size, 1
+
+# Stop Sidekiq workers
+pids.each do |pid|
+  Process.kill('KILL', pid)
+  Process.wait pid
+end

data/tests/retry_test/simple_assert.rb

@@ -0,0 +1,8 @@
+def assert(text, actual, expected)
+  if actual == expected
+    puts "#{text}: #{actual} (Success)"
+  else
+    puts "#{text}: #{actual} (Failed). Expected: #{expected}"
+    exit 1
+  end
+end

data/tests/retry_test/worker.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class RetryTestWorker
+  include Sidekiq::Worker
+
+  EXPECTED_NUM_TIMES_BEEN_RUN = 2
+
+  sidekiq_options retry: EXPECTED_NUM_TIMES_BEEN_RUN
+
+  sidekiq_retry_in do |count, exception|
+    1 # retry in one second
+  end
+
+  def perform
+    sleep 1
+
+    Sidekiq.redis do |redis|
+      redis.incr('times_has_been_run')
+    end
+
+    Process.kill('KILL', Process.pid) # Job suicide, OOM killer imitation
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0.pre.alpha
 platform: ruby
 authors:
 - TEA
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-19 00:00:00.000000000 Z
+date: 2019-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -49,10 +49,14 @@ files:
 - spec/reliable_fetch_spec.rb
 - spec/semi_reliable_fetch_spec.rb
 - spec/spec_helper.rb
-- test/README.md
-- test/config.rb
-- test/reliability_test.rb
-- test/worker.rb
+- tests/README.md
+- tests/reliability_test/config.rb
+- tests/reliability_test/reliability_test.rb
+- tests/reliability_test/worker.rb
+- tests/retry_test/config.rb
+- tests/retry_test/retry_test.rb
+- tests/retry_test/simple_assert.rb
+- tests/retry_test/worker.rb
 homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
@@ -68,12 +72,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq

data/test/README.md

@@ -1,34 +0,0 @@
-# How to run
-
-```
-cd test
-bundle exec ruby reliability_test.rb
-```
-
-You can adjust some parameters of the test in the `config.rb`
-
-
-# How it works
-
-This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
-number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
-
-# How to run tests
-
-To run rspec:
-
-```
-bundle exec rspec
-```
-
-To run performance tests:
-
-```
-cd test
-JOB_FETCHER=semi bundle exec ruby reliability_test.rb
-```
-
-JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
-
-To run both kind of tests you need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
-`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).