gitlab-sidekiq-fetcher 0.5.0.pre.alpha → 0.5.1.pre.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 7da36ba54ba6a0e97cef5da210f02bdde86dc06e22960d37573e51e20046dd40
-  data.tar.gz: bd3bf13edc789374109a2c18b6797b44c1440bc2665938c654e3a627bbc5bc23
+SHA1:
+  metadata.gz: ac14337d2c1c3f109b1d5c9f632236a7b9e5513a
+  data.tar.gz: 683908ba8fb1823fa7cb7078ebd0f9210ddefee6
 SHA512:
-  metadata.gz: 852f01191a052384cfe4215766c6b27dc932ced77782a92c23f73aba091b520e83d293d57b407e45ed5d141661664a9f77b88211b50e9bed52cb15a011b0347a
-  data.tar.gz: 59ee3493c9ddcb3e145ce77259b85ed508f8e57ca97878c12f5427e25952d2a4418860cd1d90562d39aa693dd630359e88ac78fd882aaceb9c5c374b6ca7509a
+  metadata.gz: '00058b4cc9dcf70223cdb4153731a8f774e2caddca3d5287729a29888098351ee1a33b8f15bb9e5a49014dbea950193895dae0ed152806fdf4e6a4ac4df5a59c'
+  data.tar.gz: 04bd84f1a2b6c8b91aab8543c1e0bb34a9845ca6afae01fc2564cc3c5429c8d0c79c8c89984babd9ecfefe0bee21bde67cd3d6c0c94feca15cd3a15dd7fd3ac9

data/.gitlab-ci.yml

@@ -55,6 +55,14 @@ retry_test:
   services:
     - redis:alpine
 
+no_retry_test:
+  stage: test
+  script:
+    - cd tests/retry_test
+    - bundle exec ruby no_retry_test.rb
+  services:
+    - redis:alpine
+
 
 # rubocop:
 #   script:

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'gitlab-sidekiq-fetcher'
-  s.version     = '0.5.0-alpha'
+  s.version     = '0.5.1-alpha'
   s.authors     = ['TEA', 'GitLab']
   s.email       = 'valery@gitlab.com'
   s.license     = 'LGPL-3.0'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -165,6 +165,13 @@ module Sidekiq
     end
 
     def retries_exhausted?(msg)
+      # `retry` parameter can be empty when job is running the first time and when
+      # it's not specified in worker class explicitly.
+      # In that case, the default parameter gets injected into the job when
+      # it fails the first time in JobRetry#local.
+      # We should handle the case when `retry` is explicitly set to false
+      return true if msg['retry'] === false
+
       max_retries_default = Sidekiq.options.fetch(:max_retries, Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS)
 
       max_retry_attempts = retry_attempts_from(msg['retry'], max_retries_default)

data/tests/README.md

@@ -23,6 +23,9 @@ number it will kill all the workers with `kill -9` and then it will spawn new wo
 ```
 cd retry_test
 bundle exec ruby retry_test.rb
+
+# To verify that workers with "retry: false" are not retried
+bundle exec ruby no_retry_test.rb
 ```
 
 It requires Redis to be running on 6379 port.

data/tests/retry_test/config.rb

@@ -2,6 +2,7 @@
 
 require_relative '../../lib/sidekiq-reliable-fetch'
 require_relative 'worker'
+require_relative 'no_retry_worker'
 
 TEST_CLEANUP_INTERVAL = 20
 TEST_LEASE_INTERVAL = 5

data/tests/retry_test/no_retry_test.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require_relative 'config'
+require_relative '../support/utils'
+
+NUM_WORKERS = 2 # one worker will be killed and one spare worker t verify that job is not picked up
+
+Sidekiq.redis(&:flushdb)
+
+pids = spawn_workers(NUM_WORKERS)
+
+jid = NoRetryTestWorker.perform_async
+
+sleep 300
+
+Sidekiq.redis do |redis|
+  times_has_been_run = redis.get('times_has_been_run').to_i
+  assert 'The job has been run', times_has_been_run, 1
+end
+
+assert 'Found dead jobs', Sidekiq::DeadSet.new.size, 1
+
+stop_workers(pids)

data/tests/retry_test/no_retry_worker.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class NoRetryTestWorker
+  include Sidekiq::Worker
+
+  sidekiq_options retry: false
+
+  sidekiq_retry_in do |count, exception|
+    1 # retry in one second
+  end
+
+  def perform
+    sleep 1
+
+    Sidekiq.redis do |redis|
+      redis.incr('times_has_been_run')
+    end
+
+    Process.kill('KILL', Process.pid) # Job suicide, OOM killer imitation
+  end
+end

data/tests/retry_test/retry_test.rb

@@ -1,26 +1,14 @@
 # frozen_string_literal: true
 
 require 'sidekiq'
-require 'sidekiq/util'
-require 'sidekiq/cli'
 require_relative 'config'
-require_relative 'simple_assert'
+require_relative '../support/utils'
 
 NUM_WORKERS = RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN + 1
 
 Sidekiq.redis(&:flushdb)
 
-def spawn_workers
-  pids = []
-
-  NUM_WORKERS.times do
-    pids << spawn('sidekiq -r ./config.rb')
-  end
-
-  pids
-end
-
-pids = spawn_workers
+pids = spawn_workers(NUM_WORKERS)
 
 jid = RetryTestWorker.perform_async
 
@@ -28,13 +16,9 @@ sleep 300
 
 Sidekiq.redis do |redis|
   times_has_been_run = redis.get('times_has_been_run').to_i
-  assert "The job has been run", times_has_been_run, 2
+  assert 'The job has been run', times_has_been_run, RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN
 end
 
-assert "Found dead jobs", Sidekiq::DeadSet.new.size, 1
+assert 'Found dead jobs', Sidekiq::DeadSet.new.size, 1
 
-# Stop Sidekiq workers
-pids.each do |pid|
-  Process.kill('KILL', pid)
-  Process.wait pid
-end
+stop_workers(pids)

data/tests/support/utils.rb

@@ -0,0 +1,26 @@
+def assert(text, actual, expected)
+  if actual == expected
+    puts "#{text}: #{actual} (Success)"
+  else
+    puts "#{text}: #{actual} (Failed). Expected: #{expected}"
+    exit 1
+  end
+end
+
+def spawn_workers(number)
+  pids = []
+
+  number.times do
+    pids << spawn('sidekiq -r ./config.rb')
+  end
+
+  pids
+end
+
+# Stop Sidekiq workers
+def stop_workers(pids)
+  pids.each do |pid|
+    Process.kill('KILL', pid)
+    Process.wait pid
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.5.0.pre.alpha
+  version: 0.5.1.pre.alpha
 platform: ruby
 authors:
 - TEA
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-02 00:00:00.000000000 Z
+date: 2019-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -54,9 +54,11 @@ files:
 - tests/reliability_test/reliability_test.rb
 - tests/reliability_test/worker.rb
 - tests/retry_test/config.rb
+- tests/retry_test/no_retry_test.rb
+- tests/retry_test/no_retry_worker.rb
 - tests/retry_test/retry_test.rb
-- tests/retry_test/simple_assert.rb
 - tests/retry_test/worker.rb
+- tests/support/utils.rb
 homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
@@ -76,7 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq

data/tests/retry_test/simple_assert.rb

@@ -1,8 +0,0 @@
-def assert(text, actual, expected)
-  if actual == expected
-    puts "#{text}: #{actual} (Success)"
-  else
-    puts "#{text}: #{actual} (Failed). Expected: #{expected}"
-    exit 1
-  end
-end