gitlab-sidekiq-fetcher 0.5.0.pre.alpha → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7da36ba54ba6a0e97cef5da210f02bdde86dc06e22960d37573e51e20046dd40
-  data.tar.gz: bd3bf13edc789374109a2c18b6797b44c1440bc2665938c654e3a627bbc5bc23
+  metadata.gz: a71699d717aeb95cb406ed566e394a34e7978df8289db2fe89aff82046dd8e19
+  data.tar.gz: 4390a2a95507b8a6c5c08a9d3dac384f58f0d11311afd1da5973e4c61240120a
 SHA512:
-  metadata.gz: 852f01191a052384cfe4215766c6b27dc932ced77782a92c23f73aba091b520e83d293d57b407e45ed5d141661664a9f77b88211b50e9bed52cb15a011b0347a
-  data.tar.gz: 59ee3493c9ddcb3e145ce77259b85ed508f8e57ca97878c12f5427e25952d2a4418860cd1d90562d39aa693dd630359e88ac78fd882aaceb9c5c374b6ca7509a
+  metadata.gz: fa95de2f9b33f01b45c547b23bfd4728ab88ed304ecbeaa2416e03b1a6d6bff678f097b37ae85679ef69f520a37a6f7115207170284229df706172f60d103b19
+  data.tar.gz: d7981fd4afe0abf8454caffac4c1fa8e290dd8617ce28edf45b4a29a9deb9b3a9cd583f64038cc3808b814923ab74fc370d65d9a42257b4af9fa97ea48542959

data/.gitignore

@@ -1,2 +1,3 @@
 *.gem
 coverage
+.DS_Store

data/.gitlab-ci.yml

@@ -25,7 +25,7 @@ rspec:
 .integration:
   stage: test
   script:
-    - cd tests/reliability_test
+    - cd tests/reliability
     - bundle exec ruby reliability_test.rb
   services:
     - redis:alpine
@@ -47,11 +47,19 @@ integration_basic:
   variables:
     JOB_FETCHER: basic
 
-retry_test:
+kill_interruption:
   stage: test
   script:
-    - cd tests/retry_test
-    - bundle exec ruby retry_test.rb
+    - cd tests/interruption
+    - bundle exec ruby test_kill_signal.rb
+  services:
+    - redis:alpine
+
+term_interruption:
+  stage: test
+  script:
+    - cd tests/interruption
+    - bundle exec ruby test_term_signal.rb
   services:
     - redis:alpine
 

data/Gemfile

@@ -7,6 +7,6 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 group :test do
   gem "rspec", '~> 3'
   gem "pry"
-  gem "sidekiq", '~> 5.0'
+  gem "sidekiq", '~> 6.0.0'
   gem 'simplecov', require: false
 end

data/Gemfile.lock

@@ -2,7 +2,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     docile (1.3.1)
     json (2.1.0)
@@ -10,10 +10,10 @@ GEM
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rack (2.0.5)
-    rack-protection (2.0.4)
+    rack (2.2.3)
+    rack-protection (2.0.8.1)
       rack
-    redis (4.0.2)
+    redis (4.2.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -27,10 +27,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    sidekiq (5.2.2)
-      connection_pool (~> 2.2, >= 2.2.2)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
+    sidekiq (6.0.7)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      rack-protection (>= 2.0.0)
+      redis (>= 4.1.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -43,8 +44,8 @@ PLATFORMS
 DEPENDENCIES
   pry
   rspec (~> 3)
-  sidekiq (~> 5.0)
+  sidekiq (~> 6.0.0)
   simplecov
 
 BUNDLED WITH
-   1.17.1
+   1.17.2

data/README.md

@@ -10,6 +10,17 @@ There are two strategies implemented: [Reliable fetch](http://redis.io/commands/
 semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
 However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.
 
+### Interruption handling
+
+Sidekiq expects any job to report succcess or to fail. In the last case, Sidekiq puts `retry_count` counter
+into the job and keeps to re-run the job until the counter reched the maximum allowed value. When the job has
+not been given a chance to finish its work(to report success or fail), for example, when it was killed forcibly or when the job was requeued, after receiving TERM signal, the standard retry mechanisme does not get into the game and the job will be retried indefinatelly. This is why Reliable fetcher maintains a special counter `interrupted_count`
+which is used to limit the amount of such retries. In both cases, Reliable Fetcher increments counter `interrupted_count` and rejects the job from running again when the counter exceeds `max_retries_after_interruption` times (default: 3 times).
+Such a job will be put to `interrupted` queue. This queue mostly behaves as Sidekiq Dead queue so it only stores a limited amount of jobs for a limited term. Same as for Dead queue, all the limits are configurable via `interrupted_max_jobs` (default: 10_000) and `interrupted_timeout_in_seconds` (default: 3 months) Sidekiq option keys.
+
+You can also disable special handling of interrupted jobs by setting `max_retries_after_interruption` into `-1`.
+In this case, interrupted jobs will be run without any limits from Reliable Fetcher and they won't be put into Interrupted queue.
+
 
 ## Installation
 

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,14 +1,14 @@
 Gem::Specification.new do |s|
-  s.name        = 'gitlab-sidekiq-fetcher'
-  s.version     = '0.5.0-alpha'
-  s.authors     = ['TEA', 'GitLab']
-  s.email       = 'valery@gitlab.com'
-  s.license     = 'LGPL-3.0'
-  s.homepage    = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
-  s.summary     = 'Reliable fetch extension for Sidekiq'
-  s.description = 'Redis reliable queue pattern implemented in Sidekiq'
+  s.name          = 'gitlab-sidekiq-fetcher'
+  s.version       = '0.6.1'
+  s.authors       = ['TEA', 'GitLab']
+  s.email         = 'valery@gitlab.com'
+  s.license       = 'LGPL-3.0'
+  s.homepage      = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
+  s.summary       = 'Reliable fetch extension for Sidekiq'
+  s.description   = 'Redis reliable queue pattern implemented in Sidekiq'
   s.require_paths = ['lib']
-  s.files = `git ls-files`.split($\)
-  s.test_files  = []
-  s.add_dependency 'sidekiq', '~> 5'
+  s.files         = `git ls-files`.split($\)
+  s.test_files    = []
+  s.add_dependency 'sidekiq', '>= 5', '< 6.1'
 end

data/lib/sidekiq-reliable-fetch.rb

@@ -1,4 +1,5 @@
 require 'sidekiq'
+require 'sidekiq/api'
 
 require_relative 'sidekiq/base_reliable_fetch'
 require_relative 'sidekiq/reliable_fetch'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'sidekiq/job_retry'
+require_relative 'interrupted_set'
 
 module Sidekiq
   class BaseReliableFetch
@@ -18,6 +18,9 @@ module Sidekiq
     # Defines the COUNT parameter that will be passed to Redis SCAN command
     SCAN_COUNT = 1000
 
+    # How much time a job can be interrupted
+    DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION = 3
+
     UnitOfWork = Struct.new(:queue, :job) do
       def acknowledge
         Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
@@ -84,136 +87,145 @@ module Sidekiq
     def self.bulk_requeue(inprogress, _options)
       return if inprogress.empty?
 
-      Sidekiq.logger.debug('Re-queueing terminated jobs')
-
       Sidekiq.redis do |conn|
         inprogress.each do |unit_of_work|
           conn.multi do |multi|
-            multi.lpush(unit_of_work.queue, unit_of_work.job)
+            preprocess_interrupted_job(unit_of_work.job, unit_of_work.queue, multi)
+
             multi.lrem(working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
           end
         end
       end
-
-      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => e
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
     end
 
-    def self.heartbeat_key(hostname, pid)
-      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
-    end
+    def self.clean_working_queue!(working_queue)
+      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
 
-    def self.working_queue_name(queue)
-      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+      Sidekiq.redis do |conn|
+        while job = conn.rpop(working_queue)
+          preprocess_interrupted_job(job, original_queue)
+        end
+      end
     end
 
-    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
-                :queues, :use_semi_reliable_fetch,
-                :strictly_ordered_queues
+    def self.preprocess_interrupted_job(job, queue, conn = nil)
+      msg = Sidekiq.load_json(job)
+      msg['interrupted_count'] = msg['interrupted_count'].to_i + 1
 
-    def initialize(options)
-      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
-      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
-      @last_try_to_take_lease_at = 0
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
+      if interruption_exhausted?(msg)
+        send_to_quarantine(msg, conn)
+      else
+        requeue_job(queue, msg, conn)
+      end
     end
 
-    def retrieve_work
-      clean_working_queues! if take_lease
-
-      retrieve_unit_of_work
-    end
+    # Detect "old" jobs and requeue them because the worker they were assigned
+    # to probably failed miserably.
+    def self.clean_working_queues!
+      Sidekiq.logger.info('Cleaning working queues')
 
-    def retrieve_unit_of_work
-      raise NotImplementedError,
-        "#{self.class} does not implement #{__method__}"
-    end
+      Sidekiq.redis do |conn|
+        conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
+          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
+          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
 
-    private
+          continue if hostname.nil? || pid.nil?
 
-    def clean_working_queue!(working_queue)
-      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
+          clean_working_queue!(key) if worker_dead?(hostname, pid, conn)
+        end
+      end
+    end
 
-      Sidekiq.redis do |conn|
-        count = 0
+    def self.worker_dead?(hostname, pid, conn)
+      !conn.get(heartbeat_key(hostname, pid))
+    end
 
-        while job = conn.rpop(working_queue)
-          msg = begin
-                  Sidekiq.load_json(job)
-                rescue => e
-                  Sidekiq.logger.info("Skipped job: #{job} as we couldn't parse it")
-                  next
-                end
+    def self.heartbeat_key(hostname, pid)
+      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
+    end
 
-          msg['retry_count'] = msg['retry_count'].to_i + 1
+    def self.working_queue_name(queue)
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+    end
 
-          if retries_exhausted?(msg)
-            send_to_morgue(msg)
-          else
-            job = Sidekiq.dump_json(msg)
+    def self.interruption_exhausted?(msg)
+      return false if max_retries_after_interruption(msg['class']) < 0
 
-            conn.lpush(original_queue, job)
+      msg['interrupted_count'].to_i >= max_retries_after_interruption(msg['class'])
+    end
 
-            count += 1
-          end
-        end
+    def self.max_retries_after_interruption(worker_class)
+      max_retries_after_interruption = nil
 
-        Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
+      max_retries_after_interruption ||= begin
+        Object.const_get(worker_class).sidekiq_options[:max_retries_after_interruption]
+      rescue NameError
       end
-    end
 
-    def retries_exhausted?(msg)
-      max_retries_default = Sidekiq.options.fetch(:max_retries, Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS)
+      max_retries_after_interruption ||= Sidekiq.options[:max_retries_after_interruption]
+      max_retries_after_interruption ||= DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION
+      max_retries_after_interruption
+    end
 
-      max_retry_attempts = retry_attempts_from(msg['retry'], max_retries_default)
+    def self.send_to_quarantine(msg, multi_connection = nil)
+      Sidekiq.logger.warn(
+        class: msg['class'],
+        jid: msg['jid'],
+        message: %(Reliable Fetcher: adding dead #{msg['class']} job #{msg['jid']} to interrupted queue)
+      )
 
-      msg['retry_count'] >= max_retry_attempts
+      job = Sidekiq.dump_json(msg)
+      Sidekiq::InterruptedSet.new.put(job, connection: multi_connection)
     end
 
-    def retry_attempts_from(msg_retry, default)
-      if msg_retry.is_a?(Integer)
-        msg_retry
-      else
-        default
+    # If you want this method to be run is a scope of multi connection
+    # you need to pass it
+    def self.requeue_job(queue, msg, conn)
+      with_connection(conn) do |conn|
+        conn.lpush(queue, Sidekiq.dump_json(msg))
       end
-    end
 
-    def send_to_morgue(msg)
-      Sidekiq.logger.warn(
-        class: msg['class'],
+      Sidekiq.logger.info(
+        message: "Pushed job #{msg['jid']} back to queue #{queue}",
         jid: msg['jid'],
-        message: %(Reliable Fetcher: adding dead #{msg['class']} job #{msg['jid']})
+        queue: queue
       )
+    end
+
+    # Yield block with an existing connection or creates another one
+    def self.with_connection(conn, &block)
+      return yield(conn) if conn
 
-      payload = Sidekiq.dump_json(msg)
-      Sidekiq::DeadSet.new.kill(payload, notify_failure: false)
+      Sidekiq.redis { |conn| yield(conn) }
     end
 
-    # Detect "old" jobs and requeue them because the worker they were assigned
-    # to probably failed miserably.
-    def clean_working_queues!
-      Sidekiq.logger.info("Cleaning working queues")
+    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
+                :queues, :use_semi_reliable_fetch,
+                :strictly_ordered_queues
 
-      Sidekiq.redis do |conn|
-        conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
-          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
-          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
+    def initialize(options)
+      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
+      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
+      @last_try_to_take_lease_at = 0
+      @strictly_ordered_queues = !!options[:strict]
+      @queues = options[:queues].map { |q| "queue:#{q}" }
+    end
 
-          continue if hostname.nil? || pid.nil?
+    def retrieve_work
+      self.class.clean_working_queues! if take_lease
 
-          clean_working_queue!(key) if worker_dead?(hostname, pid)
-        end
-      end
+      retrieve_unit_of_work
     end
 
-    def worker_dead?(hostname, pid)
-      Sidekiq.redis do |conn|
-        !conn.get(self.class.heartbeat_key(hostname, pid))
-      end
+    def retrieve_unit_of_work
+      raise NotImplementedError,
+        "#{self.class} does not implement #{__method__}"
     end
 
+    private
+
     def take_lease
       return unless allowed_to_take_a_lease?
 

data/lib/sidekiq/interrupted_set.rb

@@ -0,0 +1,47 @@
+require 'sidekiq/api'
+
+module Sidekiq
+  class InterruptedSet < ::Sidekiq::JobSet
+    DEFAULT_MAX_CAPACITY = 10_000
+    DEFAULT_MAX_TIMEOUT = 90 * 24 * 60 * 60 # 3 months
+
+    def initialize
+      super "interrupted"
+    end
+
+    def put(message, opts = {})
+      now = Time.now.to_f
+
+      with_multi_connection(opts[:connection]) do |conn|
+        conn.zadd(name, now.to_s, message)
+        conn.zremrangebyscore(name, '-inf', now - self.class.timeout)
+        conn.zremrangebyrank(name, 0, - self.class.max_jobs)
+      end
+
+      true
+    end
+
+    # Yield block inside an existing multi connection or creates new one
+    def with_multi_connection(conn, &block)
+      return yield(conn) if conn
+
+      Sidekiq.redis do |c|
+        c.multi do |multi|
+          yield(multi)
+        end
+      end
+    end
+
+    def retry_all
+      each(&:retry) while size > 0
+    end
+
+    def self.max_jobs
+      Sidekiq.options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
+    end
+
+    def self.timeout
+      Sidekiq.options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
+    end
+  end
+end

data/spec/base_reliable_fetch_spec.rb

@@ -5,10 +5,11 @@ require 'sidekiq/reliable_fetch'
 require 'sidekiq/semi_reliable_fetch'
 
 describe Sidekiq::BaseReliableFetch do
+  let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
+
   before { Sidekiq.redis(&:flushdb) }
 
   describe 'UnitOfWork' do
-    let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
     let(:fetcher) { Sidekiq::ReliableFetch.new(queues: ['foo']) }
 
     describe '#requeue' do
@@ -39,24 +40,47 @@ describe Sidekiq::BaseReliableFetch do
   end
 
   describe '.bulk_requeue' do
+    let!(:queue1) { Sidekiq::Queue.new('foo') }
+    let!(:queue2) { Sidekiq::Queue.new('bar') }
+
     it 'requeues the bulk' do
-      queue1 = Sidekiq::Queue.new('foo')
-      queue2 = Sidekiq::Queue.new('bar')
+      uow = described_class::UnitOfWork
+      jobs = [ uow.new('queue:foo', job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
+      described_class.bulk_requeue(jobs, queues: [])
 
-      expect(queue1.size).to eq 0
-      expect(queue2.size).to eq 0
+      expect(queue1.size).to eq 2
+      expect(queue2.size).to eq 1
+    end
 
+    it 'puts jobs into interrupted queue' do
       uow = described_class::UnitOfWork
-      jobs = [ uow.new('queue:foo', 'bob'), uow.new('queue:foo', 'bar'), uow.new('queue:bar', 'widget') ]
+      interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
+      jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
+      described_class.bulk_requeue(jobs, queues: [])
+
+      expect(queue1.size).to eq 1
+      expect(queue2.size).to eq 1
+      expect(Sidekiq::InterruptedSet.new.size).to eq 1
+    end
+
+    it 'does not put jobs into interrupted queue if it is disabled' do
+      Sidekiq.options[:max_retries_after_interruption] = -1
+
+      uow = described_class::UnitOfWork
+      interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
+      jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
       described_class.bulk_requeue(jobs, queues: [])
 
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
+      expect(Sidekiq::InterruptedSet.new.size).to eq 0
+
+      Sidekiq.options[:max_retries_after_interruption] = 3
     end
   end
 
   it 'sets heartbeat' do
-    config = double(:sidekiq_config, options: {})
+    config = double(:sidekiq_config, options: { queues: [] })
 
     heartbeat_thread = described_class.setup_reliable_fetch!(config)
 

data/spec/fetch_shared_examples.rb

@@ -4,7 +4,7 @@ shared_examples 'a Sidekiq fetcher' do
   before { Sidekiq.redis(&:flushdb) }
 
   describe '#retrieve_work' do
-    let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
+    let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
     let(:fetcher) { described_class.new(queues: ['assigned']) }
 
     it 'retrieves the job and puts it to working queue' do
@@ -24,13 +24,13 @@ shared_examples 'a Sidekiq fetcher' do
       expect(fetcher.retrieve_work).to be_nil
     end
 
-    it 'requeues jobs from dead working queue with incremented retry_count' do
+    it 'requeues jobs from dead working queue with incremented interrupted_count' do
       Sidekiq.redis do |conn|
         conn.rpush(other_process_working_queue_name('assigned'), job)
       end
 
       expected_job = Sidekiq.load_json(job)
-      expected_job['retry_count'] = 1
+      expected_job['interrupted_count'] = 1
       expected_job = Sidekiq.dump_json(expected_job)
 
       uow = fetcher.retrieve_work
@@ -61,8 +61,7 @@ shared_examples 'a Sidekiq fetcher' do
     it 'does not clean up orphaned jobs more than once per cleanup interval' do
       Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
 
-      expect_any_instance_of(described_class)
-        .to receive(:clean_working_queues!).once
+      expect(described_class).to receive(:clean_working_queues!).once
 
       threads = 10.times.map do
         Thread.new do

data/tests/README.md

@@ -18,15 +18,20 @@ You need to have redis server running on default HTTP port `6379`. To use other
 This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
 number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
 
-# How to run retry tests
+# How to run interruption tests
 
 ```
-cd retry_test
-bundle exec ruby retry_test.rb
+cd tests/interruption
+
+# Verify "KILL" signal
+bundle exec ruby test_kill_signal.rb
+
+# Verify "TERM" signal
+bundle exec ruby test_term_signal.rb
 ```
 
 It requires Redis to be running on 6379 port.
 
 ## How it works
 
-It spawns Sidekiq workers then creates a job that will kill itself after a moment. The  reliable fetcher will bring it back. The purpose is to verify that job is run no more then `retry` parameter says even when job was killed.
+It spawns Sidekiq workers then creates a job that will kill itself after a moment. The  reliable fetcher will bring it back. The purpose is to verify that job is run no more then allowed number of times.

data/tests/interruption/test_kill_signal.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require_relative 'config'
+require_relative '../support/utils'
+
+EXPECTED_NUM_TIMES_BEEN_RUN = 3
+NUM_WORKERS = EXPECTED_NUM_TIMES_BEEN_RUN + 1
+
+Sidekiq.redis(&:flushdb)
+
+pids = spawn_workers(NUM_WORKERS)
+
+RetryTestWorker.perform_async
+
+sleep 300
+
+Sidekiq.redis do |redis|
+  times_has_been_run = redis.get('times_has_been_run').to_i
+  assert 'The job has been run', times_has_been_run, EXPECTED_NUM_TIMES_BEEN_RUN
+end
+
+assert 'Found interruption exhausted jobs', Sidekiq::InterruptedSet.new.size, 1
+
+stop_workers(pids)

data/tests/interruption/test_term_signal.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require_relative 'config'
+require_relative '../support/utils'
+
+EXPECTED_NUM_TIMES_BEEN_RUN = 3
+NUM_WORKERS = EXPECTED_NUM_TIMES_BEEN_RUN + 1
+
+Sidekiq.redis(&:flushdb)
+
+pids = spawn_workers(NUM_WORKERS)
+
+RetryTestWorker.perform_async('TERM', 60)
+
+sleep 300
+
+Sidekiq.redis do |redis|
+  times_has_been_run = redis.get('times_has_been_run').to_i
+  assert 'The job has been run', times_has_been_run, EXPECTED_NUM_TIMES_BEEN_RUN
+end
+
+assert 'Found interruption exhausted jobs', Sidekiq::InterruptedSet.new.size, 1
+
+stop_workers(pids)

data/tests/interruption/worker.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class RetryTestWorker
+  include Sidekiq::Worker
+
+  def perform(signal = 'KILL', wait_seconds = 1)
+    Sidekiq.redis do |redis|
+      redis.incr('times_has_been_run')
+    end
+
+    Process.kill(signal, Process.pid)
+
+    sleep wait_seconds
+  end
+end

data/tests/reliability/worker.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class ReliabilityTestWorker
+  include Sidekiq::Worker
+
+  def perform
+    # To mimic long running job and to increase the probability of losing the job
+    sleep 1
+
+    Sidekiq.redis do |redis|
+      redis.lpush(REDIS_FINISHED_LIST, jid)
+    end
+  end
+end

data/tests/support/utils.rb

@@ -0,0 +1,26 @@
+def assert(text, actual, expected)
+  if actual == expected
+    puts "#{text}: #{actual} (Success)"
+  else
+    puts "#{text}: #{actual} (Failed). Expected: #{expected}"
+    exit 1
+  end
+end
+
+def spawn_workers(number)
+  pids = []
+
+  number.times do
+    pids << spawn('sidekiq -r ./config.rb')
+  end
+
+  pids
+end
+
+# Stop Sidekiq workers
+def stop_workers(pids)
+  pids.each do |pid|
+    Process.kill('KILL', pid)
+    Process.wait pid
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.5.0.pre.alpha
+  version: 0.6.1
 platform: ruby
 authors:
 - TEA
@@ -9,22 +9,28 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-02 00:00:00.000000000 Z
+date: 2020-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.1'
 description: Redis reliable queue pattern implemented in Sidekiq
 email: valery@gitlab.com
 executables: []
@@ -42,6 +48,7 @@ files:
 - gitlab-sidekiq-fetcher.gemspec
 - lib/sidekiq-reliable-fetch.rb
 - lib/sidekiq/base_reliable_fetch.rb
+- lib/sidekiq/interrupted_set.rb
 - lib/sidekiq/reliable_fetch.rb
 - lib/sidekiq/semi_reliable_fetch.rb
 - spec/base_reliable_fetch_spec.rb
@@ -50,13 +57,14 @@ files:
 - spec/semi_reliable_fetch_spec.rb
 - spec/spec_helper.rb
 - tests/README.md
-- tests/reliability_test/config.rb
-- tests/reliability_test/reliability_test.rb
-- tests/reliability_test/worker.rb
-- tests/retry_test/config.rb
-- tests/retry_test/retry_test.rb
-- tests/retry_test/simple_assert.rb
-- tests/retry_test/worker.rb
+- tests/interruption/config.rb
+- tests/interruption/test_kill_signal.rb
+- tests/interruption/test_term_signal.rb
+- tests/interruption/worker.rb
+- tests/reliability/config.rb
+- tests/reliability/reliability_test.rb
+- tests/reliability/worker.rb
+- tests/support/utils.rb
 homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
 - LGPL-3.0
@@ -72,11 +80,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq

data/tests/reliability_test/worker.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class ReliabilityTestWorker
-  include Sidekiq::Worker
-
-  def perform
-    # To mimic long running job and to increase the probability of losing the job
-    sleep 1
-
-    Sidekiq.redis do |redis|
-      redis.lpush(REDIS_FINISHED_LIST, get_sidekiq_job_id)
-    end
-  end
-
-  def get_sidekiq_job_id
-    context_data = Thread.current[:sidekiq_context]&.first
-
-    return unless context_data
-
-    index = context_data.index('JID-')
-
-    return unless index
-
-    context_data[index + 4..-1]
-  end
-end

data/tests/retry_test/retry_test.rb

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-require 'sidekiq'
-require 'sidekiq/util'
-require 'sidekiq/cli'
-require_relative 'config'
-require_relative 'simple_assert'
-
-NUM_WORKERS = RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN + 1
-
-Sidekiq.redis(&:flushdb)
-
-def spawn_workers
-  pids = []
-
-  NUM_WORKERS.times do
-    pids << spawn('sidekiq -r ./config.rb')
-  end
-
-  pids
-end
-
-pids = spawn_workers
-
-jid = RetryTestWorker.perform_async
-
-sleep 300
-
-Sidekiq.redis do |redis|
-  times_has_been_run = redis.get('times_has_been_run').to_i
-  assert "The job has been run", times_has_been_run, 2
-end
-
-assert "Found dead jobs", Sidekiq::DeadSet.new.size, 1
-
-# Stop Sidekiq workers
-pids.each do |pid|
-  Process.kill('KILL', pid)
-  Process.wait pid
-end

data/tests/retry_test/simple_assert.rb

@@ -1,8 +0,0 @@
-def assert(text, actual, expected)
-  if actual == expected
-    puts "#{text}: #{actual} (Success)"
-  else
-    puts "#{text}: #{actual} (Failed). Expected: #{expected}"
-    exit 1
-  end
-end

data/tests/retry_test/worker.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-class RetryTestWorker
-  include Sidekiq::Worker
-
-  EXPECTED_NUM_TIMES_BEEN_RUN = 2
-
-  sidekiq_options retry: EXPECTED_NUM_TIMES_BEEN_RUN
-
-  sidekiq_retry_in do |count, exception|
-    1 # retry in one second
-  end
-
-  def perform
-    sleep 1
-
-    Sidekiq.redis do |redis|
-      redis.incr('times_has_been_run')
-    end
-
-    Process.kill('KILL', Process.pid) # Job suicide, OOM killer imitation
-  end
-end