gitlab-sidekiq-fetcher 0.5.1 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb3707eefa697c806e40fc41b6406714341c5f8dd7115451391313a5f7f01725
-  data.tar.gz: 51046486212181fd92a7cbc2f48f707391ab3e1438ad814b29f2a12c84c2b8cf
+  metadata.gz: efdc9461358b538f2b0692cc179639b802a6f0bf4959b73e1d0821a4c40f0713
+  data.tar.gz: ccbe252394f22e6fafb198ddba9481e23776e7cfa2848807e1e5c34a5269c76d
 SHA512:
-  metadata.gz: b7453b6ff9e45f2d9cc7b8ad3b120c17bb04093214c79967ed6a75698b26fe358bf10ab6424dff95f5cab2b3839ac20e5a2af23a01985f4fe715d2f00c3d086b
-  data.tar.gz: 03afebd011c716cdc96b59b2b18e80514701f6bef3674ab3bdeb568b35c1730c5c8906a3eb40ad9c8aff93b971dd88b538898b326ece66642ecf64f879524088
+  metadata.gz: 010da1750797c367b01cd8e1d8df281fa70759e9035f0ebaea71f53349f3ebbe54bb04dea74694d50bb595938b049733119ffd5ee4c85d5fd42d719e71a07809
+  data.tar.gz: d9f89a5dc1e6d6117df1f3704159ea79160a038d8a7a47939d1c772f07c4f3be2426389cd63d14a0ecf469035cc583ba3ea943cf5a14c2232ee3cb44bf09b21c

data/.gitignore

@@ -1,2 +1,3 @@
 *.gem
 coverage
+.DS_Store

data/.gitlab-ci.yml

@@ -25,7 +25,7 @@ rspec:
 .integration:
   stage: test
   script:
-    - cd tests/reliability_test
+    - cd tests/reliability
     - bundle exec ruby reliability_test.rb
   services:
     - redis:alpine
@@ -40,30 +40,28 @@ integration_reliable:
   variables:
     JOB_FETCHER: reliable
 
-
 integration_basic:
   extends: .integration
   allow_failure: yes
   variables:
     JOB_FETCHER: basic
 
-retry_test:
+kill_interruption:
   stage: test
   script:
-    - cd tests/retry_test
-    - bundle exec ruby retry_test.rb
+    - cd tests/interruption
+    - bundle exec ruby test_kill_signal.rb
   services:
     - redis:alpine
 
-no_retry_test:
+term_interruption:
   stage: test
   script:
-    - cd tests/retry_test
-    - bundle exec ruby no_retry_test.rb
+    - cd tests/interruption
+    - bundle exec ruby test_term_signal.rb
   services:
     - redis:alpine
 
-
 # rubocop:
 #   script:
 #     - bundle exec rubocop

data/Gemfile

@@ -7,6 +7,6 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 group :test do
   gem "rspec", '~> 3'
   gem "pry"
-  gem "sidekiq", '~> 5.0'
+  gem "sidekiq", '~> 6.1'
   gem 'simplecov', require: false
 end

data/Gemfile.lock

@@ -2,7 +2,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     docile (1.3.1)
     json (2.1.0)
@@ -10,10 +10,8 @@ GEM
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rack (2.0.5)
-    rack-protection (2.0.4)
-      rack
-    redis (4.0.2)
+    rack (2.2.3)
+    redis (4.2.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -27,10 +25,10 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    sidekiq (5.2.2)
-      connection_pool (~> 2.2, >= 2.2.2)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
+    sidekiq (6.1.0)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -43,8 +41,8 @@ PLATFORMS
 DEPENDENCIES
   pry
   rspec (~> 3)
-  sidekiq (~> 5.0)
+  sidekiq (~> 6.1)
   simplecov
 
 BUNDLED WITH
-   1.17.1
+   1.17.2

data/README.md

@@ -6,10 +6,23 @@ fetches from Redis.
 
 It's based on https://github.com/TEA-ebook/sidekiq-reliable-fetch.
 
+**IMPORTANT NOTE:** Since version `0.7.0` this gem works only with `sidekiq >= 6.1` (which introduced Fetch API breaking changes). Please use version `~> 0.5` if you use older version of the `sidekiq` .
+
 There are two strategies implemented: [Reliable fetch](http://redis.io/commands/rpoplpush#pattern-reliable-queue) using `rpoplpush` command and
 semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
 However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.
 
+### Interruption handling
+
+Sidekiq expects any job to report succcess or to fail. In the last case, Sidekiq puts `retry_count` counter
+into the job and keeps to re-run the job until the counter reched the maximum allowed value. When the job has
+not been given a chance to finish its work(to report success or fail), for example, when it was killed forcibly or when the job was requeued, after receiving TERM signal, the standard retry mechanisme does not get into the game and the job will be retried indefinatelly. This is why Reliable fetcher maintains a special counter `interrupted_count`
+which is used to limit the amount of such retries. In both cases, Reliable Fetcher increments counter `interrupted_count` and rejects the job from running again when the counter exceeds `max_retries_after_interruption` times (default: 3 times).
+Such a job will be put to `interrupted` queue. This queue mostly behaves as Sidekiq Dead queue so it only stores a limited amount of jobs for a limited term. Same as for Dead queue, all the limits are configurable via `interrupted_max_jobs` (default: 10_000) and `interrupted_timeout_in_seconds` (default: 3 months) Sidekiq option keys.
+
+You can also disable special handling of interrupted jobs by setting `max_retries_after_interruption` into `-1`.
+In this case, interrupted jobs will be run without any limits from Reliable Fetcher and they won't be put into Interrupted queue.
+
 
 ## Installation
 

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,14 +1,14 @@
 Gem::Specification.new do |s|
-  s.name        = 'gitlab-sidekiq-fetcher'
-  s.version     = '0.5.1'
-  s.authors     = ['TEA', 'GitLab']
-  s.email       = 'valery@gitlab.com'
-  s.license     = 'LGPL-3.0'
-  s.homepage    = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
-  s.summary     = 'Reliable fetch extension for Sidekiq'
-  s.description = 'Redis reliable queue pattern implemented in Sidekiq'
+  s.name          = 'gitlab-sidekiq-fetcher'
+  s.version       = '0.7.1'
+  s.authors       = ['TEA', 'GitLab']
+  s.email         = 'valery@gitlab.com'
+  s.license       = 'LGPL-3.0'
+  s.homepage      = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
+  s.summary       = 'Reliable fetch extension for Sidekiq'
+  s.description   = 'Redis reliable queue pattern implemented in Sidekiq'
   s.require_paths = ['lib']
-  s.files = `git ls-files`.split($\)
-  s.test_files  = []
-  s.add_dependency 'sidekiq', '~> 5'
+  s.files         = `git ls-files`.split($\)
+  s.test_files    = []
+  s.add_dependency 'sidekiq', '~> 6.1'
 end

data/lib/sidekiq-reliable-fetch.rb

@@ -1,4 +1,5 @@
 require 'sidekiq'
+require 'sidekiq/api'
 
 require_relative 'sidekiq/base_reliable_fetch'
 require_relative 'sidekiq/reliable_fetch'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'sidekiq/job_retry'
+require_relative 'interrupted_set'
 
 module Sidekiq
   class BaseReliableFetch
@@ -18,6 +18,9 @@ module Sidekiq
     # Defines the COUNT parameter that will be passed to Redis SCAN command
     SCAN_COUNT = 1000
 
+    # How much time a job can be interrupted
+    DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION = 3
+
     UnitOfWork = Struct.new(:queue, :job) do
       def acknowledge
         Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
@@ -38,11 +41,13 @@ module Sidekiq
     end
 
     def self.setup_reliable_fetch!(config)
-      config.options[:fetch] = if config.options[:semi_reliable_fetch]
-                                 Sidekiq::SemiReliableFetch
-                               else
-                                 Sidekiq::ReliableFetch
-                               end
+      fetch_strategy = if config.options[:semi_reliable_fetch]
+                         Sidekiq::SemiReliableFetch
+                       else
+                         Sidekiq::ReliableFetch
+                       end
+
+      config.options[:fetch] = fetch_strategy.new(config.options)
 
       Sidekiq.logger.info('GitLab reliable fetch activated!')
 
@@ -81,23 +86,8 @@ module Sidekiq
       Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
     end
 
-    def self.bulk_requeue(inprogress, _options)
-      return if inprogress.empty?
-
-      Sidekiq.logger.debug('Re-queueing terminated jobs')
-
-      Sidekiq.redis do |conn|
-        inprogress.each do |unit_of_work|
-          conn.multi do |multi|
-            multi.lpush(unit_of_work.queue, unit_of_work.job)
-            multi.lrem(working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
-          end
-        end
-      end
-
-      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
-    rescue => e
-      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
+    def self.worker_dead?(hostname, pid, conn)
+      !conn.get(heartbeat_key(hostname, pid))
     end
 
     def self.heartbeat_key(hostname, pid)
@@ -113,6 +103,8 @@ module Sidekiq
                 :strictly_ordered_queues
 
     def initialize(options)
+      raise ArgumentError, 'missing queue list' unless options[:queues]
+
       @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
       @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
       @last_try_to_take_lease_at = 0
@@ -128,99 +120,116 @@ module Sidekiq
 
     def retrieve_unit_of_work
       raise NotImplementedError,
-        "#{self.class} does not implement #{__method__}"
+            "#{self.class} does not implement #{__method__}"
     end
 
-    private
-
-    def clean_working_queue!(working_queue)
-      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
+    def bulk_requeue(inprogress, _options)
+      return if inprogress.empty?
 
       Sidekiq.redis do |conn|
-        count = 0
-
-        while job = conn.rpop(working_queue)
-          msg = begin
-                  Sidekiq.load_json(job)
-                rescue => e
-                  Sidekiq.logger.info("Skipped job: #{job} as we couldn't parse it")
-                  next
-                end
-
-          msg['retry_count'] = msg['retry_count'].to_i + 1
-
-          if retries_exhausted?(msg)
-            send_to_morgue(msg)
-          else
-            job = Sidekiq.dump_json(msg)
-
-            conn.lpush(original_queue, job)
+        inprogress.each do |unit_of_work|
+          conn.multi do |multi|
+            preprocess_interrupted_job(unit_of_work.job, unit_of_work.queue, multi)
 
-            count += 1
+            multi.lrem(self.class.working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
           end
         end
-
-        Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
       end
+    rescue => e
+      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
     end
 
-    def retries_exhausted?(msg)
-      # `retry` parameter can be empty when job is running the first time and when
-      # it's not specified in worker class explicitly.
-      # In that case, the default parameter gets injected into the job when
-      # it fails the first time in JobRetry#local.
-      # We should handle the case when `retry` is explicitly set to false
-      return true if msg['retry'] === false
-
-      max_retries_default = Sidekiq.options.fetch(:max_retries, Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS)
-
-      max_retry_attempts = retry_attempts_from(msg['retry'], max_retries_default)
+    private
 
-      msg['retry_count'] >= max_retry_attempts
-    end
+    def preprocess_interrupted_job(job, queue, conn = nil)
+      msg = Sidekiq.load_json(job)
+      msg['interrupted_count'] = msg['interrupted_count'].to_i + 1
 
-    def retry_attempts_from(msg_retry, default)
-      if msg_retry.is_a?(Integer)
-        msg_retry
+      if interruption_exhausted?(msg)
+        send_to_quarantine(msg, conn)
       else
-        default
+        requeue_job(queue, msg, conn)
       end
     end
 
-    def send_to_morgue(msg)
-      Sidekiq.logger.warn(
-        class: msg['class'],
+    # If you want this method to be run in a scope of multi connection
+    # you need to pass it
+    def requeue_job(queue, msg, conn)
+      with_connection(conn) do |conn|
+        conn.lpush(queue, Sidekiq.dump_json(msg))
+      end
+
+      Sidekiq.logger.info(
+        message: "Pushed job #{msg['jid']} back to queue #{queue}",
         jid: msg['jid'],
-        message: %(Reliable Fetcher: adding dead #{msg['class']} job #{msg['jid']})
+        queue: queue
       )
-
-      payload = Sidekiq.dump_json(msg)
-      Sidekiq::DeadSet.new.kill(payload, notify_failure: false)
     end
 
     # Detect "old" jobs and requeue them because the worker they were assigned
     # to probably failed miserably.
     def clean_working_queues!
-      Sidekiq.logger.info("Cleaning working queues")
+      Sidekiq.logger.info('Cleaning working queues')
 
       Sidekiq.redis do |conn|
         conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
           # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
           hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
 
-          continue if hostname.nil? || pid.nil?
+          next if hostname.nil? || pid.nil?
 
-          clean_working_queue!(key) if worker_dead?(hostname, pid)
+          clean_working_queue!(key) if self.class.worker_dead?(hostname, pid, conn)
         end
       end
     end
 
-    def worker_dead?(hostname, pid)
+    def clean_working_queue!(working_queue)
+      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
+
       Sidekiq.redis do |conn|
-        !conn.get(self.class.heartbeat_key(hostname, pid))
+        while job = conn.rpop(working_queue)
+          preprocess_interrupted_job(job, original_queue)
+        end
       end
     end
 
+    def interruption_exhausted?(msg)
+      return false if max_retries_after_interruption(msg['class']) < 0
+
+      msg['interrupted_count'].to_i >= max_retries_after_interruption(msg['class'])
+    end
+
+    def max_retries_after_interruption(worker_class)
+      max_retries_after_interruption = nil
+
+      max_retries_after_interruption ||= begin
+        Object.const_get(worker_class).sidekiq_options[:max_retries_after_interruption]
+      rescue NameError
+      end
+
+      max_retries_after_interruption ||= Sidekiq.options[:max_retries_after_interruption]
+      max_retries_after_interruption ||= DEFAULT_MAX_RETRIES_AFTER_INTERRUPTION
+      max_retries_after_interruption
+    end
+
+    def send_to_quarantine(msg, multi_connection = nil)
+      Sidekiq.logger.warn(
+        class: msg['class'],
+        jid: msg['jid'],
+        message: %(Reliable Fetcher: adding dead #{msg['class']} job #{msg['jid']} to interrupted queue)
+      )
+
+      job = Sidekiq.dump_json(msg)
+      Sidekiq::InterruptedSet.new.put(job, connection: multi_connection)
+    end
+
+    # Yield block with an existing connection or creates another one
+    def with_connection(conn)
+      return yield(conn) if conn
+
+      Sidekiq.redis { |redis_conn| yield(redis_conn) }
+    end
+
     def take_lease
       return unless allowed_to_take_a_lease?
 

data/lib/sidekiq/interrupted_set.rb

@@ -0,0 +1,47 @@
+require 'sidekiq/api'
+
+module Sidekiq
+  class InterruptedSet < ::Sidekiq::JobSet
+    DEFAULT_MAX_CAPACITY = 10_000
+    DEFAULT_MAX_TIMEOUT = 90 * 24 * 60 * 60 # 3 months
+
+    def initialize
+      super "interrupted"
+    end
+
+    def put(message, opts = {})
+      now = Time.now.to_f
+
+      with_multi_connection(opts[:connection]) do |conn|
+        conn.zadd(name, now.to_s, message)
+        conn.zremrangebyscore(name, '-inf', now - self.class.timeout)
+        conn.zremrangebyrank(name, 0, - self.class.max_jobs)
+      end
+
+      true
+    end
+
+    # Yield block inside an existing multi connection or creates new one
+    def with_multi_connection(conn, &block)
+      return yield(conn) if conn
+
+      Sidekiq.redis do |c|
+        c.multi do |multi|
+          yield(multi)
+        end
+      end
+    end
+
+    def retry_all
+      each(&:retry) while size > 0
+    end
+
+    def self.max_jobs
+      Sidekiq.options[:interrupted_max_jobs] || DEFAULT_MAX_CAPACITY
+    end
+
+    def self.timeout
+      Sidekiq.options[:interrupted_timeout_in_seconds] || DEFAULT_MAX_TIMEOUT
+    end
+  end
+end

data/lib/sidekiq/reliable_fetch.rb

@@ -6,23 +6,21 @@ module Sidekiq
     # we inject a regular sleep into the loop.
     RELIABLE_FETCH_IDLE_TIMEOUT = 5 # seconds
 
-    attr_reader :queues_iterator, :queues_size
+    attr_reader :queues_size
 
     def initialize(options)
       super
 
+      @queues = queues.uniq if strictly_ordered_queues
       @queues_size = queues.size
-      @queues_iterator = queues.cycle
     end
 
     private
 
     def retrieve_unit_of_work
-      @queues_iterator.rewind if strictly_ordered_queues
-
-      queues_size.times do
-        queue = queues_iterator.next
+      queues_list = strictly_ordered_queues ? queues : queues.shuffle
 
+      queues_list.each do |queue|
         work = Sidekiq.redis do |conn|
           conn.rpoplpush(queue, self.class.working_queue_name(queue))
         end

data/spec/base_reliable_fetch_spec.rb

@@ -5,10 +5,11 @@ require 'sidekiq/reliable_fetch'
 require 'sidekiq/semi_reliable_fetch'
 
 describe Sidekiq::BaseReliableFetch do
+  let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
+
   before { Sidekiq.redis(&:flushdb) }
 
   describe 'UnitOfWork' do
-    let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
     let(:fetcher) { Sidekiq::ReliableFetch.new(queues: ['foo']) }
 
     describe '#requeue' do
@@ -38,25 +39,49 @@ describe Sidekiq::BaseReliableFetch do
     end
   end
 
-  describe '.bulk_requeue' do
+  describe '#bulk_requeue' do
+    let(:options) { { queues: %w[foo bar] } }
+    let!(:queue1) { Sidekiq::Queue.new('foo') }
+    let!(:queue2) { Sidekiq::Queue.new('bar') }
+
     it 'requeues the bulk' do
-      queue1 = Sidekiq::Queue.new('foo')
-      queue2 = Sidekiq::Queue.new('bar')
+      uow = described_class::UnitOfWork
+      jobs = [ uow.new('queue:foo', job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
+      described_class.new(options).bulk_requeue(jobs, nil)
 
-      expect(queue1.size).to eq 0
-      expect(queue2.size).to eq 0
+      expect(queue1.size).to eq 2
+      expect(queue2.size).to eq 1
+    end
 
+    it 'puts jobs into interrupted queue' do
       uow = described_class::UnitOfWork
-      jobs = [ uow.new('queue:foo', 'bob'), uow.new('queue:foo', 'bar'), uow.new('queue:bar', 'widget') ]
-      described_class.bulk_requeue(jobs, queues: [])
+      interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
+      jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
+      described_class.new(options).bulk_requeue(jobs, nil)
+
+      expect(queue1.size).to eq 1
+      expect(queue2.size).to eq 1
+      expect(Sidekiq::InterruptedSet.new.size).to eq 1
+    end
+
+    it 'does not put jobs into interrupted queue if it is disabled' do
+      Sidekiq.options[:max_retries_after_interruption] = -1
+
+      uow = described_class::UnitOfWork
+      interrupted_job = Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo'], interrupted_count: 3)
+      jobs = [ uow.new('queue:foo', interrupted_job), uow.new('queue:foo', job), uow.new('queue:bar', job) ]
+      described_class.new(options).bulk_requeue(jobs, nil)
 
       expect(queue1.size).to eq 2
       expect(queue2.size).to eq 1
+      expect(Sidekiq::InterruptedSet.new.size).to eq 0
+
+      Sidekiq.options[:max_retries_after_interruption] = 3
     end
   end
 
   it 'sets heartbeat' do
-    config = double(:sidekiq_config, options: {})
+    config = double(:sidekiq_config, options: { queues: %w[foo bar] })
 
     heartbeat_thread = described_class.setup_reliable_fetch!(config)
 

data/spec/fetch_shared_examples.rb

@@ -4,8 +4,8 @@ shared_examples 'a Sidekiq fetcher' do
   before { Sidekiq.redis(&:flushdb) }
 
   describe '#retrieve_work' do
-    let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
-    let(:fetcher) { described_class.new(queues: ['assigned']) }
+    let(:job) { Sidekiq.dump_json(class: 'Bob', args: [1, 2, 'foo']) }
+    let(:fetcher) { described_class.new(queues: queues) }
 
     it 'retrieves the job and puts it to working queue' do
       Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
@@ -24,13 +24,13 @@ shared_examples 'a Sidekiq fetcher' do
       expect(fetcher.retrieve_work).to be_nil
     end
 
-    it 'requeues jobs from dead working queue with incremented retry_count' do
+    it 'requeues jobs from dead working queue with incremented interrupted_count' do
       Sidekiq.redis do |conn|
         conn.rpush(other_process_working_queue_name('assigned'), job)
       end
 
       expected_job = Sidekiq.load_json(job)
-      expected_job['retry_count'] = 1
+      expected_job['interrupted_count'] = 1
       expected_job = Sidekiq.dump_json(expected_job)
 
       uow = fetcher.retrieve_work
@@ -42,6 +42,22 @@ shared_examples 'a Sidekiq fetcher' do
       end
     end
 
+    it 'ignores working queue keys in unknown formats' do
+      # Add a spurious non-numeric char segment at the end; this simulates any other
+      # incorrect form in general
+      malformed_key = "#{other_process_working_queue_name('assigned')}:X"
+      Sidekiq.redis do |conn|
+        conn.rpush(malformed_key, job)
+      end
+
+      uow = fetcher.retrieve_work
+
+      Sidekiq.redis do |conn|
+        expect(conn.llen(malformed_key)).to eq 1
+      end
+    end
+
+
     it 'does not requeue jobs from live working queue' do
       working_queue = live_other_process_working_queue_name('assigned')
 
@@ -61,12 +77,11 @@ shared_examples 'a Sidekiq fetcher' do
     it 'does not clean up orphaned jobs more than once per cleanup interval' do
       Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
 
-      expect_any_instance_of(described_class)
-        .to receive(:clean_working_queues!).once
+      expect(fetcher).to receive(:clean_working_queues!).once
 
       threads = 10.times.map do
         Thread.new do
-          described_class.new(queues: ['assigned']).retrieve_work
+          fetcher.retrieve_work
         end
       end
 

data/tests/README.md

@@ -18,18 +18,20 @@ You need to have redis server running on default HTTP port `6379`. To use other
 This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
 number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
 
-# How to run retry tests
+# How to run interruption tests
 
 ```
-cd retry_test
-bundle exec ruby retry_test.rb
+cd tests/interruption
 
-# To verify that workers with "retry: false" are not retried
-bundle exec ruby no_retry_test.rb
+# Verify "KILL" signal
+bundle exec ruby test_kill_signal.rb
+
+# Verify "TERM" signal
+bundle exec ruby test_term_signal.rb
 ```
 
 It requires Redis to be running on 6379 port.
 
 ## How it works
 
-It spawns Sidekiq workers then creates a job that will kill itself after a moment. The  reliable fetcher will bring it back. The purpose is to verify that job is run no more then `retry` parameter says even when job was killed.
+It spawns Sidekiq workers then creates a job that will kill itself after a moment. The  reliable fetcher will bring it back. The purpose is to verify that job is run no more then allowed number of times.

data/tests/{retry_test → interruption}/config.rb

@@ -2,7 +2,6 @@
 
 require_relative '../../lib/sidekiq-reliable-fetch'
 require_relative 'worker'
-require_relative 'no_retry_worker'
 
 TEST_CLEANUP_INTERVAL = 20
 TEST_LEASE_INTERVAL = 5

data/tests/{retry_test/no_retry_test.rb → interruption/test_kill_signal.rb}

@@ -4,21 +4,22 @@ require 'sidekiq'
 require_relative 'config'
 require_relative '../support/utils'
 
-NUM_WORKERS = 2 # one worker will be killed and one spare worker t verify that job is not picked up
+EXPECTED_NUM_TIMES_BEEN_RUN = 3
+NUM_WORKERS = EXPECTED_NUM_TIMES_BEEN_RUN + 1
 
 Sidekiq.redis(&:flushdb)
 
 pids = spawn_workers(NUM_WORKERS)
 
-jid = NoRetryTestWorker.perform_async
+RetryTestWorker.perform_async
 
 sleep 300
 
 Sidekiq.redis do |redis|
   times_has_been_run = redis.get('times_has_been_run').to_i
-  assert 'The job has been run', times_has_been_run, 1
+  assert 'The job has been run', times_has_been_run, EXPECTED_NUM_TIMES_BEEN_RUN
 end
 
-assert 'Found dead jobs', Sidekiq::DeadSet.new.size, 1
+assert 'Found interruption exhausted jobs', Sidekiq::InterruptedSet.new.size, 1
 
 stop_workers(pids)

data/tests/{retry_test/retry_test.rb → interruption/test_term_signal.rb}

@@ -4,21 +4,22 @@ require 'sidekiq'
 require_relative 'config'
 require_relative '../support/utils'
 
-NUM_WORKERS = RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN + 1
+EXPECTED_NUM_TIMES_BEEN_RUN = 3
+NUM_WORKERS = EXPECTED_NUM_TIMES_BEEN_RUN + 1
 
 Sidekiq.redis(&:flushdb)
 
 pids = spawn_workers(NUM_WORKERS)
 
-jid = RetryTestWorker.perform_async
+RetryTestWorker.perform_async('TERM', 60)
 
 sleep 300
 
 Sidekiq.redis do |redis|
   times_has_been_run = redis.get('times_has_been_run').to_i
-  assert 'The job has been run', times_has_been_run, RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN
+  assert 'The job has been run', times_has_been_run, EXPECTED_NUM_TIMES_BEEN_RUN
 end
 
-assert 'Found dead jobs', Sidekiq::DeadSet.new.size, 1
+assert 'Found interruption exhausted jobs', Sidekiq::InterruptedSet.new.size, 1
 
 stop_workers(pids)

data/tests/interruption/worker.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class RetryTestWorker
+  include Sidekiq::Worker
+
+  def perform(signal = 'KILL', wait_seconds = 1)
+    Sidekiq.redis do |redis|
+      redis.incr('times_has_been_run')
+    end
+
+    Process.kill(signal, Process.pid)
+
+    sleep wait_seconds
+  end
+end

data/tests/{reliability_test → reliability}/reliability_test.rb

@@ -57,7 +57,7 @@ end
 def spawn_workers
   pids = []
   NUMBER_OF_WORKERS.times do
-    pids << spawn('sidekiq -r ./config.rb')
+    pids << spawn('sidekiq -q default -q low -q high -r ./config.rb')
   end
 
   pids

data/tests/reliability/worker.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class ReliabilityTestWorker
+  include Sidekiq::Worker
+
+  def perform
+    # To mimic long running job and to increase the probability of losing the job
+    sleep 1
+
+    Sidekiq.redis do |redis|
+      redis.lpush(REDIS_FINISHED_LIST, jid)
+    end
+  end
+end

data/tests/support/utils.rb

@@ -11,7 +11,7 @@ def spawn_workers(number)
   pids = []
 
   number.times do
-    pids << spawn('sidekiq -r ./config.rb')
+    pids << spawn('sidekiq -q default -q high -q low -r ./config.rb')
   end
 
   pids

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-sidekiq-fetcher
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.7.1
 platform: ruby
 authors:
 - TEA
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-06 00:00:00.000000000 Z
+date: 2021-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '6.1'
 description: Redis reliable queue pattern implemented in Sidekiq
 email: valery@gitlab.com
 executables: []
@@ -42,6 +42,7 @@ files:
 - gitlab-sidekiq-fetcher.gemspec
 - lib/sidekiq-reliable-fetch.rb
 - lib/sidekiq/base_reliable_fetch.rb
+- lib/sidekiq/interrupted_set.rb
 - lib/sidekiq/reliable_fetch.rb
 - lib/sidekiq/semi_reliable_fetch.rb
 - spec/base_reliable_fetch_spec.rb
@@ -50,14 +51,13 @@ files:
 - spec/semi_reliable_fetch_spec.rb
 - spec/spec_helper.rb
 - tests/README.md
-- tests/reliability_test/config.rb
-- tests/reliability_test/reliability_test.rb
-- tests/reliability_test/worker.rb
-- tests/retry_test/config.rb
-- tests/retry_test/no_retry_test.rb
-- tests/retry_test/no_retry_worker.rb
-- tests/retry_test/retry_test.rb
-- tests/retry_test/worker.rb
+- tests/interruption/config.rb
+- tests/interruption/test_kill_signal.rb
+- tests/interruption/test_term_signal.rb
+- tests/interruption/worker.rb
+- tests/reliability/config.rb
+- tests/reliability/reliability_test.rb
+- tests/reliability/worker.rb
 - tests/support/utils.rb
 homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
 licenses:
@@ -78,7 +78,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Reliable fetch extension for Sidekiq

data/tests/reliability_test/worker.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class ReliabilityTestWorker
-  include Sidekiq::Worker
-
-  def perform
-    # To mimic long running job and to increase the probability of losing the job
-    sleep 1
-
-    Sidekiq.redis do |redis|
-      redis.lpush(REDIS_FINISHED_LIST, get_sidekiq_job_id)
-    end
-  end
-
-  def get_sidekiq_job_id
-    context_data = Thread.current[:sidekiq_context]&.first
-
-    return unless context_data
-
-    index = context_data.index('JID-')
-
-    return unless index
-
-    context_data[index + 4..-1]
-  end
-end

data/tests/retry_test/no_retry_worker.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-class NoRetryTestWorker
-  include Sidekiq::Worker
-
-  sidekiq_options retry: false
-
-  sidekiq_retry_in do |count, exception|
-    1 # retry in one second
-  end
-
-  def perform
-    sleep 1
-
-    Sidekiq.redis do |redis|
-      redis.incr('times_has_been_run')
-    end
-
-    Process.kill('KILL', Process.pid) # Job suicide, OOM killer imitation
-  end
-end

data/tests/retry_test/worker.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-class RetryTestWorker
-  include Sidekiq::Worker
-
-  EXPECTED_NUM_TIMES_BEEN_RUN = 2
-
-  sidekiq_options retry: EXPECTED_NUM_TIMES_BEEN_RUN
-
-  sidekiq_retry_in do |count, exception|
-    1 # retry in one second
-  end
-
-  def perform
-    sleep 1
-
-    Sidekiq.redis do |redis|
-      redis.incr('times_has_been_run')
-    end
-
-    Process.kill('KILL', Process.pid) # Job suicide, OOM killer imitation
-  end
-end