gitlab-sidekiq-fetcher 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 00056b1430ab40c06094813fcf06296ef24eef40
-  data.tar.gz: 1ee1138aeaed7c353e2eaddab6982d7a1d384d82
+SHA256:
+  metadata.gz: cc2e47cf7679deb6a6d526e199a09c20a671e3b55bad22d7c05ce17405eb6103
+  data.tar.gz: e71949587df8a635223ca8fa36339949df771f493e7edda0a4d9c34198600fb5
 SHA512:
-  metadata.gz: 5c141c0c16201c5b788268b3d829cae2b1820682fb907f56f61e75aa205c223d83c519c544ac3bf0d0c8f7dd5758c3f754e93bbfe184e92b80c2bd9e134b06a3
-  data.tar.gz: d1cef7c3751a5b6dd04155a21d4ef9742bb274552eb66ab41cf2044cbcc94f21ca494aac7ee3e6db01fd9891e259e8bb2e52bc26c5d3299dd72b84f5088fe0f0
+  metadata.gz: ea7d6b7283354053a4f9fc24f419ab56da97efff3ce3ac3a2a10517ab2a1bd184a44b0ffb9d39b1dbdf27b23032c58b1139a18d8f2ec1bf5357deed65187e3ed
+  data.tar.gz: ae4c78eca271dc63abf98bc112a582a2f56fc4e1e7be843189c30193ff0dabd99232f5a019b9b69d45cd1f08ca7bd88afe0c4341e0256c88d352441474a08400

data/.gitignore

@@ -1,2 +1,2 @@
-Gemfile.lock
 *.gem
+coverage

data/.gitlab-ci.yml

@@ -0,0 +1,53 @@
+image: "ruby:2.5"
+
+before_script:
+  - ruby -v
+  - which ruby
+  - gem install bundler --no-ri --no-rdoc
+  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+
+variables:
+  REDIS_URL: "redis://redis"
+
+rspec:
+  stage: test
+  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
+  script:
+    - bundle exec rspec
+  services:
+    - redis:alpine
+  artifacts:
+    expire_in: 31d
+    when: always
+    paths:
+      - coverage/
+
+.integration:
+  stage: test
+  script:
+    - cd test
+    - bundle exec ruby reliability_test.rb
+  services:
+    - redis:alpine
+
+integration_semi:
+  extends: .integration
+  variables:
+    JOB_FETCHER: semi
+
+integration_reliable:
+  extends: .integration
+  variables:
+    JOB_FETCHER: reliable
+
+
+integration_basic:
+  extends: .integration
+  allow_failure: yes
+  variables:
+    JOB_FETCHER: basic
+
+
+# rubocop:
+#   script:
+#     - bundle exec rubocop

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+group :test do
+  gem "rspec", '~> 3'
+  gem "pry"
+  gem "sidekiq", '~> 5.0'
+  gem 'simplecov', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,50 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    connection_pool (2.2.2)
+    diff-lcs (1.3)
+    docile (1.3.1)
+    json (2.1.0)
+    method_source (0.9.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rack (2.0.5)
+    rack-protection (2.0.4)
+      rack
+    redis (4.0.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    sidekiq (5.2.2)
+      connection_pool (~> 2.2, >= 2.2.2)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.5, < 5)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry
+  rspec (~> 3)
+  sidekiq (~> 5.0)
+  simplecov
+
+BUNDLED WITH
+   1.17.1

data/README.md

@@ -5,10 +5,11 @@ gitlab-sidekiq-fetcher
 fetches from Redis.
 
 It's based on https://github.com/TEA-ebook/sidekiq-reliable-fetch.
-At this time we only added Sidekiq 5+ support to it.
 
-It implements in Sidekiq the reliable queue pattern using [Redis' rpoplpush
-command](http://redis.io/commands/rpoplpush#pattern-reliable-queue).
+There are two strategies implemented: [Reliable fetch](http://redis.io/commands/rpoplpush#pattern-reliable-queue) using `rpoplpush` command and
+semi-reliable fetch that uses regular `brpop` and `lpush` to pick the job and put it to working queue. The main benefit of "Reliable" strategy is that `rpoplpush` is atomic, eliminating a race condition in which jobs can be lost.
+However, it comes at a cost because `rpoplpush` can't watch multiple lists at the same time so we need to iterate over the entire queue list which significantly increases pressure on Redis when there are more than a few queues. The "semi-reliable" strategy is much more reliable than the default Sidekiq fetcher, though. Compared to the reliable fetch strategy, it does not increase pressure on Redis significantly.
+
 
 ## Installation
 
@@ -24,12 +25,22 @@ Enable reliable fetches by calling this gem from your Sidekiq configuration:
 
 ```ruby
 Sidekiq.configure_server do |config|
-  Sidekiq::ReliableFetcher.setup_reliable_fetch!(config)
+  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
 
   # …
 end
 ```
 
+There is an additional parameter `config.options[:semi_reliable_fetch]` you can use to switch between two strategies:
+
+```ruby
+Sidekiq.configure_server do |config|
+  config.options[:semi_reliable_fetch] = true # Default value is false
+
+  Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+end
+```
+
 ## License
 
 LGPL-3.0, see the LICENSE file.

data/gitlab-sidekiq-fetcher.gemspec

@@ -1,16 +1,14 @@
 Gem::Specification.new do |s|
   s.name        = 'gitlab-sidekiq-fetcher'
-  s.version     = '0.3.0'
+  s.version     = '0.4.0'
   s.authors     = ['TEA', 'GitLab']
   s.email       = 'valery@gitlab.com'
   s.license     = 'LGPL-3.0'
-  s.homepage    = 'https://github.com/TEA-ebook/sidekiq-reliable-fetch'
+  s.homepage    = 'https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/'
   s.summary     = 'Reliable fetch extension for Sidekiq'
   s.description = 'Redis reliable queue pattern implemented in Sidekiq'
   s.require_paths = ['lib']
-
   s.files = `git ls-files`.split($\)
   s.test_files  = []
-
   s.add_dependency 'sidekiq', '~> 5'
 end

data/lib/sidekiq-reliable-fetch.rb

@@ -1,4 +1,5 @@
 require 'sidekiq'
 
-require_relative 'sidekiq/reliable_fetcher'
-require_relative 'sidekiq-reliable-fetch/web'
+require_relative 'sidekiq/base_reliable_fetch'
+require_relative 'sidekiq/reliable_fetch'
+require_relative 'sidekiq/semi_reliable_fetch'

data/lib/sidekiq/base_reliable_fetch.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class BaseReliableFetch
+    DEFAULT_CLEANUP_INTERVAL = 60 * 60  # 1 hour
+    HEARTBEAT_INTERVAL       = 20       # seconds
+    HEARTBEAT_LIFESPAN       = 60       # seconds
+    HEARTBEAT_RETRY_DELAY    = 1        # seconds
+    WORKING_QUEUE_PREFIX     = 'working'
+
+    # Defines how often we try to take a lease to not flood our
+    # Redis server with SET requests
+    DEFAULT_LEASE_INTERVAL = 2 * 60 # seconds
+    LEASE_KEY              = 'reliable-fetcher-cleanup-lock'
+
+    # Defines the COUNT parameter that will be passed to Redis SCAN command
+    SCAN_COUNT = 1000
+
+    UnitOfWork = Struct.new(:queue, :job) do
+      def acknowledge
+        Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
+      end
+
+      def queue_name
+        queue.sub(/.*queue:/, '')
+      end
+
+      def requeue
+        Sidekiq.redis do |conn|
+          conn.multi do |multi|
+            multi.lpush(queue, job)
+            multi.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job)
+          end
+        end
+      end
+    end
+
+    def self.setup_reliable_fetch!(config)
+      config.options[:fetch] = if config.options[:semi_reliable_fetch]
+                                 Sidekiq::SemiReliableFetch
+                               else
+                                 Sidekiq::ReliableFetch
+                               end
+
+      Sidekiq.logger.info('GitLab reliable fetch activated!')
+
+      start_heartbeat_thread
+    end
+
+    def self.start_heartbeat_thread
+      Thread.new do
+        loop do
+          begin
+            heartbeat
+
+            sleep HEARTBEAT_INTERVAL
+          rescue => e
+            Sidekiq.logger.error("Heartbeat thread error: #{e.message}")
+
+            sleep HEARTBEAT_RETRY_DELAY
+          end
+        end
+      end
+    end
+
+    def self.pid
+      @pid ||= ::Process.pid
+    end
+
+    def self.hostname
+      @hostname ||= Socket.gethostname
+    end
+
+    def self.heartbeat
+      Sidekiq.redis do |conn|
+        conn.set(heartbeat_key(hostname, pid), 1, ex: HEARTBEAT_LIFESPAN)
+      end
+
+      Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
+    end
+
+    def self.bulk_requeue(inprogress, _options)
+      return if inprogress.empty?
+
+      Sidekiq.logger.debug('Re-queueing terminated jobs')
+
+      Sidekiq.redis do |conn|
+        inprogress.each do |unit_of_work|
+          conn.multi do |multi|
+            multi.lpush(unit_of_work.queue, unit_of_work.job)
+            multi.lrem(working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
+          end
+        end
+      end
+
+      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
+    rescue => e
+      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
+    end
+
+    def self.heartbeat_key(hostname, pid)
+      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
+    end
+
+    def self.working_queue_name(queue)
+      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
+    end
+
+    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
+                :queues, :use_semi_reliable_fetch,
+                :strictly_ordered_queues
+
+    def initialize(options)
+      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
+      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
+      @last_try_to_take_lease_at = 0
+      @strictly_ordered_queues = !!options[:strict]
+      @queues = options[:queues].map { |q| "queue:#{q}" }
+    end
+
+    def retrieve_work
+      clean_working_queues! if take_lease
+
+      retrieve_unit_of_work
+    end
+
+    def retrieve_unit_of_work
+      raise NotImplementedError,
+        "#{self.class} does not implement #{__method__}"
+    end
+
+    private
+
+    def clean_working_queue!(working_queue)
+      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
+
+      Sidekiq.redis do |conn|
+        count = 0
+
+        while conn.rpoplpush(working_queue, original_queue) do
+          count += 1
+        end
+
+        Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
+      end
+    end
+
+    # Detect "old" jobs and requeue them because the worker they were assigned
+    # to probably failed miserably.
+    def clean_working_queues!
+      Sidekiq.logger.info("Cleaning working queues")
+
+      Sidekiq.redis do |conn|
+        conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
+          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
+          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
+
+          continue if hostname.nil? || pid.nil?
+
+          clean_working_queue!(key) if worker_dead?(hostname, pid)
+        end
+      end
+    end
+
+    def worker_dead?(hostname, pid)
+      Sidekiq.redis do |conn|
+        !conn.get(self.class.heartbeat_key(hostname, pid))
+      end
+    end
+
+    def take_lease
+      return unless allowed_to_take_a_lease?
+
+      @last_try_to_take_lease_at = Time.now.to_f
+
+      Sidekiq.redis do |conn|
+        conn.set(LEASE_KEY, 1, nx: true, ex: cleanup_interval)
+      end
+    end
+
+    def allowed_to_take_a_lease?
+      Time.now.to_f - last_try_to_take_lease_at > lease_interval
+    end
+  end
+end

data/lib/sidekiq/reliable_fetch.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class ReliableFetch < BaseReliableFetch
+    # For reliable fetch we don't use Redis' blocking operations so
+    # we inject a regular sleep into the loop.
+    RELIABLE_FETCH_IDLE_TIMEOUT = 5 # seconds
+
+    attr_reader :queues_iterator, :queues_size
+
+    def initialize(options)
+      super
+
+      @queues_size = queues.size
+      @queues_iterator = queues.cycle
+    end
+
+    private
+
+    def retrieve_unit_of_work
+      @queues_iterator.rewind if strictly_ordered_queues
+
+      queues_size.times do
+        queue = queues_iterator.next
+
+        work = Sidekiq.redis do |conn|
+          conn.rpoplpush(queue, self.class.working_queue_name(queue))
+        end
+
+        return UnitOfWork.new(queue, work) if work
+      end
+
+      # We didn't find a job in any of the configured queues. Let's sleep a bit
+      # to avoid uselessly burning too much CPU
+      sleep(RELIABLE_FETCH_IDLE_TIMEOUT)
+
+      nil
+    end
+  end
+end

data/lib/sidekiq/semi_reliable_fetch.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  class SemiReliableFetch < BaseReliableFetch
+    # We want the fetch operation to timeout every few seconds so the thread
+    # can check if the process is shutting down. This constant is only used
+    # for semi-reliable fetch.
+    SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
+
+    def initialize(options)
+      super
+
+      if strictly_ordered_queues
+        @queues = @queues.uniq
+        @queues << SEMI_RELIABLE_FETCH_TIMEOUT
+      end
+    end
+
+    private
+
+    def retrieve_unit_of_work
+      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
+      return unless work
+
+      unit_of_work = UnitOfWork.new(*work)
+
+      Sidekiq.redis do |conn|
+        conn.lpush(self.class.working_queue_name(unit_of_work.queue), unit_of_work.job)
+      end
+
+      unit_of_work
+    end
+
+    def queues_cmd
+      if strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle.uniq
+        queues << SEMI_RELIABLE_FETCH_TIMEOUT
+        queues
+      end
+    end
+  end
+end