gitlab-sidekiq-fetcher 0.1.0 → 0.3.0

data/.rspec

@@ -1 +0,0 @@
---require spec_helper

data/Gemfile

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-source "https://rubygems.org"
-
-git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
-
-group :test do
-  gem "rspec", '~> 3'
-  gem "pry"
-  gem "sidekiq", '~> 5.0'
-  gem 'simplecov', require: false
-end

data/Gemfile.lock

@@ -1,50 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    coderay (1.1.2)
-    connection_pool (2.2.2)
-    diff-lcs (1.3)
-    docile (1.3.1)
-    json (2.1.0)
-    method_source (0.9.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
-    rack (2.0.5)
-    rack-protection (2.0.4)
-      rack
-    redis (4.0.2)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    sidekiq (5.2.2)
-      connection_pool (~> 2.2, >= 2.2.2)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
-    simplecov (0.16.1)
-      docile (~> 1.1)
-      json (>= 1.8, < 3)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.2)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  pry
-  rspec (~> 3)
-  sidekiq (~> 5.0)
-  simplecov
-
-BUNDLED WITH
-   1.17.1

data/lib/sidekiq/base_reliable_fetch.rb

@@ -1,185 +0,0 @@
-# frozen_string_literal: true
-
-module Sidekiq
-  class BaseReliableFetch
-    DEFAULT_CLEANUP_INTERVAL = 60 * 60  # 1 hour
-    HEARTBEAT_INTERVAL       = 20       # seconds
-    HEARTBEAT_LIFESPAN       = 60       # seconds
-    HEARTBEAT_RETRY_DELAY    = 1        # seconds
-    WORKING_QUEUE_PREFIX     = 'working'
-
-    # Defines how often we try to take a lease to not flood our
-    # Redis server with SET requests
-    DEFAULT_LEASE_INTERVAL = 2 * 60 # seconds
-    LEASE_KEY              = 'reliable-fetcher-cleanup-lock'
-
-    # Defines the COUNT parameter that will be passed to Redis SCAN command
-    SCAN_COUNT = 1000
-
-    UnitOfWork = Struct.new(:queue, :job) do
-      def acknowledge
-        Sidekiq.redis { |conn| conn.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job) }
-      end
-
-      def queue_name
-        queue.sub(/.*queue:/, '')
-      end
-
-      def requeue
-        Sidekiq.redis do |conn|
-          conn.multi do |multi|
-            multi.lpush(queue, job)
-            multi.lrem(Sidekiq::BaseReliableFetch.working_queue_name(queue), 1, job)
-          end
-        end
-      end
-    end
-
-    def self.setup_reliable_fetch!(config)
-      config.options[:fetch] = if config.options[:semi_reliable_fetch]
-                                 Sidekiq::SemiReliableFetch
-                               else
-                                 Sidekiq::ReliableFetch
-                               end
-
-      Sidekiq.logger.info('GitLab reliable fetch activated!')
-
-      start_heartbeat_thread
-    end
-
-    def self.start_heartbeat_thread
-      Thread.new do
-        loop do
-          begin
-            heartbeat
-
-            sleep HEARTBEAT_INTERVAL
-          rescue => e
-            Sidekiq.logger.error("Heartbeat thread error: #{e.message}")
-
-            sleep HEARTBEAT_RETRY_DELAY
-          end
-        end
-      end
-    end
-
-    def self.pid
-      @pid ||= ::Process.pid
-    end
-
-    def self.hostname
-      @hostname ||= Socket.gethostname
-    end
-
-    def self.heartbeat
-      Sidekiq.redis do |conn|
-        conn.set(heartbeat_key(hostname, pid), 1, ex: HEARTBEAT_LIFESPAN)
-      end
-
-      Sidekiq.logger.debug("Heartbeat for hostname: #{hostname} and pid: #{pid}")
-    end
-
-    def self.bulk_requeue(inprogress, _options)
-      return if inprogress.empty?
-
-      Sidekiq.logger.debug('Re-queueing terminated jobs')
-
-      Sidekiq.redis do |conn|
-        inprogress.each do |unit_of_work|
-          conn.multi do |multi|
-            multi.lpush(unit_of_work.queue, unit_of_work.job)
-            multi.lrem(working_queue_name(unit_of_work.queue), 1, unit_of_work.job)
-          end
-        end
-      end
-
-      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
-    rescue => e
-      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{e.message}")
-    end
-
-    def self.heartbeat_key(hostname, pid)
-      "reliable-fetcher-heartbeat-#{hostname}-#{pid}"
-    end
-
-    def self.working_queue_name(queue)
-      "#{WORKING_QUEUE_PREFIX}:#{queue}:#{hostname}:#{pid}"
-    end
-
-    attr_reader :cleanup_interval, :last_try_to_take_lease_at, :lease_interval,
-                :queues, :use_semi_reliable_fetch,
-                :strictly_ordered_queues
-
-    def initialize(options)
-      @cleanup_interval = options.fetch(:cleanup_interval, DEFAULT_CLEANUP_INTERVAL)
-      @lease_interval = options.fetch(:lease_interval, DEFAULT_LEASE_INTERVAL)
-      @last_try_to_take_lease_at = 0
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
-    end
-
-    def retrieve_work
-      clean_working_queues! if take_lease
-
-      retrieve_unit_of_work
-    end
-
-    def retrieve_unit_of_work
-      raise NotImplementedError,
-        "#{self.class} does not implement #{__method__}"
-    end
-
-    private
-
-    def clean_working_queue!(working_queue)
-      original_queue = working_queue.gsub(/#{WORKING_QUEUE_PREFIX}:|:[^:]*:[0-9]*\z/, '')
-
-      Sidekiq.redis do |conn|
-        count = 0
-
-        while conn.rpoplpush(working_queue, original_queue) do
-          count += 1
-        end
-
-        Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
-      end
-    end
-
-    # Detect "old" jobs and requeue them because the worker they were assigned
-    # to probably failed miserably.
-    def clean_working_queues!
-      Sidekiq.logger.info("Cleaning working queues")
-
-      Sidekiq.redis do |conn|
-        conn.scan_each(match: "#{WORKING_QUEUE_PREFIX}:queue:*", count: SCAN_COUNT) do |key|
-          # Example: "working:name_of_the_job:queue:{hostname}:{PID}"
-          hostname, pid = key.scan(/:([^:]*):([0-9]*)\z/).flatten
-
-          continue if hostname.nil? || pid.nil?
-
-          clean_working_queue!(key) if worker_dead?(hostname, pid)
-        end
-      end
-    end
-
-    def worker_dead?(hostname, pid)
-      Sidekiq.redis do |conn|
-        !conn.get(self.class.heartbeat_key(hostname, pid))
-      end
-    end
-
-    def take_lease
-      return unless allowed_to_take_a_lease?
-
-      @last_try_to_take_lease_at = Time.now.to_f
-
-      Sidekiq.redis do |conn|
-        conn.set(LEASE_KEY, 1, nx: true, ex: cleanup_interval)
-      end
-    end
-
-    def allowed_to_take_a_lease?
-      Time.now.to_f - last_try_to_take_lease_at > lease_interval
-    end
-  end
-end

data/lib/sidekiq/reliable_fetch.rb

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module Sidekiq
-  class ReliableFetch < BaseReliableFetch
-    # For reliable fetch we don't use Redis' blocking operations so
-    # we inject a regular sleep into the loop.
-    RELIABLE_FETCH_IDLE_TIMEOUT = 5 # seconds
-
-    attr_reader :queues_iterator, :queues_size
-
-    def initialize(options)
-      super
-
-      @queues_size = queues.size
-      @queues_iterator = queues.cycle
-    end
-
-    private
-
-    def retrieve_unit_of_work
-      @queues_iterator.rewind if strictly_ordered_queues
-
-      queues_size.times do
-        queue = queues_iterator.next
-
-        work = Sidekiq.redis do |conn|
-          conn.rpoplpush(queue, self.class.working_queue_name(queue))
-        end
-
-        return UnitOfWork.new(queue, work) if work
-      end
-
-      # We didn't find a job in any of the configured queues. Let's sleep a bit
-      # to avoid uselessly burning too much CPU
-      sleep(RELIABLE_FETCH_IDLE_TIMEOUT)
-
-      nil
-    end
-  end
-end

data/lib/sidekiq/semi_reliable_fetch.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Sidekiq
-  class SemiReliableFetch < BaseReliableFetch
-    # We want the fetch operation to timeout every few seconds so the thread
-    # can check if the process is shutting down. This constant is only used
-    # for semi-reliable fetch.
-    SEMI_RELIABLE_FETCH_TIMEOUT = 2 # seconds
-
-    def initialize(options)
-      super
-
-      if strictly_ordered_queues
-        @queues = @queues.uniq
-        @queues << SEMI_RELIABLE_FETCH_TIMEOUT
-      end
-    end
-
-    private
-
-    def retrieve_unit_of_work
-      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
-      return unless work
-
-      unit_of_work = UnitOfWork.new(*work)
-
-      Sidekiq.redis do |conn|
-        conn.lpush(self.class.working_queue_name(unit_of_work.queue), unit_of_work.job)
-      end
-
-      unit_of_work
-    end
-
-    def queues_cmd
-      if strictly_ordered_queues
-        @queues
-      else
-        queues = @queues.shuffle.uniq
-        queues << SEMI_RELIABLE_FETCH_TIMEOUT
-        queues
-      end
-    end
-  end
-end

data/spec/base_reliable_fetch_spec.rb

@@ -1,73 +0,0 @@
-require 'spec_helper'
-require 'fetch_shared_examples'
-require 'sidekiq/base_reliable_fetch'
-require 'sidekiq/reliable_fetch'
-require 'sidekiq/semi_reliable_fetch'
-
-describe Sidekiq::BaseReliableFetch do
-  before { Sidekiq.redis(&:flushdb) }
-
-  describe 'UnitOfWork' do
-    let(:fetcher) { Sidekiq::ReliableFetch.new(queues: ['foo']) }
-
-    describe '#requeue' do
-      it 'requeues job' do
-        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
-
-        uow = fetcher.retrieve_work
-
-        uow.requeue
-
-        expect(Sidekiq::Queue.new('foo').size).to eq 1
-        expect(working_queue_size('foo')).to eq 0
-      end
-    end
-
-    describe '#acknowledge' do
-      it 'acknowledges job' do
-        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
-
-        uow = fetcher.retrieve_work
-
-        expect { uow.acknowledge }
-          .to change { working_queue_size('foo') }.by(-1)
-
-        expect(Sidekiq::Queue.new('foo').size).to eq 0
-      end
-    end
-  end
-
-  describe '.bulk_requeue' do
-    it 'requeues the bulk' do
-      queue1 = Sidekiq::Queue.new('foo')
-      queue2 = Sidekiq::Queue.new('bar')
-
-      expect(queue1.size).to eq 0
-      expect(queue2.size).to eq 0
-
-      uow = described_class::UnitOfWork
-      jobs = [ uow.new('queue:foo', 'bob'), uow.new('queue:foo', 'bar'), uow.new('queue:bar', 'widget') ]
-      described_class.bulk_requeue(jobs, queues: [])
-
-      expect(queue1.size).to eq 2
-      expect(queue2.size).to eq 1
-    end
-  end
-
-  it 'sets heartbeat' do
-    config = double(:sidekiq_config, options: {})
-
-    heartbeat_thread = described_class.setup_reliable_fetch!(config)
-
-    Sidekiq.redis do |conn|
-      sleep 0.2 # Give the time to heartbeat thread to make a loop
-
-      heartbeat_key = described_class.heartbeat_key(Socket.gethostname, ::Process.pid)
-      heartbeat = conn.get(heartbeat_key)
-
-      expect(heartbeat).not_to be_nil
-    end
-
-    heartbeat_thread.kill
-  end
-end

data/spec/fetch_shared_examples.rb

@@ -1,118 +0,0 @@
-shared_examples 'a Sidekiq fetcher' do
-  let(:queues) { ['assigned'] }
-
-  before { Sidekiq.redis(&:flushdb) }
-
-  describe '#retrieve_work' do
-    let(:fetcher) { described_class.new(queues: ['assigned']) }
-
-    it 'retrieves the job and puts it to working queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:assigned', 'msg') }
-
-      uow = fetcher.retrieve_work
-
-      expect(working_queue_size('assigned')).to eq 1
-      expect(uow.queue_name).to eq 'assigned'
-      expect(uow.job).to eq 'msg'
-      expect(Sidekiq::Queue.new('assigned').size).to eq 0
-    end
-
-    it 'does not retrieve a job from foreign queue' do
-      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', 'msg') }
-
-      expect(fetcher.retrieve_work).to be_nil
-    end
-
-    it 'requeues jobs from dead working queue' do
-      Sidekiq.redis do |conn|
-        conn.rpush(other_process_working_queue_name('assigned'), 'msg')
-      end
-
-      uow = fetcher.retrieve_work
-
-      expect(uow.job).to eq 'msg'
-
-      Sidekiq.redis do |conn|
-        expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
-      end
-    end
-
-    it 'does not requeue jobs from live working queue' do
-      working_queue = live_other_process_working_queue_name('assigned')
-
-      Sidekiq.redis do |conn|
-        conn.rpush(working_queue, 'msg')
-      end
-
-      uow = fetcher.retrieve_work
-
-      expect(uow).to be_nil
-
-      Sidekiq.redis do |conn|
-        expect(conn.llen(working_queue)).to eq 1
-      end
-    end
-
-    it 'does not clean up orphaned jobs more than once per cleanup interval' do
-      Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
-
-      expect_any_instance_of(described_class)
-        .to receive(:clean_working_queues!).once
-
-      threads = 10.times.map do
-        Thread.new do
-          described_class.new(queues: ['assigned']).retrieve_work
-        end
-      end
-
-      threads.map(&:join)
-    end
-
-    it 'retrieves by order when strictly order is enabled' do
-      fetcher = described_class.new(strict: true, queues: ['first', 'second'])
-
-      Sidekiq.redis do |conn|
-        conn.rpush('queue:first', ['msg3', 'msg2', 'msg1'])
-        conn.rpush('queue:second', 'msg4')
-      end
-
-      jobs = (1..4).map { fetcher.retrieve_work.job }
-
-      expect(jobs).to eq ['msg1', 'msg2', 'msg3', 'msg4']
-    end
-
-    it 'does not starve any queue when queues are not strictly ordered' do
-      fetcher = described_class.new(queues: ['first', 'second'])
-
-      Sidekiq.redis do |conn|
-        conn.rpush('queue:first', (1..200).map { |i| "msg#{i}" })
-        conn.rpush('queue:second', 'this_job_should_not_stuck')
-      end
-
-      jobs = (1..100).map { fetcher.retrieve_work.job }
-
-      expect(jobs).to include 'this_job_should_not_stuck'
-    end
-  end
-end
-
-def working_queue_size(queue_name)
-  Sidekiq.redis do |c|
-    c.llen(Sidekiq::BaseReliableFetch.working_queue_name("queue:#{queue_name}"))
-  end
-end
-
-def other_process_working_queue_name(queue)
-  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
-end
-
-def live_other_process_working_queue_name(queue)
-  pid = ::Process.pid + 1
-  hostname = Socket.gethostname
-
-  Sidekiq.redis do |conn|
-    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key(hostname, pid), 1)
-  end
-
-  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}"
-end