gitlab-sidekiq-fetcher 0.3.0 → 0.4.0

data/spec/base_reliable_fetch_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+require 'fetch_shared_examples'
+require 'sidekiq/base_reliable_fetch'
+require 'sidekiq/reliable_fetch'
+require 'sidekiq/semi_reliable_fetch'
+
+describe Sidekiq::BaseReliableFetch do
+  before { Sidekiq.redis(&:flushdb) }
+
+  describe 'UnitOfWork' do
+    let(:fetcher) { Sidekiq::ReliableFetch.new(queues: ['foo']) }
+
+    describe '#requeue' do
+      it 'requeues job' do
+        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
+
+        uow = fetcher.retrieve_work
+
+        uow.requeue
+
+        expect(Sidekiq::Queue.new('foo').size).to eq 1
+        expect(working_queue_size('foo')).to eq 0
+      end
+    end
+
+    describe '#acknowledge' do
+      it 'acknowledges job' do
+        Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
+
+        uow = fetcher.retrieve_work
+
+        expect { uow.acknowledge }
+          .to change { working_queue_size('foo') }.by(-1)
+
+        expect(Sidekiq::Queue.new('foo').size).to eq 0
+      end
+    end
+  end
+
+  describe '.bulk_requeue' do
+    it 'requeues the bulk' do
+      queue1 = Sidekiq::Queue.new('foo')
+      queue2 = Sidekiq::Queue.new('bar')
+
+      expect(queue1.size).to eq 0
+      expect(queue2.size).to eq 0
+
+      uow = described_class::UnitOfWork
+      jobs = [ uow.new('queue:foo', 'bob'), uow.new('queue:foo', 'bar'), uow.new('queue:bar', 'widget') ]
+      described_class.bulk_requeue(jobs, queues: [])
+
+      expect(queue1.size).to eq 2
+      expect(queue2.size).to eq 1
+    end
+  end
+
+  it 'sets heartbeat' do
+    config = double(:sidekiq_config, options: {})
+
+    heartbeat_thread = described_class.setup_reliable_fetch!(config)
+
+    Sidekiq.redis do |conn|
+      sleep 0.2 # Give the time to heartbeat thread to make a loop
+
+      heartbeat_key = described_class.heartbeat_key(Socket.gethostname, ::Process.pid)
+      heartbeat = conn.get(heartbeat_key)
+
+      expect(heartbeat).not_to be_nil
+    end
+
+    heartbeat_thread.kill
+  end
+end

data/spec/fetch_shared_examples.rb

@@ -0,0 +1,118 @@
+shared_examples 'a Sidekiq fetcher' do
+  let(:queues) { ['assigned'] }
+
+  before { Sidekiq.redis(&:flushdb) }
+
+  describe '#retrieve_work' do
+    let(:fetcher) { described_class.new(queues: ['assigned']) }
+
+    it 'retrieves the job and puts it to working queue' do
+      Sidekiq.redis { |conn| conn.rpush('queue:assigned', 'msg') }
+
+      uow = fetcher.retrieve_work
+
+      expect(working_queue_size('assigned')).to eq 1
+      expect(uow.queue_name).to eq 'assigned'
+      expect(uow.job).to eq 'msg'
+      expect(Sidekiq::Queue.new('assigned').size).to eq 0
+    end
+
+    it 'does not retrieve a job from foreign queue' do
+      Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', 'msg') }
+
+      expect(fetcher.retrieve_work).to be_nil
+    end
+
+    it 'requeues jobs from dead working queue' do
+      Sidekiq.redis do |conn|
+        conn.rpush(other_process_working_queue_name('assigned'), 'msg')
+      end
+
+      uow = fetcher.retrieve_work
+
+      expect(uow.job).to eq 'msg'
+
+      Sidekiq.redis do |conn|
+        expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
+      end
+    end
+
+    it 'does not requeue jobs from live working queue' do
+      working_queue = live_other_process_working_queue_name('assigned')
+
+      Sidekiq.redis do |conn|
+        conn.rpush(working_queue, 'msg')
+      end
+
+      uow = fetcher.retrieve_work
+
+      expect(uow).to be_nil
+
+      Sidekiq.redis do |conn|
+        expect(conn.llen(working_queue)).to eq 1
+      end
+    end
+
+    it 'does not clean up orphaned jobs more than once per cleanup interval' do
+      Sidekiq.redis = Sidekiq::RedisConnection.create(url: REDIS_URL, size: 10)
+
+      expect_any_instance_of(described_class)
+        .to receive(:clean_working_queues!).once
+
+      threads = 10.times.map do
+        Thread.new do
+          described_class.new(queues: ['assigned']).retrieve_work
+        end
+      end
+
+      threads.map(&:join)
+    end
+
+    it 'retrieves by order when strictly order is enabled' do
+      fetcher = described_class.new(strict: true, queues: ['first', 'second'])
+
+      Sidekiq.redis do |conn|
+        conn.rpush('queue:first', ['msg3', 'msg2', 'msg1'])
+        conn.rpush('queue:second', 'msg4')
+      end
+
+      jobs = (1..4).map { fetcher.retrieve_work.job }
+
+      expect(jobs).to eq ['msg1', 'msg2', 'msg3', 'msg4']
+    end
+
+    it 'does not starve any queue when queues are not strictly ordered' do
+      fetcher = described_class.new(queues: ['first', 'second'])
+
+      Sidekiq.redis do |conn|
+        conn.rpush('queue:first', (1..200).map { |i| "msg#{i}" })
+        conn.rpush('queue:second', 'this_job_should_not_stuck')
+      end
+
+      jobs = (1..100).map { fetcher.retrieve_work.job }
+
+      expect(jobs).to include 'this_job_should_not_stuck'
+    end
+  end
+end
+
+def working_queue_size(queue_name)
+  Sidekiq.redis do |c|
+    c.llen(Sidekiq::BaseReliableFetch.working_queue_name("queue:#{queue_name}"))
+  end
+end
+
+def other_process_working_queue_name(queue)
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{Socket.gethostname}:#{::Process.pid + 1}"
+end
+
+def live_other_process_working_queue_name(queue)
+  pid = ::Process.pid + 1
+  hostname = Socket.gethostname
+
+  Sidekiq.redis do |conn|
+    conn.set(Sidekiq::BaseReliableFetch.heartbeat_key(hostname, pid), 1)
+  end
+
+  "#{Sidekiq::BaseReliableFetch::WORKING_QUEUE_PREFIX}:queue:#{queue}:#{hostname}:#{pid}"
+end

data/spec/reliable_fetch_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+require 'fetch_shared_examples'
+require 'sidekiq/reliable_fetch'
+
+describe Sidekiq::ReliableFetch do
+  include_examples 'a Sidekiq fetcher'
+end

data/spec/semi_reliable_fetch_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+require 'fetch_shared_examples'
+require 'sidekiq/semi_reliable_fetch'
+
+describe Sidekiq::SemiReliableFetch do
+  include_examples 'a Sidekiq fetcher'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,115 @@
+require 'sidekiq'
+require 'sidekiq/util'
+require 'sidekiq/api'
+require 'pry'
+require 'simplecov'
+
+SimpleCov.start
+
+REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost:6379/10'
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: REDIS_URL }
+end
+
+Sidekiq.logger.level = Logger::ERROR
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

data/test/README.md

@@ -0,0 +1,34 @@
+# How to run
+
+```
+cd test
+bundle exec ruby reliability_test.rb
+```
+
+You can adjust some parameters of the test in the `config.rb`
+
+
+# How it works
+
+This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
+number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
+
+# How to run tests
+
+To run rspec:
+
+```
+bundle exec rspec
+```
+
+To run performance tests:
+
+```
+cd test
+JOB_FETCHER=semi bundle exec ruby reliability_test.rb
+```
+
+JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
+
+To run both kind of tests you need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
+`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).

data/test/config.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative '../lib/sidekiq/base_reliable_fetch'
+require_relative '../lib/sidekiq/reliable_fetch'
+require_relative '../lib/sidekiq/semi_reliable_fetch'
+require_relative 'worker'
+
+REDIS_FINISHED_LIST = 'reliable-fetcher-finished-jids'
+
+NUMBER_OF_WORKERS = ENV['NUMBER_OF_WORKERS'] || 10
+NUMBER_OF_JOBS = ENV['NUMBER_OF_JOBS'] || 1000
+JOB_FETCHER = (ENV['JOB_FETCHER'] || :semi).to_sym # :basic, :semi, :reliable
+TEST_CLEANUP_INTERVAL = 20
+TEST_LEASE_INTERVAL = 5
+WAIT_CLEANUP = TEST_CLEANUP_INTERVAL +
+               TEST_LEASE_INTERVAL +
+               Sidekiq::ReliableFetch::HEARTBEAT_LIFESPAN
+
+Sidekiq.configure_server do |config|
+  if %i[semi reliable].include?(JOB_FETCHER)
+    config.options[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
+
+    # We need to override these parameters to not wait too long
+    # The default values are good for production use only
+    # These will be ignored for :basic
+    config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
+    config.options[:lease_interval] = TEST_LEASE_INTERVAL
+
+    Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+  end
+end

data/test/reliability_test.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require 'sidekiq/util'
+require 'sidekiq/cli'
+require_relative 'config'
+
+def spawn_workers_and_stop_them_on_a_half_way
+  pids = spawn_workers
+
+  wait_until do |queue_size|
+    queue_size < NUMBER_OF_JOBS / 2
+  end
+
+  first_half_pids, second_half_pids = split_array(pids)
+
+  puts 'Killing half of the workers...'
+  signal_to_workers('KILL', first_half_pids)
+
+  puts 'Stopping another half of the workers...'
+  signal_to_workers('TERM', second_half_pids)
+end
+
+def spawn_workers_and_let_them_finish
+  puts 'Spawn workers and let them finish...'
+
+  pids = spawn_workers
+
+  wait_until do |queue_size|
+    queue_size.zero?
+  end
+
+  if %i[semi reliable].include? JOB_FETCHER
+    puts 'Waiting for clean up process that will requeue dead jobs...'
+    sleep WAIT_CLEANUP
+  end
+
+  signal_to_workers('TERM', pids)
+end
+
+def wait_until
+  loop do
+    sleep 3
+
+    queue_size = current_queue_size
+    puts "Jobs in the queue:#{queue_size}"
+
+    break if yield(queue_size)
+  end
+end
+
+def signal_to_workers(signal, pids)
+  pids.each { |pid| Process.kill(signal, pid) }
+  pids.each { |pid| Process.wait(pid) }
+end
+
+def spawn_workers
+  pids = []
+  NUMBER_OF_WORKERS.times do
+    pids << spawn('sidekiq -r ./config.rb')
+  end
+
+  pids
+end
+
+def current_queue_size
+  Sidekiq.redis { |c| c.llen('queue:default') }
+end
+
+def duplicates
+  Sidekiq.redis { |c| c.llen(REDIS_FINISHED_LIST) }
+end
+
+# Splits array into two halves
+def split_array(arr)
+  first_arr = arr.take(arr.size / 2)
+  second_arr = arr - first_arr
+  [first_arr, second_arr]
+end
+
+##########################################################
+
+puts '########################################'
+puts "Mode: #{JOB_FETCHER}"
+puts '########################################'
+
+Sidekiq.redis(&:flushdb)
+
+jobs = []
+
+NUMBER_OF_JOBS.times do
+  jobs << TestWorker.perform_async
+end
+
+puts "Queued #{NUMBER_OF_JOBS} jobs"
+
+spawn_workers_and_stop_them_on_a_half_way
+spawn_workers_and_let_them_finish
+
+jobs_lost = 0
+
+Sidekiq.redis do |redis|
+  jobs.each do |job|
+    next if redis.lrem(REDIS_FINISHED_LIST, 1, job) == 1
+    jobs_lost += 1
+  end
+end
+
+puts "Remaining unprocessed: #{jobs_lost}"
+puts "Duplicates found: #{duplicates}"
+
+if jobs_lost.zero? && duplicates.zero?
+  exit 0
+else
+  exit 1
+end