gitlab-sidekiq-fetcher 0.1.0 → 0.3.0

data/spec/reliable_fetch_spec.rb

@@ -1,7 +0,0 @@
-require 'spec_helper'
-require 'fetch_shared_examples'
-require 'sidekiq/reliable_fetch'
-
-describe Sidekiq::ReliableFetch do
-  include_examples 'a Sidekiq fetcher'
-end

data/spec/semi_reliable_fetch_spec.rb

@@ -1,7 +0,0 @@
-require 'spec_helper'
-require 'fetch_shared_examples'
-require 'sidekiq/semi_reliable_fetch'
-
-describe Sidekiq::SemiReliableFetch do
-  include_examples 'a Sidekiq fetcher'
-end

data/spec/spec_helper.rb

@@ -1,115 +0,0 @@
-require 'sidekiq'
-require 'sidekiq/util'
-require 'sidekiq/api'
-require 'pry'
-require 'simplecov'
-
-SimpleCov.start
-
-REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost:6379/10'
-
-Sidekiq.configure_client do |config|
-  config.redis = { url: REDIS_URL }
-end
-
-Sidekiq.logger.level = Logger::ERROR
-# This file was generated by the `rspec --init` command. Conventionally, all
-# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
-# The generated `.rspec` file contains `--require spec_helper` which will cause
-# this file to always be loaded, without a need to explicitly require it in any
-# files.
-#
-# Given that it is always loaded, you are encouraged to keep this file as
-# light-weight as possible. Requiring heavyweight dependencies from this file
-# will add to the boot time of your test suite on EVERY test run, even for an
-# individual file that may not need all of that loaded. Instead, consider making
-# a separate helper file that requires the additional dependencies and performs
-# the additional setup, and require it from the spec files that actually need
-# it.
-#
-# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
-RSpec.configure do |config|
-  # rspec-expectations config goes here. You can use an alternate
-  # assertion/expectation library such as wrong or the stdlib/minitest
-  # assertions if you prefer.
-  config.expect_with :rspec do |expectations|
-    # This option will default to `true` in RSpec 4. It makes the `description`
-    # and `failure_message` of custom matchers include text for helper methods
-    # defined using `chain`, e.g.:
-    #     be_bigger_than(2).and_smaller_than(4).description
-    #     # => "be bigger than 2 and smaller than 4"
-    # ...rather than:
-    #     # => "be bigger than 2"
-    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
-  end
-
-  # rspec-mocks config goes here. You can use an alternate test double
-  # library (such as bogus or mocha) by changing the `mock_with` option here.
-  config.mock_with :rspec do |mocks|
-    # Prevents you from mocking or stubbing a method that does not exist on
-    # a real object. This is generally recommended, and will default to
-    # `true` in RSpec 4.
-    mocks.verify_partial_doubles = true
-  end
-
-  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
-  # have no way to turn it off -- the option exists only for backwards
-  # compatibility in RSpec 3). It causes shared context metadata to be
-  # inherited by the metadata hash of host groups and examples, rather than
-  # triggering implicit auto-inclusion in groups with matching metadata.
-  config.shared_context_metadata_behavior = :apply_to_host_groups
-
-# The settings below are suggested to provide a good initial experience
-# with RSpec, but feel free to customize to your heart's content.
-=begin
-  # This allows you to limit a spec run to individual examples or groups
-  # you care about by tagging them with `:focus` metadata. When nothing
-  # is tagged with `:focus`, all examples get run. RSpec also provides
-  # aliases for `it`, `describe`, and `context` that include `:focus`
-  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
-  config.filter_run_when_matching :focus
-
-  # Allows RSpec to persist some state between runs in order to support
-  # the `--only-failures` and `--next-failure` CLI options. We recommend
-  # you configure your source control system to ignore this file.
-  config.example_status_persistence_file_path = "spec/examples.txt"
-
-  # Limits the available syntax to the non-monkey patched syntax that is
-  # recommended. For more details, see:
-  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
-  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
-  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
-  config.disable_monkey_patching!
-
-  # This setting enables warnings. It's recommended, but in some cases may
-  # be too noisy due to issues in dependencies.
-  config.warnings = true
-
-  # Many RSpec users commonly either run the entire suite or an individual
-  # file, and it's useful to allow more verbose output when running an
-  # individual spec file.
-  if config.files_to_run.one?
-    # Use the documentation formatter for detailed output,
-    # unless a formatter has already been configured
-    # (e.g. via a command-line flag).
-    config.default_formatter = "doc"
-  end
-
-  # Print the 10 slowest examples and example groups at the
-  # end of the spec run, to help surface which specs are running
-  # particularly slow.
-  config.profile_examples = 10
-
-  # Run specs in random order to surface order dependencies. If you find an
-  # order dependency and want to debug it, you can fix the order by providing
-  # the seed, which is printed after each run.
-  #     --seed 1234
-  config.order = :random
-
-  # Seed global randomization in this process using the `--seed` CLI option.
-  # Setting this allows you to use `--seed` to deterministically reproduce
-  # test failures related to randomization by passing the same `--seed` value
-  # as the one that triggered the failure.
-  Kernel.srand config.seed
-=end
-end

data/test/README.md

@@ -1,34 +0,0 @@
-# How to run
-
-```
-cd test
-bundle exec ruby reliability_test.rb
-```
-
-You can adjust some parameters of the test in the `config.rb`
-
-
-# How it works
-
-This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
-number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
-
-# How to run tests
-
-To run rspec:
-
-```
-bundle exec rspec
-```
-
-To run performance tests:
-
-```
-cd test
-JOB_FETCHER=semi bundle exec ruby reliability_test.rb
-```
-
-JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
-
-To run both kind of tests you need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
-`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).

data/test/config.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../lib/sidekiq/base_reliable_fetch'
-require_relative '../lib/sidekiq/reliable_fetch'
-require_relative '../lib/sidekiq/semi_reliable_fetch'
-require_relative 'worker'
-
-REDIS_FINISHED_LIST = 'reliable-fetcher-finished-jids'
-
-NUMBER_OF_WORKERS = ENV['NUMBER_OF_WORKERS'] || 10
-NUMBER_OF_JOBS = ENV['NUMBER_OF_JOBS'] || 1000
-JOB_FETCHER = (ENV['JOB_FETCHER'] || :semi).to_sym # :basic, :semi, :reliable
-TEST_CLEANUP_INTERVAL = 20
-TEST_LEASE_INTERVAL = 5
-WAIT_CLEANUP = TEST_CLEANUP_INTERVAL +
-               TEST_LEASE_INTERVAL +
-               Sidekiq::ReliableFetch::HEARTBEAT_LIFESPAN
-
-Sidekiq.configure_server do |config|
-  if %i[semi reliable].include?(JOB_FETCHER)
-    config.options[:semi_reliable_fetch] = (JOB_FETCHER == :semi)
-
-    # We need to override these parameters to not wait too long
-    # The default values are good for production use only
-    # These will be ignored for :basic
-    config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
-    config.options[:lease_interval] = TEST_LEASE_INTERVAL
-
-    Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
-  end
-end

data/test/reliability_test.rb

@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-require 'sidekiq'
-require 'sidekiq/util'
-require 'sidekiq/cli'
-require_relative 'config'
-
-def spawn_workers_and_stop_them_on_a_half_way
-  pids = spawn_workers
-
-  wait_until do |queue_size|
-    queue_size < NUMBER_OF_JOBS / 2
-  end
-
-  first_half_pids, second_half_pids = split_array(pids)
-
-  puts 'Killing half of the workers...'
-  signal_to_workers('KILL', first_half_pids)
-
-  puts 'Stopping another half of the workers...'
-  signal_to_workers('TERM', second_half_pids)
-end
-
-def spawn_workers_and_let_them_finish
-  puts 'Spawn workers and let them finish...'
-
-  pids = spawn_workers
-
-  wait_until do |queue_size|
-    queue_size.zero?
-  end
-
-  if %i[semi reliable].include? JOB_FETCHER
-    puts 'Waiting for clean up process that will requeue dead jobs...'
-    sleep WAIT_CLEANUP
-  end
-
-  signal_to_workers('TERM', pids)
-end
-
-def wait_until
-  loop do
-    sleep 3
-
-    queue_size = current_queue_size
-    puts "Jobs in the queue:#{queue_size}"
-
-    break if yield(queue_size)
-  end
-end
-
-def signal_to_workers(signal, pids)
-  pids.each { |pid| Process.kill(signal, pid) }
-  pids.each { |pid| Process.wait(pid) }
-end
-
-def spawn_workers
-  pids = []
-  NUMBER_OF_WORKERS.times do
-    pids << spawn('sidekiq -r ./config.rb')
-  end
-
-  pids
-end
-
-def current_queue_size
-  Sidekiq.redis { |c| c.llen('queue:default') }
-end
-
-def duplicates
-  Sidekiq.redis { |c| c.llen(REDIS_FINISHED_LIST) }
-end
-
-# Splits array into two halves
-def split_array(arr)
-  first_arr = arr.take(arr.size / 2)
-  second_arr = arr - first_arr
-  [first_arr, second_arr]
-end
-
-##########################################################
-
-puts '########################################'
-puts "Mode: #{JOB_FETCHER}"
-puts '########################################'
-
-Sidekiq.redis(&:flushdb)
-
-jobs = []
-
-NUMBER_OF_JOBS.times do
-  jobs << TestWorker.perform_async
-end
-
-puts "Queued #{NUMBER_OF_JOBS} jobs"
-
-spawn_workers_and_stop_them_on_a_half_way
-spawn_workers_and_let_them_finish
-
-jobs_lost = 0
-
-Sidekiq.redis do |redis|
-  jobs.each do |job|
-    next if redis.lrem(REDIS_FINISHED_LIST, 1, job) == 1
-    jobs_lost += 1
-  end
-end
-
-puts "Remaining unprocessed: #{jobs_lost}"
-puts "Duplicates found: #{duplicates}"
-
-if jobs_lost.zero? && duplicates.zero?
-  exit 0
-else
-  exit 1
-end

data/test/worker.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-class TestWorker
-  include Sidekiq::Worker
-
-  def perform
-    # To mimic long running job and to increase the probability of losing the job
-    sleep 1
-
-    Sidekiq.redis do |redis|
-      redis.lpush(REDIS_FINISHED_LIST, get_sidekiq_job_id)
-    end
-  end
-
-  def get_sidekiq_job_id
-    context_data = Thread.current[:sidekiq_context]&.first
-
-    return unless context_data
-
-    index = context_data.index('JID-')
-
-    return unless index
-
-    context_data[index + 4..-1]
-  end
-end