gitlab-sidekiq-fetcher 0.4.0 → 0.5.0.pre.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cc2e47cf7679deb6a6d526e199a09c20a671e3b55bad22d7c05ce17405eb6103
4
- data.tar.gz: e71949587df8a635223ca8fa36339949df771f493e7edda0a4d9c34198600fb5
3
+ metadata.gz: 7da36ba54ba6a0e97cef5da210f02bdde86dc06e22960d37573e51e20046dd40
4
+ data.tar.gz: bd3bf13edc789374109a2c18b6797b44c1440bc2665938c654e3a627bbc5bc23
5
5
  SHA512:
6
- metadata.gz: ea7d6b7283354053a4f9fc24f419ab56da97efff3ce3ac3a2a10517ab2a1bd184a44b0ffb9d39b1dbdf27b23032c58b1139a18d8f2ec1bf5357deed65187e3ed
7
- data.tar.gz: ae4c78eca271dc63abf98bc112a582a2f56fc4e1e7be843189c30193ff0dabd99232f5a019b9b69d45cd1f08ca7bd88afe0c4341e0256c88d352441474a08400
6
+ metadata.gz: 852f01191a052384cfe4215766c6b27dc932ced77782a92c23f73aba091b520e83d293d57b407e45ed5d141661664a9f77b88211b50e9bed52cb15a011b0347a
7
+ data.tar.gz: 59ee3493c9ddcb3e145ce77259b85ed508f8e57ca97878c12f5427e25952d2a4418860cd1d90562d39aa693dd630359e88ac78fd882aaceb9c5c374b6ca7509a
data/.gitlab-ci.yml CHANGED
@@ -3,7 +3,7 @@ image: "ruby:2.5"
3
3
  before_script:
4
4
  - ruby -v
5
5
  - which ruby
6
- - gem install bundler --no-ri --no-rdoc
6
+ - gem install bundler
7
7
  - bundle install --jobs $(nproc) "${FLAGS[@]}"
8
8
 
9
9
  variables:
@@ -25,7 +25,7 @@ rspec:
25
25
  .integration:
26
26
  stage: test
27
27
  script:
28
- - cd test
28
+ - cd tests/reliability_test
29
29
  - bundle exec ruby reliability_test.rb
30
30
  services:
31
31
  - redis:alpine
@@ -47,6 +47,14 @@ integration_basic:
47
47
  variables:
48
48
  JOB_FETCHER: basic
49
49
 
50
+ retry_test:
51
+ stage: test
52
+ script:
53
+ - cd tests/retry_test
54
+ - bundle exec ruby retry_test.rb
55
+ services:
56
+ - redis:alpine
57
+
50
58
 
51
59
  # rubocop:
52
60
  # script:
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'gitlab-sidekiq-fetcher'
3
- s.version = '0.4.0'
3
+ s.version = '0.5.0-alpha'
4
4
  s.authors = ['TEA', 'GitLab']
5
5
  s.email = 'valery@gitlab.com'
6
6
  s.license = 'LGPL-3.0'
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'sidekiq/job_retry'
4
+
3
5
  module Sidekiq
4
6
  class BaseReliableFetch
5
7
  DEFAULT_CLEANUP_INTERVAL = 60 * 60 # 1 hour
@@ -137,14 +139,58 @@ module Sidekiq
137
139
  Sidekiq.redis do |conn|
138
140
  count = 0
139
141
 
140
- while conn.rpoplpush(working_queue, original_queue) do
141
- count += 1
142
+ while job = conn.rpop(working_queue)
143
+ msg = begin
144
+ Sidekiq.load_json(job)
145
+ rescue => e
146
+ Sidekiq.logger.info("Skipped job: #{job} as we couldn't parse it")
147
+ next
148
+ end
149
+
150
+ msg['retry_count'] = msg['retry_count'].to_i + 1
151
+
152
+ if retries_exhausted?(msg)
153
+ send_to_morgue(msg)
154
+ else
155
+ job = Sidekiq.dump_json(msg)
156
+
157
+ conn.lpush(original_queue, job)
158
+
159
+ count += 1
160
+ end
142
161
  end
143
162
 
144
163
  Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
145
164
  end
146
165
  end
147
166
 
167
+ def retries_exhausted?(msg)
168
+ max_retries_default = Sidekiq.options.fetch(:max_retries, Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS)
169
+
170
+ max_retry_attempts = retry_attempts_from(msg['retry'], max_retries_default)
171
+
172
+ msg['retry_count'] >= max_retry_attempts
173
+ end
174
+
175
+ def retry_attempts_from(msg_retry, default)
176
+ if msg_retry.is_a?(Integer)
177
+ msg_retry
178
+ else
179
+ default
180
+ end
181
+ end
182
+
183
+ def send_to_morgue(msg)
184
+ Sidekiq.logger.warn(
185
+ class: msg['class'],
186
+ jid: msg['jid'],
187
+ message: %(Reliable Fetcher: adding dead #{msg['class']} job #{msg['jid']})
188
+ )
189
+
190
+ payload = Sidekiq.dump_json(msg)
191
+ Sidekiq::DeadSet.new.kill(payload, notify_failure: false)
192
+ end
193
+
148
194
  # Detect "old" jobs and requeue them because the worker they were assigned
149
195
  # to probably failed miserably.
150
196
  def clean_working_queues!
@@ -8,11 +8,12 @@ describe Sidekiq::BaseReliableFetch do
8
8
  before { Sidekiq.redis(&:flushdb) }
9
9
 
10
10
  describe 'UnitOfWork' do
11
+ let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
11
12
  let(:fetcher) { Sidekiq::ReliableFetch.new(queues: ['foo']) }
12
13
 
13
14
  describe '#requeue' do
14
15
  it 'requeues job' do
15
- Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
16
+ Sidekiq.redis { |conn| conn.rpush('queue:foo', job) }
16
17
 
17
18
  uow = fetcher.retrieve_work
18
19
 
@@ -25,7 +26,7 @@ describe Sidekiq::BaseReliableFetch do
25
26
 
26
27
  describe '#acknowledge' do
27
28
  it 'acknowledges job' do
28
- Sidekiq.redis { |conn| conn.rpush('queue:foo', 'msg') }
29
+ Sidekiq.redis { |conn| conn.rpush('queue:foo', job) }
29
30
 
30
31
  uow = fetcher.retrieve_work
31
32
 
@@ -4,33 +4,38 @@ shared_examples 'a Sidekiq fetcher' do
4
4
  before { Sidekiq.redis(&:flushdb) }
5
5
 
6
6
  describe '#retrieve_work' do
7
+ let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
7
8
  let(:fetcher) { described_class.new(queues: ['assigned']) }
8
9
 
9
10
  it 'retrieves the job and puts it to working queue' do
10
- Sidekiq.redis { |conn| conn.rpush('queue:assigned', 'msg') }
11
+ Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
11
12
 
12
13
  uow = fetcher.retrieve_work
13
14
 
14
15
  expect(working_queue_size('assigned')).to eq 1
15
16
  expect(uow.queue_name).to eq 'assigned'
16
- expect(uow.job).to eq 'msg'
17
+ expect(uow.job).to eq job
17
18
  expect(Sidekiq::Queue.new('assigned').size).to eq 0
18
19
  end
19
20
 
20
21
  it 'does not retrieve a job from foreign queue' do
21
- Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', 'msg') }
22
+ Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', job) }
22
23
 
23
24
  expect(fetcher.retrieve_work).to be_nil
24
25
  end
25
26
 
26
- it 'requeues jobs from dead working queue' do
27
+ it 'requeues jobs from dead working queue with incremented retry_count' do
27
28
  Sidekiq.redis do |conn|
28
- conn.rpush(other_process_working_queue_name('assigned'), 'msg')
29
+ conn.rpush(other_process_working_queue_name('assigned'), job)
29
30
  end
30
31
 
32
+ expected_job = Sidekiq.load_json(job)
33
+ expected_job['retry_count'] = 1
34
+ expected_job = Sidekiq.dump_json(expected_job)
35
+
31
36
  uow = fetcher.retrieve_work
32
37
 
33
- expect(uow.job).to eq 'msg'
38
+ expect(uow.job).to eq expected_job
34
39
 
35
40
  Sidekiq.redis do |conn|
36
41
  expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
@@ -41,7 +46,7 @@ shared_examples 'a Sidekiq fetcher' do
41
46
  working_queue = live_other_process_working_queue_name('assigned')
42
47
 
43
48
  Sidekiq.redis do |conn|
44
- conn.rpush(working_queue, 'msg')
49
+ conn.rpush(working_queue, job)
45
50
  end
46
51
 
47
52
  uow = fetcher.retrieve_work
@@ -1,5 +1,6 @@
1
1
  require 'spec_helper'
2
2
  require 'fetch_shared_examples'
3
+ require 'sidekiq/base_reliable_fetch'
3
4
  require 'sidekiq/reliable_fetch'
4
5
 
5
6
  describe Sidekiq::ReliableFetch do
@@ -1,5 +1,6 @@
1
1
  require 'spec_helper'
2
2
  require 'fetch_shared_examples'
3
+ require 'sidekiq/base_reliable_fetch'
3
4
  require 'sidekiq/semi_reliable_fetch'
4
5
 
5
6
  describe Sidekiq::SemiReliableFetch do
data/tests/README.md ADDED
@@ -0,0 +1,32 @@
1
+ # How to run reliability tests
2
+
3
+ ```
4
+ cd reliability_test
5
+ bundle exec ruby reliability_test.rb
6
+ ```
7
+
8
+ You can adjust some parameters of the test in the `config.rb`.
9
+
10
+ JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
11
+
12
+ You need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
13
+ `REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).
14
+
15
+
16
+ ## How it works
17
+
18
+ This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
19
+ number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
20
+
21
+ # How to run retry tests
22
+
23
+ ```
24
+ cd retry_test
25
+ bundle exec ruby retry_test.rb
26
+ ```
27
+
28
+ It requires Redis to be running on 6379 port.
29
+
30
+ ## How it works
31
+
32
+ It spawns Sidekiq workers then creates a job that will kill itself after a moment. The reliable fetcher will bring it back. The purpose is to verify that job is run no more then `retry` parameter says even when job was killed.
@@ -1,8 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative '../lib/sidekiq/base_reliable_fetch'
4
- require_relative '../lib/sidekiq/reliable_fetch'
5
- require_relative '../lib/sidekiq/semi_reliable_fetch'
3
+ require_relative '../../lib/sidekiq-reliable-fetch'
6
4
  require_relative 'worker'
7
5
 
8
6
  REDIS_FINISHED_LIST = 'reliable-fetcher-finished-jids'
@@ -89,7 +89,7 @@ Sidekiq.redis(&:flushdb)
89
89
  jobs = []
90
90
 
91
91
  NUMBER_OF_JOBS.times do
92
- jobs << TestWorker.perform_async
92
+ jobs << ReliabilityTestWorker.perform_async
93
93
  end
94
94
 
95
95
  puts "Queued #{NUMBER_OF_JOBS} jobs"
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- class TestWorker
3
+ class ReliabilityTestWorker
4
4
  include Sidekiq::Worker
5
5
 
6
6
  def perform
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../lib/sidekiq-reliable-fetch'
4
+ require_relative 'worker'
5
+
6
+ TEST_CLEANUP_INTERVAL = 20
7
+ TEST_LEASE_INTERVAL = 5
8
+
9
+ Sidekiq.configure_server do |config|
10
+ config.options[:semi_reliable_fetch] = true
11
+
12
+ # We need to override these parameters to not wait too long
13
+ # The default values are good for production use only
14
+ # These will be ignored for :basic
15
+ config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
16
+ config.options[:lease_interval] = TEST_LEASE_INTERVAL
17
+
18
+ Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
19
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'sidekiq'
4
+ require 'sidekiq/util'
5
+ require 'sidekiq/cli'
6
+ require_relative 'config'
7
+ require_relative 'simple_assert'
8
+
9
+ NUM_WORKERS = RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN + 1
10
+
11
+ Sidekiq.redis(&:flushdb)
12
+
13
+ def spawn_workers
14
+ pids = []
15
+
16
+ NUM_WORKERS.times do
17
+ pids << spawn('sidekiq -r ./config.rb')
18
+ end
19
+
20
+ pids
21
+ end
22
+
23
+ pids = spawn_workers
24
+
25
+ jid = RetryTestWorker.perform_async
26
+
27
+ sleep 300
28
+
29
+ Sidekiq.redis do |redis|
30
+ times_has_been_run = redis.get('times_has_been_run').to_i
31
+ assert "The job has been run", times_has_been_run, 2
32
+ end
33
+
34
+ assert "Found dead jobs", Sidekiq::DeadSet.new.size, 1
35
+
36
+ # Stop Sidekiq workers
37
+ pids.each do |pid|
38
+ Process.kill('KILL', pid)
39
+ Process.wait pid
40
+ end
@@ -0,0 +1,8 @@
1
+ def assert(text, actual, expected)
2
+ if actual == expected
3
+ puts "#{text}: #{actual} (Success)"
4
+ else
5
+ puts "#{text}: #{actual} (Failed). Expected: #{expected}"
6
+ exit 1
7
+ end
8
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ class RetryTestWorker
4
+ include Sidekiq::Worker
5
+
6
+ EXPECTED_NUM_TIMES_BEEN_RUN = 2
7
+
8
+ sidekiq_options retry: EXPECTED_NUM_TIMES_BEEN_RUN
9
+
10
+ sidekiq_retry_in do |count, exception|
11
+ 1 # retry in one second
12
+ end
13
+
14
+ def perform
15
+ sleep 1
16
+
17
+ Sidekiq.redis do |redis|
18
+ redis.incr('times_has_been_run')
19
+ end
20
+
21
+ Process.kill('KILL', Process.pid) # Job suicide, OOM killer imitation
22
+ end
23
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-sidekiq-fetcher
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.5.0.pre.alpha
5
5
  platform: ruby
6
6
  authors:
7
7
  - TEA
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2018-12-19 00:00:00.000000000 Z
12
+ date: 2019-08-02 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: sidekiq
@@ -49,10 +49,14 @@ files:
49
49
  - spec/reliable_fetch_spec.rb
50
50
  - spec/semi_reliable_fetch_spec.rb
51
51
  - spec/spec_helper.rb
52
- - test/README.md
53
- - test/config.rb
54
- - test/reliability_test.rb
55
- - test/worker.rb
52
+ - tests/README.md
53
+ - tests/reliability_test/config.rb
54
+ - tests/reliability_test/reliability_test.rb
55
+ - tests/reliability_test/worker.rb
56
+ - tests/retry_test/config.rb
57
+ - tests/retry_test/retry_test.rb
58
+ - tests/retry_test/simple_assert.rb
59
+ - tests/retry_test/worker.rb
56
60
  homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
57
61
  licenses:
58
62
  - LGPL-3.0
@@ -68,12 +72,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
68
72
  version: '0'
69
73
  required_rubygems_version: !ruby/object:Gem::Requirement
70
74
  requirements:
71
- - - ">="
75
+ - - ">"
72
76
  - !ruby/object:Gem::Version
73
- version: '0'
77
+ version: 1.3.1
74
78
  requirements: []
75
- rubyforge_project:
76
- rubygems_version: 2.7.6
79
+ rubygems_version: 3.0.3
77
80
  signing_key:
78
81
  specification_version: 4
79
82
  summary: Reliable fetch extension for Sidekiq
data/test/README.md DELETED
@@ -1,34 +0,0 @@
1
- # How to run
2
-
3
- ```
4
- cd test
5
- bundle exec ruby reliability_test.rb
6
- ```
7
-
8
- You can adjust some parameters of the test in the `config.rb`
9
-
10
-
11
- # How it works
12
-
13
- This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
14
- number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
15
-
16
- # How to run tests
17
-
18
- To run rspec:
19
-
20
- ```
21
- bundle exec rspec
22
- ```
23
-
24
- To run performance tests:
25
-
26
- ```
27
- cd test
28
- JOB_FETCHER=semi bundle exec ruby reliability_test.rb
29
- ```
30
-
31
- JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
32
-
33
- To run both kind of tests you need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
34
- `REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).