gitlab-sidekiq-fetcher 0.4.0 → 0.5.0.pre.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +10 -2
- data/gitlab-sidekiq-fetcher.gemspec +1 -1
- data/lib/sidekiq/base_reliable_fetch.rb +48 -2
- data/spec/base_reliable_fetch_spec.rb +3 -2
- data/spec/fetch_shared_examples.rb +12 -7
- data/spec/reliable_fetch_spec.rb +1 -0
- data/spec/semi_reliable_fetch_spec.rb +1 -0
- data/tests/README.md +32 -0
- data/{test → tests/reliability_test}/config.rb +1 -3
- data/{test → tests/reliability_test}/reliability_test.rb +1 -1
- data/{test → tests/reliability_test}/worker.rb +1 -1
- data/tests/retry_test/config.rb +19 -0
- data/tests/retry_test/retry_test.rb +40 -0
- data/tests/retry_test/simple_assert.rb +8 -0
- data/tests/retry_test/worker.rb +23 -0
- metadata +13 -10
- data/test/README.md +0 -34
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7da36ba54ba6a0e97cef5da210f02bdde86dc06e22960d37573e51e20046dd40
|
4
|
+
data.tar.gz: bd3bf13edc789374109a2c18b6797b44c1440bc2665938c654e3a627bbc5bc23
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 852f01191a052384cfe4215766c6b27dc932ced77782a92c23f73aba091b520e83d293d57b407e45ed5d141661664a9f77b88211b50e9bed52cb15a011b0347a
|
7
|
+
data.tar.gz: 59ee3493c9ddcb3e145ce77259b85ed508f8e57ca97878c12f5427e25952d2a4418860cd1d90562d39aa693dd630359e88ac78fd882aaceb9c5c374b6ca7509a
|
data/.gitlab-ci.yml
CHANGED
@@ -3,7 +3,7 @@ image: "ruby:2.5"
|
|
3
3
|
before_script:
|
4
4
|
- ruby -v
|
5
5
|
- which ruby
|
6
|
-
- gem install bundler
|
6
|
+
- gem install bundler
|
7
7
|
- bundle install --jobs $(nproc) "${FLAGS[@]}"
|
8
8
|
|
9
9
|
variables:
|
@@ -25,7 +25,7 @@ rspec:
|
|
25
25
|
.integration:
|
26
26
|
stage: test
|
27
27
|
script:
|
28
|
-
- cd
|
28
|
+
- cd tests/reliability_test
|
29
29
|
- bundle exec ruby reliability_test.rb
|
30
30
|
services:
|
31
31
|
- redis:alpine
|
@@ -47,6 +47,14 @@ integration_basic:
|
|
47
47
|
variables:
|
48
48
|
JOB_FETCHER: basic
|
49
49
|
|
50
|
+
retry_test:
|
51
|
+
stage: test
|
52
|
+
script:
|
53
|
+
- cd tests/retry_test
|
54
|
+
- bundle exec ruby retry_test.rb
|
55
|
+
services:
|
56
|
+
- redis:alpine
|
57
|
+
|
50
58
|
|
51
59
|
# rubocop:
|
52
60
|
# script:
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'sidekiq/job_retry'
|
4
|
+
|
3
5
|
module Sidekiq
|
4
6
|
class BaseReliableFetch
|
5
7
|
DEFAULT_CLEANUP_INTERVAL = 60 * 60 # 1 hour
|
@@ -137,14 +139,58 @@ module Sidekiq
|
|
137
139
|
Sidekiq.redis do |conn|
|
138
140
|
count = 0
|
139
141
|
|
140
|
-
while conn.
|
141
|
-
|
142
|
+
while job = conn.rpop(working_queue)
|
143
|
+
msg = begin
|
144
|
+
Sidekiq.load_json(job)
|
145
|
+
rescue => e
|
146
|
+
Sidekiq.logger.info("Skipped job: #{job} as we couldn't parse it")
|
147
|
+
next
|
148
|
+
end
|
149
|
+
|
150
|
+
msg['retry_count'] = msg['retry_count'].to_i + 1
|
151
|
+
|
152
|
+
if retries_exhausted?(msg)
|
153
|
+
send_to_morgue(msg)
|
154
|
+
else
|
155
|
+
job = Sidekiq.dump_json(msg)
|
156
|
+
|
157
|
+
conn.lpush(original_queue, job)
|
158
|
+
|
159
|
+
count += 1
|
160
|
+
end
|
142
161
|
end
|
143
162
|
|
144
163
|
Sidekiq.logger.info("Requeued #{count} dead jobs to #{original_queue}")
|
145
164
|
end
|
146
165
|
end
|
147
166
|
|
167
|
+
def retries_exhausted?(msg)
|
168
|
+
max_retries_default = Sidekiq.options.fetch(:max_retries, Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS)
|
169
|
+
|
170
|
+
max_retry_attempts = retry_attempts_from(msg['retry'], max_retries_default)
|
171
|
+
|
172
|
+
msg['retry_count'] >= max_retry_attempts
|
173
|
+
end
|
174
|
+
|
175
|
+
def retry_attempts_from(msg_retry, default)
|
176
|
+
if msg_retry.is_a?(Integer)
|
177
|
+
msg_retry
|
178
|
+
else
|
179
|
+
default
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def send_to_morgue(msg)
|
184
|
+
Sidekiq.logger.warn(
|
185
|
+
class: msg['class'],
|
186
|
+
jid: msg['jid'],
|
187
|
+
message: %(Reliable Fetcher: adding dead #{msg['class']} job #{msg['jid']})
|
188
|
+
)
|
189
|
+
|
190
|
+
payload = Sidekiq.dump_json(msg)
|
191
|
+
Sidekiq::DeadSet.new.kill(payload, notify_failure: false)
|
192
|
+
end
|
193
|
+
|
148
194
|
# Detect "old" jobs and requeue them because the worker they were assigned
|
149
195
|
# to probably failed miserably.
|
150
196
|
def clean_working_queues!
|
@@ -8,11 +8,12 @@ describe Sidekiq::BaseReliableFetch do
|
|
8
8
|
before { Sidekiq.redis(&:flushdb) }
|
9
9
|
|
10
10
|
describe 'UnitOfWork' do
|
11
|
+
let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
|
11
12
|
let(:fetcher) { Sidekiq::ReliableFetch.new(queues: ['foo']) }
|
12
13
|
|
13
14
|
describe '#requeue' do
|
14
15
|
it 'requeues job' do
|
15
|
-
Sidekiq.redis { |conn| conn.rpush('queue:foo',
|
16
|
+
Sidekiq.redis { |conn| conn.rpush('queue:foo', job) }
|
16
17
|
|
17
18
|
uow = fetcher.retrieve_work
|
18
19
|
|
@@ -25,7 +26,7 @@ describe Sidekiq::BaseReliableFetch do
|
|
25
26
|
|
26
27
|
describe '#acknowledge' do
|
27
28
|
it 'acknowledges job' do
|
28
|
-
Sidekiq.redis { |conn| conn.rpush('queue:foo',
|
29
|
+
Sidekiq.redis { |conn| conn.rpush('queue:foo', job) }
|
29
30
|
|
30
31
|
uow = fetcher.retrieve_work
|
31
32
|
|
@@ -4,33 +4,38 @@ shared_examples 'a Sidekiq fetcher' do
|
|
4
4
|
before { Sidekiq.redis(&:flushdb) }
|
5
5
|
|
6
6
|
describe '#retrieve_work' do
|
7
|
+
let(:job) { Sidekiq.dump_json({ class: 'Bob', args: [1, 2, 'foo'] }) }
|
7
8
|
let(:fetcher) { described_class.new(queues: ['assigned']) }
|
8
9
|
|
9
10
|
it 'retrieves the job and puts it to working queue' do
|
10
|
-
Sidekiq.redis { |conn| conn.rpush('queue:assigned',
|
11
|
+
Sidekiq.redis { |conn| conn.rpush('queue:assigned', job) }
|
11
12
|
|
12
13
|
uow = fetcher.retrieve_work
|
13
14
|
|
14
15
|
expect(working_queue_size('assigned')).to eq 1
|
15
16
|
expect(uow.queue_name).to eq 'assigned'
|
16
|
-
expect(uow.job).to eq
|
17
|
+
expect(uow.job).to eq job
|
17
18
|
expect(Sidekiq::Queue.new('assigned').size).to eq 0
|
18
19
|
end
|
19
20
|
|
20
21
|
it 'does not retrieve a job from foreign queue' do
|
21
|
-
Sidekiq.redis { |conn| conn.rpush('queue:not_assigned',
|
22
|
+
Sidekiq.redis { |conn| conn.rpush('queue:not_assigned', job) }
|
22
23
|
|
23
24
|
expect(fetcher.retrieve_work).to be_nil
|
24
25
|
end
|
25
26
|
|
26
|
-
it 'requeues jobs from dead working queue' do
|
27
|
+
it 'requeues jobs from dead working queue with incremented retry_count' do
|
27
28
|
Sidekiq.redis do |conn|
|
28
|
-
conn.rpush(other_process_working_queue_name('assigned'),
|
29
|
+
conn.rpush(other_process_working_queue_name('assigned'), job)
|
29
30
|
end
|
30
31
|
|
32
|
+
expected_job = Sidekiq.load_json(job)
|
33
|
+
expected_job['retry_count'] = 1
|
34
|
+
expected_job = Sidekiq.dump_json(expected_job)
|
35
|
+
|
31
36
|
uow = fetcher.retrieve_work
|
32
37
|
|
33
|
-
expect(uow.job).to eq
|
38
|
+
expect(uow.job).to eq expected_job
|
34
39
|
|
35
40
|
Sidekiq.redis do |conn|
|
36
41
|
expect(conn.llen(other_process_working_queue_name('assigned'))).to eq 0
|
@@ -41,7 +46,7 @@ shared_examples 'a Sidekiq fetcher' do
|
|
41
46
|
working_queue = live_other_process_working_queue_name('assigned')
|
42
47
|
|
43
48
|
Sidekiq.redis do |conn|
|
44
|
-
conn.rpush(working_queue,
|
49
|
+
conn.rpush(working_queue, job)
|
45
50
|
end
|
46
51
|
|
47
52
|
uow = fetcher.retrieve_work
|
data/spec/reliable_fetch_spec.rb
CHANGED
data/tests/README.md
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# How to run reliability tests
|
2
|
+
|
3
|
+
```
|
4
|
+
cd reliability_test
|
5
|
+
bundle exec ruby reliability_test.rb
|
6
|
+
```
|
7
|
+
|
8
|
+
You can adjust some parameters of the test in the `config.rb`.
|
9
|
+
|
10
|
+
JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
|
11
|
+
|
12
|
+
You need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
|
13
|
+
`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).
|
14
|
+
|
15
|
+
|
16
|
+
## How it works
|
17
|
+
|
18
|
+
This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
|
19
|
+
number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
|
20
|
+
|
21
|
+
# How to run retry tests
|
22
|
+
|
23
|
+
```
|
24
|
+
cd retry_test
|
25
|
+
bundle exec ruby retry_test.rb
|
26
|
+
```
|
27
|
+
|
28
|
+
It requires Redis to be running on 6379 port.
|
29
|
+
|
30
|
+
## How it works
|
31
|
+
|
32
|
+
It spawns Sidekiq workers then creates a job that will kill itself after a moment. The reliable fetcher will bring it back. The purpose is to verify that job is run no more then `retry` parameter says even when job was killed.
|
@@ -1,8 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative '
|
4
|
-
require_relative '../lib/sidekiq/reliable_fetch'
|
5
|
-
require_relative '../lib/sidekiq/semi_reliable_fetch'
|
3
|
+
require_relative '../../lib/sidekiq-reliable-fetch'
|
6
4
|
require_relative 'worker'
|
7
5
|
|
8
6
|
REDIS_FINISHED_LIST = 'reliable-fetcher-finished-jids'
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../lib/sidekiq-reliable-fetch'
|
4
|
+
require_relative 'worker'
|
5
|
+
|
6
|
+
TEST_CLEANUP_INTERVAL = 20
|
7
|
+
TEST_LEASE_INTERVAL = 5
|
8
|
+
|
9
|
+
Sidekiq.configure_server do |config|
|
10
|
+
config.options[:semi_reliable_fetch] = true
|
11
|
+
|
12
|
+
# We need to override these parameters to not wait too long
|
13
|
+
# The default values are good for production use only
|
14
|
+
# These will be ignored for :basic
|
15
|
+
config.options[:cleanup_interval] = TEST_CLEANUP_INTERVAL
|
16
|
+
config.options[:lease_interval] = TEST_LEASE_INTERVAL
|
17
|
+
|
18
|
+
Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
|
19
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'sidekiq'
|
4
|
+
require 'sidekiq/util'
|
5
|
+
require 'sidekiq/cli'
|
6
|
+
require_relative 'config'
|
7
|
+
require_relative 'simple_assert'
|
8
|
+
|
9
|
+
NUM_WORKERS = RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN + 1
|
10
|
+
|
11
|
+
Sidekiq.redis(&:flushdb)
|
12
|
+
|
13
|
+
def spawn_workers
|
14
|
+
pids = []
|
15
|
+
|
16
|
+
NUM_WORKERS.times do
|
17
|
+
pids << spawn('sidekiq -r ./config.rb')
|
18
|
+
end
|
19
|
+
|
20
|
+
pids
|
21
|
+
end
|
22
|
+
|
23
|
+
pids = spawn_workers
|
24
|
+
|
25
|
+
jid = RetryTestWorker.perform_async
|
26
|
+
|
27
|
+
sleep 300
|
28
|
+
|
29
|
+
Sidekiq.redis do |redis|
|
30
|
+
times_has_been_run = redis.get('times_has_been_run').to_i
|
31
|
+
assert "The job has been run", times_has_been_run, 2
|
32
|
+
end
|
33
|
+
|
34
|
+
assert "Found dead jobs", Sidekiq::DeadSet.new.size, 1
|
35
|
+
|
36
|
+
# Stop Sidekiq workers
|
37
|
+
pids.each do |pid|
|
38
|
+
Process.kill('KILL', pid)
|
39
|
+
Process.wait pid
|
40
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class RetryTestWorker
|
4
|
+
include Sidekiq::Worker
|
5
|
+
|
6
|
+
EXPECTED_NUM_TIMES_BEEN_RUN = 2
|
7
|
+
|
8
|
+
sidekiq_options retry: EXPECTED_NUM_TIMES_BEEN_RUN
|
9
|
+
|
10
|
+
sidekiq_retry_in do |count, exception|
|
11
|
+
1 # retry in one second
|
12
|
+
end
|
13
|
+
|
14
|
+
def perform
|
15
|
+
sleep 1
|
16
|
+
|
17
|
+
Sidekiq.redis do |redis|
|
18
|
+
redis.incr('times_has_been_run')
|
19
|
+
end
|
20
|
+
|
21
|
+
Process.kill('KILL', Process.pid) # Job suicide, OOM killer imitation
|
22
|
+
end
|
23
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-sidekiq-fetcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0.pre.alpha
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TEA
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2019-08-02 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sidekiq
|
@@ -49,10 +49,14 @@ files:
|
|
49
49
|
- spec/reliable_fetch_spec.rb
|
50
50
|
- spec/semi_reliable_fetch_spec.rb
|
51
51
|
- spec/spec_helper.rb
|
52
|
-
-
|
53
|
-
-
|
54
|
-
-
|
55
|
-
-
|
52
|
+
- tests/README.md
|
53
|
+
- tests/reliability_test/config.rb
|
54
|
+
- tests/reliability_test/reliability_test.rb
|
55
|
+
- tests/reliability_test/worker.rb
|
56
|
+
- tests/retry_test/config.rb
|
57
|
+
- tests/retry_test/retry_test.rb
|
58
|
+
- tests/retry_test/simple_assert.rb
|
59
|
+
- tests/retry_test/worker.rb
|
56
60
|
homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
|
57
61
|
licenses:
|
58
62
|
- LGPL-3.0
|
@@ -68,12 +72,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
68
72
|
version: '0'
|
69
73
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
70
74
|
requirements:
|
71
|
-
- - "
|
75
|
+
- - ">"
|
72
76
|
- !ruby/object:Gem::Version
|
73
|
-
version:
|
77
|
+
version: 1.3.1
|
74
78
|
requirements: []
|
75
|
-
|
76
|
-
rubygems_version: 2.7.6
|
79
|
+
rubygems_version: 3.0.3
|
77
80
|
signing_key:
|
78
81
|
specification_version: 4
|
79
82
|
summary: Reliable fetch extension for Sidekiq
|
data/test/README.md
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
# How to run
|
2
|
-
|
3
|
-
```
|
4
|
-
cd test
|
5
|
-
bundle exec ruby reliability_test.rb
|
6
|
-
```
|
7
|
-
|
8
|
-
You can adjust some parameters of the test in the `config.rb`
|
9
|
-
|
10
|
-
|
11
|
-
# How it works
|
12
|
-
|
13
|
-
This tool spawns configured number of Sidekiq workers and when the amount of processed jobs is about half of origin
|
14
|
-
number it will kill all the workers with `kill -9` and then it will spawn new workers again until all the jobs are processed. To track the process and counters we use Redis keys/counters.
|
15
|
-
|
16
|
-
# How to run tests
|
17
|
-
|
18
|
-
To run rspec:
|
19
|
-
|
20
|
-
```
|
21
|
-
bundle exec rspec
|
22
|
-
```
|
23
|
-
|
24
|
-
To run performance tests:
|
25
|
-
|
26
|
-
```
|
27
|
-
cd test
|
28
|
-
JOB_FETCHER=semi bundle exec ruby reliability_test.rb
|
29
|
-
```
|
30
|
-
|
31
|
-
JOB_FETCHER can be set to one of these values: `semi`, `reliable`, `basic`
|
32
|
-
|
33
|
-
To run both kind of tests you need to have redis server running on default HTTP port `6379`. To use other HTTP port, you can define
|
34
|
-
`REDIS_URL` environment varible with the port you need(example: `REDIS_URL="redis://localhost:9999"`).
|