gitlab-sidekiq-fetcher 0.5.0.pre.alpha → 0.5.1.pre.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitlab-ci.yml +8 -0
- data/gitlab-sidekiq-fetcher.gemspec +1 -1
- data/lib/sidekiq/base_reliable_fetch.rb +7 -0
- data/tests/README.md +3 -0
- data/tests/retry_test/config.rb +1 -0
- data/tests/retry_test/no_retry_test.rb +24 -0
- data/tests/retry_test/no_retry_worker.rb +21 -0
- data/tests/retry_test/retry_test.rb +5 -21
- data/tests/support/utils.rb +26 -0
- metadata +7 -4
- data/tests/retry_test/simple_assert.rb +0 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ac14337d2c1c3f109b1d5c9f632236a7b9e5513a
|
4
|
+
data.tar.gz: 683908ba8fb1823fa7cb7078ebd0f9210ddefee6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '00058b4cc9dcf70223cdb4153731a8f774e2caddca3d5287729a29888098351ee1a33b8f15bb9e5a49014dbea950193895dae0ed152806fdf4e6a4ac4df5a59c'
|
7
|
+
data.tar.gz: 04bd84f1a2b6c8b91aab8543c1e0bb34a9845ca6afae01fc2564cc3c5429c8d0c79c8c89984babd9ecfefe0bee21bde67cd3d6c0c94feca15cd3a15dd7fd3ac9
|
data/.gitlab-ci.yml
CHANGED
@@ -165,6 +165,13 @@ module Sidekiq
|
|
165
165
|
end
|
166
166
|
|
167
167
|
def retries_exhausted?(msg)
|
168
|
+
# `retry` parameter can be empty when job is running the first time and when
|
169
|
+
# it's not specified in worker class explicitly.
|
170
|
+
# In that case, the default parameter gets injected into the job when
|
171
|
+
# it fails the first time in JobRetry#local.
|
172
|
+
# We should handle the case when `retry` is explicitly set to false
|
173
|
+
return true if msg['retry'] === false
|
174
|
+
|
168
175
|
max_retries_default = Sidekiq.options.fetch(:max_retries, Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS)
|
169
176
|
|
170
177
|
max_retry_attempts = retry_attempts_from(msg['retry'], max_retries_default)
|
data/tests/README.md
CHANGED
@@ -23,6 +23,9 @@ number it will kill all the workers with `kill -9` and then it will spawn new wo
|
|
23
23
|
```
|
24
24
|
cd retry_test
|
25
25
|
bundle exec ruby retry_test.rb
|
26
|
+
|
27
|
+
# To verify that workers with "retry: false" are not retried
|
28
|
+
bundle exec ruby no_retry_test.rb
|
26
29
|
```
|
27
30
|
|
28
31
|
It requires Redis to be running on 6379 port.
|
data/tests/retry_test/config.rb
CHANGED
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'sidekiq'
|
4
|
+
require_relative 'config'
|
5
|
+
require_relative '../support/utils'
|
6
|
+
|
7
|
+
NUM_WORKERS = 2 # one worker will be killed and one spare worker t verify that job is not picked up
|
8
|
+
|
9
|
+
Sidekiq.redis(&:flushdb)
|
10
|
+
|
11
|
+
pids = spawn_workers(NUM_WORKERS)
|
12
|
+
|
13
|
+
jid = NoRetryTestWorker.perform_async
|
14
|
+
|
15
|
+
sleep 300
|
16
|
+
|
17
|
+
Sidekiq.redis do |redis|
|
18
|
+
times_has_been_run = redis.get('times_has_been_run').to_i
|
19
|
+
assert 'The job has been run', times_has_been_run, 1
|
20
|
+
end
|
21
|
+
|
22
|
+
assert 'Found dead jobs', Sidekiq::DeadSet.new.size, 1
|
23
|
+
|
24
|
+
stop_workers(pids)
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class NoRetryTestWorker
|
4
|
+
include Sidekiq::Worker
|
5
|
+
|
6
|
+
sidekiq_options retry: false
|
7
|
+
|
8
|
+
sidekiq_retry_in do |count, exception|
|
9
|
+
1 # retry in one second
|
10
|
+
end
|
11
|
+
|
12
|
+
def perform
|
13
|
+
sleep 1
|
14
|
+
|
15
|
+
Sidekiq.redis do |redis|
|
16
|
+
redis.incr('times_has_been_run')
|
17
|
+
end
|
18
|
+
|
19
|
+
Process.kill('KILL', Process.pid) # Job suicide, OOM killer imitation
|
20
|
+
end
|
21
|
+
end
|
@@ -1,26 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'sidekiq'
|
4
|
-
require 'sidekiq/util'
|
5
|
-
require 'sidekiq/cli'
|
6
4
|
require_relative 'config'
|
7
|
-
require_relative '
|
5
|
+
require_relative '../support/utils'
|
8
6
|
|
9
7
|
NUM_WORKERS = RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN + 1
|
10
8
|
|
11
9
|
Sidekiq.redis(&:flushdb)
|
12
10
|
|
13
|
-
|
14
|
-
pids = []
|
15
|
-
|
16
|
-
NUM_WORKERS.times do
|
17
|
-
pids << spawn('sidekiq -r ./config.rb')
|
18
|
-
end
|
19
|
-
|
20
|
-
pids
|
21
|
-
end
|
22
|
-
|
23
|
-
pids = spawn_workers
|
11
|
+
pids = spawn_workers(NUM_WORKERS)
|
24
12
|
|
25
13
|
jid = RetryTestWorker.perform_async
|
26
14
|
|
@@ -28,13 +16,9 @@ sleep 300
|
|
28
16
|
|
29
17
|
Sidekiq.redis do |redis|
|
30
18
|
times_has_been_run = redis.get('times_has_been_run').to_i
|
31
|
-
assert
|
19
|
+
assert 'The job has been run', times_has_been_run, RetryTestWorker::EXPECTED_NUM_TIMES_BEEN_RUN
|
32
20
|
end
|
33
21
|
|
34
|
-
assert
|
22
|
+
assert 'Found dead jobs', Sidekiq::DeadSet.new.size, 1
|
35
23
|
|
36
|
-
|
37
|
-
pids.each do |pid|
|
38
|
-
Process.kill('KILL', pid)
|
39
|
-
Process.wait pid
|
40
|
-
end
|
24
|
+
stop_workers(pids)
|
@@ -0,0 +1,26 @@
|
|
1
|
+
def assert(text, actual, expected)
|
2
|
+
if actual == expected
|
3
|
+
puts "#{text}: #{actual} (Success)"
|
4
|
+
else
|
5
|
+
puts "#{text}: #{actual} (Failed). Expected: #{expected}"
|
6
|
+
exit 1
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
def spawn_workers(number)
|
11
|
+
pids = []
|
12
|
+
|
13
|
+
number.times do
|
14
|
+
pids << spawn('sidekiq -r ./config.rb')
|
15
|
+
end
|
16
|
+
|
17
|
+
pids
|
18
|
+
end
|
19
|
+
|
20
|
+
# Stop Sidekiq workers
|
21
|
+
def stop_workers(pids)
|
22
|
+
pids.each do |pid|
|
23
|
+
Process.kill('KILL', pid)
|
24
|
+
Process.wait pid
|
25
|
+
end
|
26
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-sidekiq-fetcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.1.pre.alpha
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TEA
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-08-
|
12
|
+
date: 2019-08-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sidekiq
|
@@ -54,9 +54,11 @@ files:
|
|
54
54
|
- tests/reliability_test/reliability_test.rb
|
55
55
|
- tests/reliability_test/worker.rb
|
56
56
|
- tests/retry_test/config.rb
|
57
|
+
- tests/retry_test/no_retry_test.rb
|
58
|
+
- tests/retry_test/no_retry_worker.rb
|
57
59
|
- tests/retry_test/retry_test.rb
|
58
|
-
- tests/retry_test/simple_assert.rb
|
59
60
|
- tests/retry_test/worker.rb
|
61
|
+
- tests/support/utils.rb
|
60
62
|
homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
|
61
63
|
licenses:
|
62
64
|
- LGPL-3.0
|
@@ -76,7 +78,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
76
78
|
- !ruby/object:Gem::Version
|
77
79
|
version: 1.3.1
|
78
80
|
requirements: []
|
79
|
-
|
81
|
+
rubyforge_project:
|
82
|
+
rubygems_version: 2.5.2
|
80
83
|
signing_key:
|
81
84
|
specification_version: 4
|
82
85
|
summary: Reliable fetch extension for Sidekiq
|