gitlab-sidekiq-fetcher 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +1 -1
- data/.gitlab-ci.yml +53 -0
- data/.rspec +1 -0
- data/Gemfile +12 -0
- data/Gemfile.lock +50 -0
- data/README.md +15 -4
- data/{README-GITLAB.md → RELEASE-GITLAB.md} +0 -0
- data/gitlab-sidekiq-fetcher.gemspec +2 -4
- data/lib/sidekiq-reliable-fetch.rb +3 -2
- data/lib/sidekiq/base_reliable_fetch.rb +185 -0
- data/lib/sidekiq/reliable_fetch.rb +40 -0
- data/lib/sidekiq/semi_reliable_fetch.rb +44 -0
- data/spec/base_reliable_fetch_spec.rb +73 -0
- data/spec/fetch_shared_examples.rb +118 -0
- data/spec/reliable_fetch_spec.rb +7 -0
- data/spec/semi_reliable_fetch_spec.rb +7 -0
- data/spec/spec_helper.rb +115 -0
- data/test/README.md +34 -0
- data/test/config.rb +31 -0
- data/test/reliability_test.rb +116 -0
- data/test/worker.rb +26 -0
- metadata +21 -10
- data/lib/sidekiq-reliable-fetch/api.rb +0 -56
- data/lib/sidekiq-reliable-fetch/web.rb +0 -24
- data/lib/sidekiq/reliable_fetcher.rb +0 -143
- data/web/views/working_queue.erb +0 -25
- data/web/views/working_queues.erb +0 -17
data/test/worker.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class TestWorker
|
4
|
+
include Sidekiq::Worker
|
5
|
+
|
6
|
+
def perform
|
7
|
+
# To mimic long running job and to increase the probability of losing the job
|
8
|
+
sleep 1
|
9
|
+
|
10
|
+
Sidekiq.redis do |redis|
|
11
|
+
redis.lpush(REDIS_FINISHED_LIST, get_sidekiq_job_id)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def get_sidekiq_job_id
|
16
|
+
context_data = Thread.current[:sidekiq_context]&.first
|
17
|
+
|
18
|
+
return unless context_data
|
19
|
+
|
20
|
+
index = context_data.index('JID-')
|
21
|
+
|
22
|
+
return unless index
|
23
|
+
|
24
|
+
context_data[index + 4..-1]
|
25
|
+
end
|
26
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-sidekiq-fetcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TEA
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-12-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sidekiq
|
@@ -32,17 +32,28 @@ extensions: []
|
|
32
32
|
extra_rdoc_files: []
|
33
33
|
files:
|
34
34
|
- ".gitignore"
|
35
|
+
- ".gitlab-ci.yml"
|
36
|
+
- ".rspec"
|
37
|
+
- Gemfile
|
38
|
+
- Gemfile.lock
|
35
39
|
- LICENSE
|
36
|
-
- README-GITLAB.md
|
37
40
|
- README.md
|
41
|
+
- RELEASE-GITLAB.md
|
38
42
|
- gitlab-sidekiq-fetcher.gemspec
|
39
43
|
- lib/sidekiq-reliable-fetch.rb
|
40
|
-
- lib/sidekiq
|
41
|
-
- lib/sidekiq
|
42
|
-
- lib/sidekiq/
|
43
|
-
-
|
44
|
-
-
|
45
|
-
|
44
|
+
- lib/sidekiq/base_reliable_fetch.rb
|
45
|
+
- lib/sidekiq/reliable_fetch.rb
|
46
|
+
- lib/sidekiq/semi_reliable_fetch.rb
|
47
|
+
- spec/base_reliable_fetch_spec.rb
|
48
|
+
- spec/fetch_shared_examples.rb
|
49
|
+
- spec/reliable_fetch_spec.rb
|
50
|
+
- spec/semi_reliable_fetch_spec.rb
|
51
|
+
- spec/spec_helper.rb
|
52
|
+
- test/README.md
|
53
|
+
- test/config.rb
|
54
|
+
- test/reliability_test.rb
|
55
|
+
- test/worker.rb
|
56
|
+
homepage: https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/
|
46
57
|
licenses:
|
47
58
|
- LGPL-3.0
|
48
59
|
metadata: {}
|
@@ -62,7 +73,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
62
73
|
version: '0'
|
63
74
|
requirements: []
|
64
75
|
rubyforge_project:
|
65
|
-
rubygems_version: 2.
|
76
|
+
rubygems_version: 2.7.6
|
66
77
|
signing_key:
|
67
78
|
specification_version: 4
|
68
79
|
summary: Reliable fetch extension for Sidekiq
|
@@ -1,56 +0,0 @@
|
|
1
|
-
module SidekiqReliableFetch
|
2
|
-
##
|
3
|
-
# Encapsulates a working queue within Sidekiq.
|
4
|
-
# Allows enumeration of all jobs within the queue.
|
5
|
-
#
|
6
|
-
# queue = SidekiqReliableFetch::WorkingQueue.new("mailer")
|
7
|
-
# queue.each do |job|
|
8
|
-
# job.klass # => 'MyWorker'
|
9
|
-
# job.args # => [1, 2, 3]
|
10
|
-
# end
|
11
|
-
#
|
12
|
-
class WorkingQueue
|
13
|
-
include Enumerable
|
14
|
-
|
15
|
-
def self.all
|
16
|
-
Sidekiq.redis { |c| c.keys('queue:*:working') }
|
17
|
-
.sort
|
18
|
-
.map { |q| SidekiqReliableFetch::WorkingQueue.new(q) }
|
19
|
-
end
|
20
|
-
|
21
|
-
attr_reader :name
|
22
|
-
|
23
|
-
def initialize(name)
|
24
|
-
@name = name
|
25
|
-
end
|
26
|
-
|
27
|
-
def size
|
28
|
-
Sidekiq.redis { |con| con.llen(@name) }
|
29
|
-
end
|
30
|
-
|
31
|
-
def each
|
32
|
-
initial_size = size
|
33
|
-
deleted_size = 0
|
34
|
-
page = 0
|
35
|
-
page_size = 50
|
36
|
-
|
37
|
-
loop do
|
38
|
-
range_start = page * page_size - deleted_size
|
39
|
-
range_end = page * page_size - deleted_size + (page_size - 1)
|
40
|
-
entries = Sidekiq.redis do |conn|
|
41
|
-
conn.lrange @name, range_start, range_end
|
42
|
-
end
|
43
|
-
break if entries.empty?
|
44
|
-
page += 1
|
45
|
-
entries.each do |entry|
|
46
|
-
yield Sidekiq::Job.new(entry, @name)
|
47
|
-
end
|
48
|
-
deleted_size = initial_size - size
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
def find_job(jid)
|
53
|
-
detect { |j| j.jid == jid }
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
require_relative 'api'
|
2
|
-
|
3
|
-
module SidekiqReliableFetch
|
4
|
-
# Hook into *Sidekiq::Web* Sinatra app which adds a new '/working' page
|
5
|
-
module Web
|
6
|
-
VIEW_PATH = File.expand_path('../../../web/views', __FILE__)
|
7
|
-
|
8
|
-
def self.registered(app)
|
9
|
-
app.get '/working' do
|
10
|
-
@queues = SidekiqReliableFetch::WorkingQueue.all
|
11
|
-
erb File.read(File.join(VIEW_PATH, 'working_queues.erb'))
|
12
|
-
end
|
13
|
-
|
14
|
-
app.get '/working/:queue' do
|
15
|
-
@queue = SidekiqReliableFetch::WorkingQueue.new(params[:queue])
|
16
|
-
erb File.read(File.join(VIEW_PATH, 'working_queue.erb'))
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
require 'sidekiq/web' unless defined?(Sidekiq::Web)
|
23
|
-
Sidekiq::Web.register(SidekiqReliableFetch::Web)
|
24
|
-
Sidekiq::Web.tabs['Working'] = 'working'
|
@@ -1,143 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
module Sidekiq
|
3
|
-
class ReliableFetcher
|
4
|
-
WORKING_QUEUE = 'working'
|
5
|
-
DEFAULT_DEAD_AFTER = 60 * 60 * 24 # 24 hours
|
6
|
-
DEFAULT_CLEANING_INTERVAL = 5000 # clean each N processed jobs
|
7
|
-
IDLE_TIMEOUT = 5 # seconds
|
8
|
-
|
9
|
-
UnitOfWork = Struct.new(:queue, :job) do
|
10
|
-
def acknowledge
|
11
|
-
# NOTE LREM is O(n), so depending on the type of jobs and their average
|
12
|
-
# duration, another data structure might be more suited.
|
13
|
-
# But as there should not be too much jobs in this queue in the same time,
|
14
|
-
# it's probably ok.
|
15
|
-
Sidekiq.redis { |conn| conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job) }
|
16
|
-
end
|
17
|
-
|
18
|
-
def queue_name
|
19
|
-
queue.sub(/.*queue:/, '')
|
20
|
-
end
|
21
|
-
|
22
|
-
def requeue
|
23
|
-
Sidekiq.redis do |conn|
|
24
|
-
conn.pipelined do
|
25
|
-
conn.lpush(queue, job)
|
26
|
-
conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
def self.setup_reliable_fetch!(config)
|
33
|
-
config.options[:fetch] = Sidekiq::ReliableFetcher
|
34
|
-
config.on(:startup) do
|
35
|
-
requeue_on_startup!(config.options[:queues])
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def initialize(options)
|
40
|
-
queues = options[:queues].map { |q| "queue:#{q}" }
|
41
|
-
|
42
|
-
@unique_queues = queues.uniq
|
43
|
-
@queues_iterator = queues.shuffle.cycle
|
44
|
-
@queues_size = queues.size
|
45
|
-
|
46
|
-
@nb_fetched_jobs = 0
|
47
|
-
@cleaning_interval = options[:cleaning_interval] || DEFAULT_CLEANING_INTERVAL
|
48
|
-
@consider_dead_after = options[:consider_dead_after] || DEFAULT_DEAD_AFTER
|
49
|
-
|
50
|
-
Sidekiq.logger.info { "GitLab reliable fetch activated!" }
|
51
|
-
end
|
52
|
-
|
53
|
-
def retrieve_work
|
54
|
-
clean_working_queues! if @cleaning_interval != -1 && @nb_fetched_jobs >= @cleaning_interval
|
55
|
-
|
56
|
-
@queues_size.times do
|
57
|
-
queue = @queues_iterator.next
|
58
|
-
work = Sidekiq.redis { |conn| conn.rpoplpush(queue, "#{queue}:#{WORKING_QUEUE}") }
|
59
|
-
|
60
|
-
if work
|
61
|
-
@nb_fetched_jobs += 1
|
62
|
-
return UnitOfWork.new(queue, work)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
# We didn't find a job in any of the configured queues. Let's sleep a bit
|
67
|
-
# to avoid uselessly burning too much CPU
|
68
|
-
sleep(IDLE_TIMEOUT)
|
69
|
-
|
70
|
-
nil
|
71
|
-
end
|
72
|
-
|
73
|
-
def self.requeue_on_startup!(queues)
|
74
|
-
Sidekiq.logger.debug { "Re-queueing working jobs" }
|
75
|
-
|
76
|
-
counter = 0
|
77
|
-
|
78
|
-
Sidekiq.redis do |conn|
|
79
|
-
queues.uniq.each do |queue|
|
80
|
-
while conn.rpoplpush("queue:#{queue}:#{WORKING_QUEUE}", "queue:#{queue}")
|
81
|
-
counter += 1
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
Sidekiq.logger.debug { "Re-queued #{counter} jobs" }
|
87
|
-
end
|
88
|
-
|
89
|
-
# By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
|
90
|
-
# an instance method will make it async to the Fetcher actor
|
91
|
-
def self.bulk_requeue(inprogress, options)
|
92
|
-
return if inprogress.empty?
|
93
|
-
|
94
|
-
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
95
|
-
|
96
|
-
Sidekiq.redis do |conn|
|
97
|
-
conn.pipelined do
|
98
|
-
inprogress.each do |unit_of_work|
|
99
|
-
conn.lpush("#{unit_of_work.queue}", unit_of_work.job)
|
100
|
-
conn.lrem("#{unit_of_work.queue}:#{WORKING_QUEUE}", 1, unit_of_work.job)
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
Sidekiq.logger.info("Pushed #{inprogress.size} messages back to Redis")
|
106
|
-
rescue => ex
|
107
|
-
Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
108
|
-
end
|
109
|
-
|
110
|
-
private
|
111
|
-
|
112
|
-
# Detect "old" jobs and requeue them because the worker they were assigned
|
113
|
-
# to probably failed miserably.
|
114
|
-
# NOTE Potential problem here if a specific job always make a worker
|
115
|
-
# really fail.
|
116
|
-
def clean_working_queues!
|
117
|
-
Sidekiq.logger.debug "Cleaning working queues"
|
118
|
-
|
119
|
-
@unique_queues.each do |queue|
|
120
|
-
clean_working_queue!(queue)
|
121
|
-
end
|
122
|
-
|
123
|
-
@nb_fetched_jobs = 0
|
124
|
-
end
|
125
|
-
|
126
|
-
def clean_working_queue!(queue)
|
127
|
-
Sidekiq.redis do |conn|
|
128
|
-
working_jobs = conn.lrange("#{queue}:#{WORKING_QUEUE}", 0, -1)
|
129
|
-
working_jobs.each do |job|
|
130
|
-
enqueued_at = Sidekiq.load_json(job)['enqueued_at'].to_i
|
131
|
-
job_duration = Time.now.to_i - enqueued_at
|
132
|
-
|
133
|
-
next if job_duration < @consider_dead_after
|
134
|
-
|
135
|
-
Sidekiq.logger.info "Requeued a dead job from #{queue}:#{WORKING_QUEUE}"
|
136
|
-
|
137
|
-
conn.lpush("#{queue}", job)
|
138
|
-
conn.lrem("#{queue}:#{WORKING_QUEUE}", 1, job)
|
139
|
-
end
|
140
|
-
end
|
141
|
-
end
|
142
|
-
end
|
143
|
-
end
|
data/web/views/working_queue.erb
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
<header class="row">
|
2
|
-
<div class="col-sm-5">
|
3
|
-
<h3><%= t('Working jobs') %></h3>
|
4
|
-
</div>
|
5
|
-
</header>
|
6
|
-
|
7
|
-
<% if @queue.size > 0 %>
|
8
|
-
<table class="table table-striped table-bordered table-white">
|
9
|
-
<thead>
|
10
|
-
<tr>
|
11
|
-
<th><%= t('Job') %></th>
|
12
|
-
<th><%= t('Arguments') %></th>
|
13
|
-
</tr>
|
14
|
-
</thead>
|
15
|
-
<% @queue.each do |entry| %>
|
16
|
-
<td><%= entry.display_class %></td>
|
17
|
-
<td>
|
18
|
-
<div class="args"><%= display_args(entry.display_args) %></div>
|
19
|
-
</td>
|
20
|
-
</tr>
|
21
|
-
<% end %>
|
22
|
-
</table>
|
23
|
-
<% else %>
|
24
|
-
<div class="alert alert-success"><%= t('No working job found') %></div>
|
25
|
-
<% end %>
|
@@ -1,17 +0,0 @@
|
|
1
|
-
<h3><%= t('Working queues') %></h3>
|
2
|
-
|
3
|
-
<div class="table_container">
|
4
|
-
<table class="queues table table-hover table-bordered table-striped table-white">
|
5
|
-
<thead>
|
6
|
-
<th><%= t('Queue') %></th>
|
7
|
-
<th><%= t('Size') %></th>
|
8
|
-
</thead>
|
9
|
-
<% @queues.each do |queue| %>
|
10
|
-
<tr>
|
11
|
-
<td>
|
12
|
-
<a href="<%= root_path %>working/<%= queue.name %>"><%= queue.name %></a>
|
13
|
-
</td>
|
14
|
-
<td><%= number_with_delimiter(queue.size) %></td>
|
15
|
-
</tr>
|
16
|
-
<% end %>
|
17
|
-
</table>
|