gitlab-exporter 13.1.0 → 13.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/config/gitlab-exporter.yml.example +2 -0
- data/lib/gitlab_exporter/sidekiq.rb +122 -84
- data/lib/gitlab_exporter/version.rb +1 -1
- data/spec/sidekiq_spec.rb +64 -0
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1faa75a67146917f990466c949a63cc5b53c4c4280bcaae76601071a88c5fb0c
|
4
|
+
data.tar.gz: acd50e384eb4e03fdbf2ba20c8f030a47b886822a3e87cdddded7326790e5f25
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6dbab1d6bce735e67ced073506ca2e9ddf1111ccd1b4a6e45a2289ce5a0ed069f5f004c2cda272a298d6328f7fe0fbd96cff30d434c722f01c1f1448b49199ce
|
7
|
+
data.tar.gz: c625442a214949794bad144d38a511c96161af0261380cac50f509f421c7da03c6549b94059451322c9f23498ca33ccc9ce528c02f4874357c79b21bf9486e04
|
data/Gemfile.lock
CHANGED
@@ -7,6 +7,7 @@ module GitLab
|
|
7
7
|
# A prober for Sidekiq queues
|
8
8
|
#
|
9
9
|
# It takes the Redis URL Sidekiq is connected to
|
10
|
+
# rubocop:disable Metrics/ClassLength
|
10
11
|
class SidekiqProber
|
11
12
|
# The maximum depth (from the head) of each queue to probe. Probing the
|
12
13
|
# entirety of a very large queue will take longer and run the risk of
|
@@ -37,35 +38,24 @@ module GitLab
|
|
37
38
|
@opts = opts
|
38
39
|
@metrics = metrics
|
39
40
|
@logger = logger
|
41
|
+
|
42
|
+
@probe_namespaced = !!opts[:probe_namespaced] # rubocop:disable Style/DoubleNegation
|
43
|
+
@probe_non_namespaced = !!opts[:probe_non_namespaced] # rubocop:disable Style/DoubleNegation
|
44
|
+
|
45
|
+
# to maintain backward compatibility if the config is missing values
|
46
|
+
@probe_namespaced = true if opts[:probe_namespaced].nil? && opts[:probe_non_namespaced].nil?
|
40
47
|
end
|
41
48
|
|
42
49
|
def probe_stats
|
43
|
-
with_sidekiq
|
44
|
-
|
45
|
-
|
46
|
-
@metrics.add("sidekiq_jobs_processed_total", stats.processed)
|
47
|
-
@metrics.add("sidekiq_jobs_failed_total", stats.failed)
|
48
|
-
@metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
|
49
|
-
@metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
|
50
|
-
@metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
|
51
|
-
@metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
|
52
|
-
|
53
|
-
@metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
|
54
|
-
@metrics.add("sidekiq_processes_size", stats.processes_size)
|
55
|
-
@metrics.add("sidekiq_workers_size", stats.workers_size)
|
56
|
-
end
|
50
|
+
with_sidekiq { _probe_stats } if @probe_namespaced
|
51
|
+
with_sidekiq(false) { _probe_stats(false) } if @probe_non_namespaced
|
57
52
|
|
58
53
|
self
|
59
54
|
end
|
60
55
|
|
61
56
|
def probe_queues
|
62
|
-
with_sidekiq
|
63
|
-
|
64
|
-
@metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
|
65
|
-
@metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
|
66
|
-
@metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
|
67
|
-
end
|
68
|
-
end
|
57
|
+
with_sidekiq { _probe_queues } if @probe_namespaced
|
58
|
+
with_sidekiq(false) { _probe_queues(false) } if @probe_non_namespaced
|
69
59
|
|
70
60
|
self
|
71
61
|
end
|
@@ -78,24 +68,8 @@ module GitLab
|
|
78
68
|
end
|
79
69
|
|
80
70
|
def probe_future_sets
|
81
|
-
|
82
|
-
with_sidekiq
|
83
|
-
Sidekiq.redis do |conn|
|
84
|
-
Sidekiq::Scheduled::SETS.each do |set|
|
85
|
-
# Default to 0; if all jobs are due in the future, there is no "negative" delay.
|
86
|
-
delay = 0
|
87
|
-
|
88
|
-
_job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
|
89
|
-
delay = now - timestamp if timestamp
|
90
|
-
|
91
|
-
@metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay)
|
92
|
-
|
93
|
-
# zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
|
94
|
-
@metrics.add("sidekiq_#{set}_set_backlog_count",
|
95
|
-
conn.zcount(set, "-inf", now.to_s))
|
96
|
-
end
|
97
|
-
end
|
98
|
-
end
|
71
|
+
with_sidekiq { _probe_future_sets } if @probe_namespaced
|
72
|
+
with_sidekiq(false) { _probe_future_sets(false) } if @probe_non_namespaced
|
99
73
|
end
|
100
74
|
|
101
75
|
# Count worker classes present in Sidekiq queues. This only looks at the
|
@@ -105,57 +79,22 @@ module GitLab
|
|
105
79
|
# will not have completely accurate statistics, but the probe performance
|
106
80
|
# will also not degrade as the queue gets larger.
|
107
81
|
def probe_jobs_limit
|
108
|
-
with_sidekiq
|
109
|
-
|
110
|
-
|
111
|
-
Sidekiq::Queue.all.each do |queue|
|
112
|
-
Sidekiq.redis do |conn|
|
113
|
-
conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
|
114
|
-
job_class = Sidekiq.load_json(job)["class"]
|
115
|
-
|
116
|
-
job_stats[job_class] += 1
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
job_stats.each do |class_name, count|
|
122
|
-
@metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
|
123
|
-
end
|
124
|
-
end
|
82
|
+
with_sidekiq { _probe_jobs_limit } if @probe_namespaced
|
83
|
+
with_sidekiq(false) { _probe_jobs_limit(false) } if @probe_non_namespaced
|
125
84
|
|
126
85
|
self
|
127
86
|
end
|
128
87
|
|
129
88
|
def probe_workers
|
130
|
-
with_sidekiq
|
131
|
-
|
132
|
-
|
133
|
-
Sidekiq::Workers.new.map do |_pid, _tid, work|
|
134
|
-
job_klass = work["payload"]["class"]
|
135
|
-
|
136
|
-
worker_stats[job_klass] += 1
|
137
|
-
end
|
138
|
-
|
139
|
-
worker_stats.each do |class_name, count|
|
140
|
-
@metrics.add("sidekiq_running_jobs", count, name: class_name)
|
141
|
-
end
|
142
|
-
end
|
89
|
+
with_sidekiq { _probe_workers } if @probe_namespaced
|
90
|
+
with_sidekiq(false) { _probe_workers(false) } if @probe_non_namespaced
|
143
91
|
|
144
92
|
self
|
145
93
|
end
|
146
94
|
|
147
95
|
def probe_retries
|
148
|
-
with_sidekiq
|
149
|
-
|
150
|
-
|
151
|
-
Sidekiq::RetrySet.new.map do |job|
|
152
|
-
retry_stats[job.klass] += 1
|
153
|
-
end
|
154
|
-
|
155
|
-
retry_stats.each do |class_name, count|
|
156
|
-
@metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
|
157
|
-
end
|
158
|
-
end
|
96
|
+
with_sidekiq { _probe_retries } if @probe_namespaced
|
97
|
+
with_sidekiq(false) { _probe_retries(false) } if @probe_non_namespaced
|
159
98
|
|
160
99
|
self
|
161
100
|
end
|
@@ -177,12 +116,106 @@ module GitLab
|
|
177
116
|
|
178
117
|
private
|
179
118
|
|
180
|
-
def
|
119
|
+
def _probe_workers(namespace = true)
|
120
|
+
labels = add_namespace_labels? ? { namespaced: namespace } : {}
|
121
|
+
worker_stats = Hash.new(0)
|
122
|
+
|
123
|
+
Sidekiq::Workers.new.map do |_pid, _tid, work|
|
124
|
+
job_klass = work["payload"]["class"]
|
125
|
+
|
126
|
+
worker_stats[job_klass] += 1
|
127
|
+
end
|
128
|
+
|
129
|
+
worker_stats.each do |class_name, count|
|
130
|
+
@metrics.add("sidekiq_running_jobs", count, **labels.merge({ name: class_name }))
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def _probe_future_sets(namespace = true)
|
135
|
+
labels = add_namespace_labels? ? { namespaced: namespace } : {}
|
136
|
+
now = Time.now.to_f
|
137
|
+
Sidekiq.redis do |conn|
|
138
|
+
Sidekiq::Scheduled::SETS.each do |set|
|
139
|
+
# Default to 0; if all jobs are due in the future, there is no "negative" delay.
|
140
|
+
delay = 0
|
141
|
+
|
142
|
+
_job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
|
143
|
+
delay = now - timestamp if timestamp
|
144
|
+
|
145
|
+
@metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay, **labels)
|
146
|
+
|
147
|
+
# zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
|
148
|
+
@metrics.add("sidekiq_#{set}_set_backlog_count",
|
149
|
+
conn.zcount(set, "-inf", now.to_s), **labels)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def _probe_stats(namespace = true)
|
155
|
+
stats = Sidekiq::Stats.new
|
156
|
+
labels = add_namespace_labels? ? { namespaced: namespace } : {}
|
157
|
+
|
158
|
+
@metrics.add("sidekiq_jobs_processed_total", stats.processed, **labels)
|
159
|
+
@metrics.add("sidekiq_jobs_failed_total", stats.failed, **labels)
|
160
|
+
@metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued, **labels)
|
161
|
+
@metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size, **labels)
|
162
|
+
@metrics.add("sidekiq_jobs_retry_size", stats.retry_size, **labels)
|
163
|
+
@metrics.add("sidekiq_jobs_dead_size", stats.dead_size, **labels)
|
164
|
+
|
165
|
+
@metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency, **labels)
|
166
|
+
@metrics.add("sidekiq_processes_size", stats.processes_size, **labels)
|
167
|
+
@metrics.add("sidekiq_workers_size", stats.workers_size, **labels)
|
168
|
+
end
|
169
|
+
|
170
|
+
def _probe_queues(namespace = true)
|
171
|
+
Sidekiq::Queue.all.each do |queue|
|
172
|
+
labels = { name: queue.name }
|
173
|
+
labels[:namespaced] = namespace if add_namespace_labels?
|
174
|
+
|
175
|
+
@metrics.add("sidekiq_queue_size", queue.size, **labels)
|
176
|
+
@metrics.add("sidekiq_queue_latency_seconds", queue.latency, **labels)
|
177
|
+
@metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, **labels)
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
def _probe_jobs_limit(namespace = true)
|
182
|
+
labels = add_namespace_labels? ? { namespaced: namespace } : {}
|
183
|
+
job_stats = Hash.new(0)
|
184
|
+
|
185
|
+
Sidekiq::Queue.all.each do |queue|
|
186
|
+
Sidekiq.redis do |conn|
|
187
|
+
conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
|
188
|
+
job_class = Sidekiq.load_json(job)["class"]
|
189
|
+
|
190
|
+
job_stats[job_class] += 1
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
job_stats.each do |class_name, count|
|
196
|
+
@metrics.add("sidekiq_enqueued_jobs", count, **labels.merge({ name: class_name }))
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
def _probe_retries(namespace = true)
|
201
|
+
labels = add_namespace_labels? ? { namespaced: namespace } : {}
|
202
|
+
retry_stats = Hash.new(0)
|
203
|
+
|
204
|
+
Sidekiq::RetrySet.new.map do |job|
|
205
|
+
retry_stats[job.klass] += 1
|
206
|
+
end
|
207
|
+
|
208
|
+
retry_stats.each do |class_name, count|
|
209
|
+
@metrics.add("sidekiq_to_be_retried_jobs", count, **labels.merge({ name: class_name }))
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
def with_sidekiq(namespaced = true)
|
181
214
|
# TODO: this is not concurrent safe as we change global context
|
182
215
|
# It means that we are unable to use many different sidekiq's
|
183
216
|
# which is not a problem as of now
|
184
217
|
Sidekiq.configure_client do |config|
|
185
|
-
config.redis = self.class.connection_pool[redis_options]
|
218
|
+
config.redis = self.class.connection_pool[redis_options(namespaced)]
|
186
219
|
end
|
187
220
|
|
188
221
|
return unless connected?
|
@@ -190,14 +223,14 @@ module GitLab
|
|
190
223
|
yield
|
191
224
|
end
|
192
225
|
|
193
|
-
def redis_options
|
226
|
+
def redis_options(namespaced = true)
|
194
227
|
options = {
|
195
228
|
url: @opts[:redis_url],
|
196
|
-
namespace: "resque:gitlab",
|
197
229
|
connect_timeout: 1,
|
198
230
|
reconnect_attempts: 0
|
199
231
|
}
|
200
232
|
|
233
|
+
options[:namespace] = "resque:gitlab" if namespaced
|
201
234
|
options[:id] = nil unless redis_enable_client?
|
202
235
|
options
|
203
236
|
end
|
@@ -218,6 +251,11 @@ module GitLab
|
|
218
251
|
@logger&.error "Error connecting to the Redis: #{e}"
|
219
252
|
@connected = false
|
220
253
|
end
|
254
|
+
|
255
|
+
def add_namespace_labels?
|
256
|
+
@probe_namespaced && @probe_non_namespaced
|
257
|
+
end
|
221
258
|
end
|
259
|
+
# rubocop:enable Metrics/ClassLength
|
222
260
|
end
|
223
261
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "gitlab_exporter/sidekiq"
|
3
|
+
|
4
|
+
describe GitLab::Exporter::SidekiqProber do
|
5
|
+
let(:probe_list) { %w[probe_stats probe_queues probe_future_sets probe_jobs_limit probe_workers probe_retries] }
|
6
|
+
let(:opt) { {} }
|
7
|
+
let(:probe) { described_class.new(**opt) }
|
8
|
+
|
9
|
+
before do
|
10
|
+
# stub connection as we want check if the probe methods are called
|
11
|
+
# with the right namespace argument
|
12
|
+
allow(probe).to receive(:connected?).and_return(true)
|
13
|
+
end
|
14
|
+
|
15
|
+
context "when no namespace options are defined" do
|
16
|
+
it "defaults to probing non-namespaced keys only" do
|
17
|
+
probe_list.each do |probe_type|
|
18
|
+
expect(probe).not_to receive("_#{probe_type}".to_sym).with(false)
|
19
|
+
expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
|
20
|
+
|
21
|
+
probe.send(probe_type)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
context "when probing both namespaces" do
|
27
|
+
let(:opt) { { probe_non_namespaced: true, probe_namespaced: true } }
|
28
|
+
|
29
|
+
it "probes both namespaces" do
|
30
|
+
probe_list.each do |probe_type|
|
31
|
+
expect(probe).to receive("_#{probe_type}".to_sym).with(false)
|
32
|
+
expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
|
33
|
+
|
34
|
+
probe.send(probe_type)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context "when probing non-namespaced only" do
|
40
|
+
let(:opt) { { probe_non_namespaced: true } }
|
41
|
+
|
42
|
+
it "probes non-namespaced only" do
|
43
|
+
probe_list.each do |probe_type|
|
44
|
+
expect(probe).to receive("_#{probe_type}".to_sym).with(false)
|
45
|
+
expect(probe).not_to receive("_#{probe_type}".to_sym).with(no_args)
|
46
|
+
|
47
|
+
probe.send(probe_type)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
context "when probing namespace only" do
|
53
|
+
let(:opt) { { probe_namespaced: true } }
|
54
|
+
|
55
|
+
it "probes namespaced only" do
|
56
|
+
probe_list.each do |probe_type|
|
57
|
+
expect(probe).not_to receive("_#{probe_type}".to_sym).with(false)
|
58
|
+
expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
|
59
|
+
|
60
|
+
probe.send(probe_type)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-exporter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 13.
|
4
|
+
version: 13.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Carranza
|
@@ -235,6 +235,7 @@ files:
|
|
235
235
|
- spec/memstats_spec.rb
|
236
236
|
- spec/prometheus_metrics_spec.rb
|
237
237
|
- spec/ruby_spec.rb
|
238
|
+
- spec/sidekiq_spec.rb
|
238
239
|
- spec/spec_helper.rb
|
239
240
|
- spec/util_spec.rb
|
240
241
|
homepage: http://gitlab.com
|
@@ -273,5 +274,6 @@ test_files:
|
|
273
274
|
- spec/memstats_spec.rb
|
274
275
|
- spec/prometheus_metrics_spec.rb
|
275
276
|
- spec/ruby_spec.rb
|
277
|
+
- spec/sidekiq_spec.rb
|
276
278
|
- spec/spec_helper.rb
|
277
279
|
- spec/util_spec.rb
|