gitlab-exporter 13.5.0 → 14.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 71244a8992a3116e4513746514842e6d6f0c53abbef7e0a7435a3889acce66c1
4
- data.tar.gz: fc013ec18e7f7a11e3d5da3a1375fe80add92f96da1f572eb8b577ee98bedebb
3
+ metadata.gz: 6420230ed0ec27d530fa71e46c0776ea2cb21b036a198335229fc304dcdfe3e0
4
+ data.tar.gz: 91205ee62a32b2e00f7978f9fc7ec063be203d593cade97ea481f85ee1388bbd
5
5
  SHA512:
6
- metadata.gz: b4d3c84092ca842c13123bb6573118adc44b60f090594db51b559bc20d2718c1e8a341494fd6ba376e6a30ce07985b2ae88923e7a68760a631936f3e962ed62f
7
- data.tar.gz: eb579c03705bdbd4d64b5c0a5b6dc40cbfa0125939b03d250d6efe44f395fe55effabbf3733ab57a0297494530ce8efc5b3521eda12c930b7ec8ffeadc8fb7b0
6
+ metadata.gz: 3be9966a1abe20b42169a4a85e20815649c87e46a31f853002dd5fb0eaff6437a692f5904334d42172fb1950627b0d3bba31a67c6992e5533051f367ed74144f
7
+ data.tar.gz: 20aefc5c04ee63033feeb80e986e6f239a711d9d88197859c154fe4e429573344d52d5c58ca9d9e611dace90804da5fba8341ed45ed5f592e33a5df8c3c6cc7e
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab-exporter (13.5.0)
4
+ gitlab-exporter (14.0.0)
5
5
  connection_pool (= 2.2.5)
6
6
  faraday (~> 1.8.0)
7
7
  pg (= 1.5.3)
@@ -107,4 +107,4 @@ DEPENDENCIES
107
107
  rubocop (~> 0.42)
108
108
 
109
109
  BUNDLED WITH
110
- 2.4.5
110
+ 2.4.20
@@ -95,8 +95,6 @@ probes:
95
95
  opts:
96
96
  redis_url: "redis://localhost:6379"
97
97
  redis_enable_client: true
98
- probe_namespaced: true
99
- probe_non_namespaced: true
100
98
 
101
99
  ruby: &ruby
102
100
  class_name: RubyProber
@@ -7,7 +7,6 @@ module GitLab
7
7
  # A prober for Sidekiq queues
8
8
  #
9
9
  # It takes the Redis URL Sidekiq is connected to
10
- # rubocop:disable Metrics/ClassLength
11
10
  class SidekiqProber
12
11
  # The maximum depth (from the head) of each queue to probe. Probing the
13
12
  # entirety of a very large queue will take longer and run the risk of
@@ -41,24 +40,35 @@ module GitLab
41
40
  @opts = opts
42
41
  @metrics = metrics
43
42
  @logger = logger
44
-
45
- @probe_namespaced = !!opts[:probe_namespaced] # rubocop:disable Style/DoubleNegation
46
- @probe_non_namespaced = !!opts[:probe_non_namespaced] # rubocop:disable Style/DoubleNegation
47
-
48
- # to maintain backward compatibility if the config is missing values
49
- @probe_namespaced = true if opts[:probe_namespaced].nil? && opts[:probe_non_namespaced].nil?
50
43
  end
51
44
 
52
45
  def probe_stats
53
- with_sidekiq { _probe_stats } if @probe_namespaced
54
- with_sidekiq(false) { _probe_stats(false) } if @probe_non_namespaced
46
+ with_sidekiq do
47
+ stats = Sidekiq::Stats.new
48
+
49
+ @metrics.add("sidekiq_jobs_processed_total", stats.processed)
50
+ @metrics.add("sidekiq_jobs_failed_total", stats.failed)
51
+ @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
52
+ @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
53
+ @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
54
+ @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
55
+
56
+ @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
57
+ @metrics.add("sidekiq_processes_size", stats.processes_size)
58
+ @metrics.add("sidekiq_workers_size", stats.workers_size)
59
+ end
55
60
 
56
61
  self
57
62
  end
58
63
 
59
64
  def probe_queues
60
- with_sidekiq { _probe_queues } if @probe_namespaced
61
- with_sidekiq(false) { _probe_queues(false) } if @probe_non_namespaced
65
+ with_sidekiq do
66
+ Sidekiq::Queue.all.each do |queue|
67
+ @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
68
+ @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
69
+ @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
70
+ end
71
+ end
62
72
 
63
73
  self
64
74
  end
@@ -71,8 +81,24 @@ module GitLab
71
81
  end
72
82
 
73
83
  def probe_future_sets
74
- with_sidekiq { _probe_future_sets } if @probe_namespaced
75
- with_sidekiq(false) { _probe_future_sets(false) } if @probe_non_namespaced
84
+ now = Time.now.to_f
85
+ with_sidekiq do
86
+ Sidekiq.redis do |conn|
87
+ Sidekiq::Scheduled::SETS.each do |set|
88
+ # Default to 0; if all jobs are due in the future, there is no "negative" delay.
89
+ delay = 0
90
+
91
+ _job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
92
+ delay = now - timestamp if timestamp
93
+
94
+ @metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay)
95
+
96
+ # zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
97
+ @metrics.add("sidekiq_#{set}_set_backlog_count",
98
+ conn.zcount(set, "-inf", now.to_s))
99
+ end
100
+ end
101
+ end
76
102
  end
77
103
 
78
104
  # Count worker classes present in Sidekiq queues. This only looks at the
@@ -82,22 +108,57 @@ module GitLab
82
108
  # will not have completely accurate statistics, but the probe performance
83
109
  # will also not degrade as the queue gets larger.
84
110
  def probe_jobs_limit
85
- with_sidekiq { _probe_jobs_limit } if @probe_namespaced
86
- with_sidekiq(false) { _probe_jobs_limit(false) } if @probe_non_namespaced
111
+ with_sidekiq do
112
+ job_stats = Hash.new(0)
113
+
114
+ Sidekiq::Queue.all.each do |queue|
115
+ Sidekiq.redis do |conn|
116
+ conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
117
+ job_class = Sidekiq.load_json(job)["class"]
118
+
119
+ job_stats[job_class] += 1
120
+ end
121
+ end
122
+ end
123
+
124
+ job_stats.each do |class_name, count|
125
+ @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
126
+ end
127
+ end
87
128
 
88
129
  self
89
130
  end
90
131
 
91
132
  def probe_workers
92
- with_sidekiq { _probe_workers } if @probe_namespaced
93
- with_sidekiq(false) { _probe_workers(false) } if @probe_non_namespaced
133
+ with_sidekiq do
134
+ worker_stats = Hash.new(0)
135
+
136
+ Sidekiq::Workers.new.map do |_pid, _tid, work|
137
+ job_klass = work["payload"]["class"]
138
+
139
+ worker_stats[job_klass] += 1
140
+ end
141
+
142
+ worker_stats.each do |class_name, count|
143
+ @metrics.add("sidekiq_running_jobs", count, name: class_name)
144
+ end
145
+ end
94
146
 
95
147
  self
96
148
  end
97
149
 
98
150
  def probe_retries
99
- with_sidekiq { _probe_retries } if @probe_namespaced
100
- with_sidekiq(false) { _probe_retries(false) } if @probe_non_namespaced
151
+ with_sidekiq do
152
+ retry_stats = Hash.new(0)
153
+
154
+ Sidekiq::RetrySet.new.map do |job|
155
+ retry_stats[job.klass] += 1
156
+ end
157
+
158
+ retry_stats.each do |class_name, count|
159
+ @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
160
+ end
161
+ end
101
162
 
102
163
  self
103
164
  end
@@ -119,104 +180,10 @@ module GitLab
119
180
 
120
181
  private
121
182
 
122
- def _probe_workers(namespace = true)
123
- labels = add_namespace_labels? ? { namespaced: namespace } : {}
124
- worker_stats = Hash.new(0)
125
-
126
- Sidekiq::Workers.new.map do |_pid, _tid, work|
127
- job_klass = work["payload"]["class"]
128
-
129
- worker_stats[job_klass] += 1
130
- end
131
-
132
- worker_stats.each do |class_name, count|
133
- @metrics.add("sidekiq_running_jobs", count, **labels.merge({ name: class_name }))
134
- end
135
- end
136
-
137
- def _probe_future_sets(namespace = true)
138
- labels = add_namespace_labels? ? { namespaced: namespace } : {}
139
- now = Time.now.to_f
140
- Sidekiq.redis do |conn|
141
- Sidekiq::Scheduled::SETS.each do |set|
142
- # Default to 0; if all jobs are due in the future, there is no "negative" delay.
143
- delay = 0
144
-
145
- _job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
146
- delay = now - timestamp if timestamp
147
-
148
- @metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay, **labels)
149
-
150
- # zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
151
- @metrics.add("sidekiq_#{set}_set_backlog_count",
152
- conn.zcount(set, "-inf", now.to_s), **labels)
153
- end
154
- end
155
- end
156
-
157
- def _probe_stats(namespace = true)
158
- stats = Sidekiq::Stats.new
159
- labels = add_namespace_labels? ? { namespaced: namespace } : {}
160
-
161
- @metrics.add("sidekiq_jobs_processed_total", stats.processed, **labels)
162
- @metrics.add("sidekiq_jobs_failed_total", stats.failed, **labels)
163
- @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued, **labels)
164
- @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size, **labels)
165
- @metrics.add("sidekiq_jobs_retry_size", stats.retry_size, **labels)
166
- @metrics.add("sidekiq_jobs_dead_size", stats.dead_size, **labels)
167
-
168
- @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency, **labels)
169
- @metrics.add("sidekiq_processes_size", stats.processes_size, **labels)
170
- @metrics.add("sidekiq_workers_size", stats.workers_size, **labels)
171
- end
172
-
173
- def _probe_queues(namespace = true)
174
- Sidekiq::Queue.all.each do |queue|
175
- labels = { name: queue.name }
176
- labels[:namespaced] = namespace if add_namespace_labels?
177
-
178
- @metrics.add("sidekiq_queue_size", queue.size, **labels)
179
- @metrics.add("sidekiq_queue_latency_seconds", queue.latency, **labels)
180
- @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, **labels)
181
- end
182
- end
183
-
184
- def _probe_jobs_limit(namespace = true)
185
- labels = add_namespace_labels? ? { namespaced: namespace } : {}
186
- job_stats = Hash.new(0)
187
-
188
- Sidekiq::Queue.all.each do |queue|
189
- Sidekiq.redis do |conn|
190
- conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
191
- job_class = Sidekiq.load_json(job)["class"]
192
-
193
- job_stats[job_class] += 1
194
- end
195
- end
196
- end
197
-
198
- job_stats.each do |class_name, count|
199
- @metrics.add("sidekiq_enqueued_jobs", count, **labels.merge({ name: class_name }))
200
- end
201
- end
202
-
203
- def _probe_retries(namespace = true)
204
- labels = add_namespace_labels? ? { namespaced: namespace } : {}
205
- retry_stats = Hash.new(0)
206
-
207
- Sidekiq::RetrySet.new.map do |job|
208
- retry_stats[job.klass] += 1
209
- end
210
-
211
- retry_stats.each do |class_name, count|
212
- @metrics.add("sidekiq_to_be_retried_jobs", count, **labels.merge({ name: class_name }))
213
- end
214
- end
215
-
216
- def with_sidekiq(namespaced = true)
183
+ def with_sidekiq
217
184
  SIDEKIQ_REDIS_LOCK.synchronize {
218
185
  Sidekiq.configure_client do |config|
219
- config.redis = self.class.connection_pool[redis_options(namespaced)]
186
+ config.redis = self.class.connection_pool[redis_options]
220
187
  end
221
188
 
222
189
  return unless connected?
@@ -225,14 +192,13 @@ module GitLab
225
192
  }
226
193
  end
227
194
 
228
- def redis_options(namespaced = true)
195
+ def redis_options
229
196
  options = {
230
197
  url: @opts[:redis_url],
231
198
  connect_timeout: 1,
232
199
  reconnect_attempts: 0
233
200
  }
234
201
 
235
- options[:namespace] = "resque:gitlab" if namespaced
236
202
  options[:id] = nil unless redis_enable_client?
237
203
  options
238
204
  end
@@ -253,11 +219,6 @@ module GitLab
253
219
  @logger&.error "Error connecting to the Redis: #{e}"
254
220
  @connected = false
255
221
  end
256
-
257
- def add_namespace_labels?
258
- @probe_namespaced && @probe_non_namespaced
259
- end
260
222
  end
261
- # rubocop:enable Metrics/ClassLength
262
223
  end
263
224
  end
@@ -1,5 +1,5 @@
1
1
  module GitLab
2
2
  module Exporter
3
- VERSION = "13.5.0".freeze
3
+ VERSION = "14.0.0".freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-exporter
3
3
  version: !ruby/object:Gem::Version
4
- version: 13.5.0
4
+ version: 14.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Carranza
@@ -236,7 +236,6 @@ files:
236
236
  - spec/memstats_spec.rb
237
237
  - spec/prometheus_metrics_spec.rb
238
238
  - spec/ruby_spec.rb
239
- - spec/sidekiq_spec.rb
240
239
  - spec/spec_helper.rb
241
240
  - spec/util_spec.rb
242
241
  homepage: https://gitlab.com/gitlab-org/ruby/gems/gitlab-exporter
@@ -275,6 +274,5 @@ test_files:
275
274
  - spec/memstats_spec.rb
276
275
  - spec/prometheus_metrics_spec.rb
277
276
  - spec/ruby_spec.rb
278
- - spec/sidekiq_spec.rb
279
277
  - spec/spec_helper.rb
280
278
  - spec/util_spec.rb
data/spec/sidekiq_spec.rb DELETED
@@ -1,64 +0,0 @@
1
- require "spec_helper"
2
- require "gitlab_exporter/sidekiq"
3
-
4
- describe GitLab::Exporter::SidekiqProber do
5
- let(:probe_list) { %w[probe_stats probe_queues probe_future_sets probe_jobs_limit probe_workers probe_retries] }
6
- let(:opt) { {} }
7
- let(:probe) { described_class.new(**opt) }
8
-
9
- before do
10
- # stub connection as we want check if the probe methods are called
11
- # with the right namespace argument
12
- allow(probe).to receive(:connected?).and_return(true)
13
- end
14
-
15
- context "when no namespace options are defined" do
16
- it "defaults to probing non-namespaced keys only" do
17
- probe_list.each do |probe_type|
18
- expect(probe).not_to receive("_#{probe_type}".to_sym).with(false)
19
- expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
20
-
21
- probe.send(probe_type)
22
- end
23
- end
24
- end
25
-
26
- context "when probing both namespaces" do
27
- let(:opt) { { probe_non_namespaced: true, probe_namespaced: true } }
28
-
29
- it "probes both namespaces" do
30
- probe_list.each do |probe_type|
31
- expect(probe).to receive("_#{probe_type}".to_sym).with(false)
32
- expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
33
-
34
- probe.send(probe_type)
35
- end
36
- end
37
- end
38
-
39
- context "when probing non-namespaced only" do
40
- let(:opt) { { probe_non_namespaced: true } }
41
-
42
- it "probes non-namespaced only" do
43
- probe_list.each do |probe_type|
44
- expect(probe).to receive("_#{probe_type}".to_sym).with(false)
45
- expect(probe).not_to receive("_#{probe_type}".to_sym).with(no_args)
46
-
47
- probe.send(probe_type)
48
- end
49
- end
50
- end
51
-
52
- context "when probing namespace only" do
53
- let(:opt) { { probe_namespaced: true } }
54
-
55
- it "probes namespaced only" do
56
- probe_list.each do |probe_type|
57
- expect(probe).not_to receive("_#{probe_type}".to_sym).with(false)
58
- expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
59
-
60
- probe.send(probe_type)
61
- end
62
- end
63
- end
64
- end