gitlab-exporter 13.1.0 → 13.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e25742f97cb40db0a84768e5b780c3d4d29af3c35ce456cc54ab5cebbf39ded7
4
- data.tar.gz: 13cf4d4d86554c783d9c7cb1a7c9db42189b0fd6fe520e10f15b084cb5b43761
3
+ metadata.gz: 8a115af5d2f6c523fe9a6fb55e09e47bbae42b6fdee96d20e2f0987e758546be
4
+ data.tar.gz: cbb66dac910580239114923f0b70ad40a6580cac4a1b29e69e308559c6619e3e
5
5
  SHA512:
6
- metadata.gz: 7f48b26649ec291a96b41976d6b674171667c43ab0f8de188809e9d22fdb3d0ee0d17c33ac1ef30a4f217f1f52c701335a88ed600a915797d8013a264359bdde
7
- data.tar.gz: 954a83f62a78a78244503a2b47c5bfe9fc2fac338e46cf692d03731b53b459ec715e83b4bbd5df5320c55efe47c8e7db17b0af944fb490e09ee1b1453b4bf6a3
6
+ metadata.gz: bfe2f5921dbd280826ba2e959d1bdc09fdf721f691e32c31f12e8948805a4103b30e29cc3286b56be1c4fd9d7e766b5b5e4b7df8ae117d1a53a6b03e4bfdda7d
7
+ data.tar.gz: d0dde2e781ad6599e7944c400d2efd92272b43c813229a26754cf944da6972563154c35d9f5f1f75678ecb4537edef61cb6cc13f0a8cb433a7a5922b4c755993
data/.gitlab-ci.yml CHANGED
@@ -7,7 +7,7 @@ include:
7
7
  - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
8
8
 
9
9
  variables:
10
- RUBY_VERSION: "2.7"
10
+ RUBY_VERSION: "3.2"
11
11
 
12
12
  stages:
13
13
  - test
@@ -43,7 +43,7 @@ rspec:
43
43
  before_script: *before_scripts
44
44
  parallel:
45
45
  matrix:
46
- - RUBY_VERSION: ["2.7", "3.0", "3.1", "3.2"]
46
+ - RUBY_VERSION: ["3.0", "3.1", "3.2"]
47
47
 
48
48
  rspec_integration:
49
49
  script:
@@ -55,7 +55,7 @@ rspec_integration:
55
55
  REDIS_URL: "redis://redis"
56
56
  parallel:
57
57
  matrix:
58
- - RUBY_VERSION: ["2.7", "3.0", "3.1", "3.2"]
58
+ - RUBY_VERSION: ["3.0", "3.1", "3.2"]
59
59
 
60
60
  rubocop:
61
61
  script:
data/.rubocop_todo.yml CHANGED
@@ -1,19 +1,35 @@
1
1
  # This configuration was generated by
2
2
  # `rubocop --auto-gen-config`
3
- # on 2021-01-11 12:24:03 UTC using RuboCop version 0.93.1.
3
+ # on 2023-09-25 18:54:37 UTC using RuboCop version 1.56.3.
4
4
  # The point is for the user to remove these configuration records
5
5
  # one by one as the offenses are removed from the code base.
6
6
  # Note that changes in the inspected code, or installation of new
7
7
  # versions of RuboCop, may require this file to be generated again.
8
8
 
9
9
  # Offense count: 1
10
- # Configuration parameters: Include.
10
+ # Configuration parameters: Severity, Include.
11
11
  # Include: **/*.gemspec
12
12
  Gemspec/RequiredRubyVersion:
13
13
  Exclude:
14
14
  - 'gitlab-exporter.gemspec'
15
15
 
16
+ # Offense count: 1
17
+ # This cop supports safe autocorrection (--autocorrect).
18
+ # Configuration parameters: EnforcedStyleAlignWith, Severity.
19
+ # SupportedStylesAlignWith: start_of_line, begin
20
+ Layout/BeginEndAlignment:
21
+ Exclude:
22
+ - 'lib/gitlab_exporter/git.rb'
23
+
24
+ # Offense count: 1
25
+ # This cop supports safe autocorrection (--autocorrect).
26
+ Layout/RescueEnsureAlignment:
27
+ Exclude:
28
+ - 'lib/gitlab_exporter/git.rb'
29
+
16
30
  # Offense count: 3
31
+ # This cop supports safe autocorrection (--autocorrect).
32
+ # Configuration parameters: AllowedMethods, AllowedPatterns.
17
33
  Lint/AmbiguousBlockAssociation:
18
34
  Exclude:
19
35
  - 'spec/git_spec.rb'
@@ -27,38 +43,83 @@ Lint/MissingCopEnableDirective:
27
43
  - 'spec/git_process_proper_spec.rb'
28
44
 
29
45
  # Offense count: 1
30
- # Cop supports --auto-correct.
46
+ # This cop supports unsafe autocorrection (--autocorrect-all).
31
47
  Lint/NonDeterministicRequireOrder:
32
48
  Exclude:
33
49
  - 'spec/spec_helper.rb'
34
50
 
35
- # Offense count: 16
36
- # Configuration parameters: CountComments, CountAsOne, ExcludedMethods.
37
- # ExcludedMethods: refine
51
+ # Offense count: 17
52
+ # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
53
+ # AllowedMethods: refine
38
54
  Metrics/BlockLength:
39
- Max: 349
55
+ Max: 285
40
56
 
41
- # Offense count: 1
42
- # Configuration parameters: IgnoredMethods.
57
+ # Offense count: 3
58
+ # Configuration parameters: AllowedMethods, AllowedPatterns.
43
59
  Metrics/CyclomaticComplexity:
44
60
  Max: 8
45
61
 
46
62
  # Offense count: 1
47
- # Configuration parameters: IgnoredMethods.
63
+ # Configuration parameters: AllowedMethods, AllowedPatterns.
48
64
  Metrics/PerceivedComplexity:
49
65
  Max: 9
50
66
 
67
+ # Offense count: 9
68
+ # Configuration parameters: EnforcedStyle, CheckMethodNames, CheckSymbols, AllowedIdentifiers, AllowedPatterns.
69
+ # SupportedStyles: snake_case, normalcase, non_integer
70
+ # AllowedIdentifiers: capture3, iso8601, rfc1123_date, rfc822, rfc2822, rfc3339, x86_64
71
+ Naming/VariableNumber:
72
+ Exclude:
73
+ - 'spec/database/row_count_spec.rb'
74
+
75
+ # Offense count: 16
76
+ # This cop supports safe autocorrection (--autocorrect).
77
+ # Configuration parameters: EnforcedStyle.
78
+ # SupportedStyles: separated, grouped
79
+ Style/AccessorGrouping:
80
+ Exclude:
81
+ - 'lib/gitlab_exporter/memstats/mapping.rb'
82
+
83
+ # Offense count: 8
84
+ # This cop supports unsafe autocorrection (--autocorrect-all).
85
+ Style/GlobalStdStream:
86
+ Exclude:
87
+ - 'lib/gitlab_exporter/cli.rb'
88
+ - 'spec/database/bloat_spec.rb'
89
+
51
90
  # Offense count: 1
52
- # Configuration parameters: EnforcedStyle, AllowModifiersOnSymbols.
53
- # SupportedStyles: inline, group
54
- Style/AccessModifierDeclarations:
91
+ # This cop supports unsafe autocorrection (--autocorrect-all).
92
+ # Configuration parameters: AllowedReceivers.
93
+ # AllowedReceivers: Thread.current
94
+ Style/HashEachMethods:
55
95
  Exclude:
56
96
  - 'lib/gitlab_exporter/util.rb'
57
97
 
98
+ # Offense count: 9
99
+ # Configuration parameters: AllowedMethods.
100
+ # AllowedMethods: respond_to_missing?
101
+ Style/OptionalBooleanParameter:
102
+ Exclude:
103
+ - 'lib/gitlab_exporter/prometheus.rb'
104
+ - 'lib/gitlab_exporter/sidekiq.rb'
105
+
106
+ # Offense count: 1
107
+ # This cop supports safe autocorrection (--autocorrect).
108
+ Style/RedundantRegexpEscape:
109
+ Exclude:
110
+ - 'lib/gitlab_exporter/git.rb'
111
+
58
112
  # Offense count: 2
59
- # Cop supports --auto-correct.
60
- # Configuration parameters: ConvertCodeThatCanStartToReturnNil, AllowedMethods.
113
+ # This cop supports unsafe autocorrection (--autocorrect-all).
114
+ # Configuration parameters: ConvertCodeThatCanStartToReturnNil, AllowedMethods, MaxChainLength.
61
115
  # AllowedMethods: present?, blank?, presence, try, try!
62
116
  Style/SafeNavigation:
63
117
  Exclude:
64
118
  - 'lib/gitlab_exporter/database/base.rb'
119
+
120
+ # Offense count: 1
121
+ # This cop supports unsafe autocorrection (--autocorrect-all).
122
+ # Configuration parameters: Mode.
123
+ Style/StringConcatenation:
124
+ Exclude:
125
+ - 'lib/gitlab_exporter/database/row_count.rb'
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 2.7.7
1
+ 3.2.2
data/Gemfile CHANGED
@@ -4,5 +4,5 @@ gemspec
4
4
 
5
5
  group :test do
6
6
  gem "rspec", "~>3.12"
7
- gem "rubocop", "~>0.42"
7
+ gem "rubocop", "~> 1.52"
8
8
  end
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab-exporter (13.1.0)
4
+ gitlab-exporter (13.3.0)
5
5
  connection_pool (= 2.2.5)
6
6
  faraday (~> 1.8.0)
7
7
  pg (= 1.5.3)
@@ -16,7 +16,8 @@ PATH
16
16
  GEM
17
17
  remote: https://rubygems.org/
18
18
  specs:
19
- ast (2.4.1)
19
+ ast (2.4.2)
20
+ base64 (0.1.1)
20
21
  connection_pool (2.2.5)
21
22
  diff-lcs (1.5.0)
22
23
  faraday (1.8.0)
@@ -38,26 +39,30 @@ GEM
38
39
  faraday-net_http_persistent (1.2.0)
39
40
  faraday-patron (1.0.0)
40
41
  faraday-rack (1.0.0)
42
+ json (2.6.3)
43
+ language_server-protocol (3.17.0.3)
41
44
  multipart-post (2.2.3)
42
45
  mustermann (2.0.2)
43
46
  ruby2_keywords (~> 0.0.1)
44
47
  nio4r (2.5.8)
45
- parallel (1.20.1)
46
- parser (3.0.0.0)
48
+ parallel (1.23.0)
49
+ parser (3.2.2.3)
47
50
  ast (~> 2.4.1)
51
+ racc
48
52
  pg (1.5.3)
49
53
  puma (5.6.5)
50
54
  nio4r (~> 2.0)
51
55
  quantile (0.2.1)
56
+ racc (1.7.1)
52
57
  rack (2.2.5)
53
58
  rack-protection (2.2.4)
54
59
  rack
55
- rainbow (3.0.0)
60
+ rainbow (3.1.1)
56
61
  redis (4.4.0)
57
62
  redis-namespace (1.9.0)
58
63
  redis (>= 4)
59
- regexp_parser (2.0.3)
60
- rexml (3.2.4)
64
+ regexp_parser (2.8.1)
65
+ rexml (3.2.6)
61
66
  rspec (3.12.0)
62
67
  rspec-core (~> 3.12.0)
63
68
  rspec-expectations (~> 3.12.0)
@@ -71,18 +76,21 @@ GEM
71
76
  diff-lcs (>= 1.2.0, < 2.0)
72
77
  rspec-support (~> 3.12.0)
73
78
  rspec-support (3.12.0)
74
- rubocop (0.93.1)
79
+ rubocop (1.56.3)
80
+ base64 (~> 0.1.1)
81
+ json (~> 2.3)
82
+ language_server-protocol (>= 3.17.0)
75
83
  parallel (~> 1.10)
76
- parser (>= 2.7.1.5)
84
+ parser (>= 3.2.2.3)
77
85
  rainbow (>= 2.2.2, < 4.0)
78
- regexp_parser (>= 1.8)
79
- rexml
80
- rubocop-ast (>= 0.6.0)
86
+ regexp_parser (>= 1.8, < 3.0)
87
+ rexml (>= 3.2.5, < 4.0)
88
+ rubocop-ast (>= 1.28.1, < 2.0)
81
89
  ruby-progressbar (~> 1.7)
82
- unicode-display_width (>= 1.4.0, < 2.0)
83
- rubocop-ast (1.4.0)
84
- parser (>= 2.7.1.5)
85
- ruby-progressbar (1.11.0)
90
+ unicode-display_width (>= 2.4.0, < 3.0)
91
+ rubocop-ast (1.29.0)
92
+ parser (>= 3.2.1.0)
93
+ ruby-progressbar (1.13.0)
86
94
  ruby2_keywords (0.0.5)
87
95
  sidekiq (6.4.0)
88
96
  connection_pool (>= 2.2.2)
@@ -94,7 +102,7 @@ GEM
94
102
  rack-protection (= 2.2.4)
95
103
  tilt (~> 2.0)
96
104
  tilt (2.0.11)
97
- unicode-display_width (1.7.0)
105
+ unicode-display_width (2.4.2)
98
106
  webrick (1.7.0)
99
107
 
100
108
  PLATFORMS
@@ -104,7 +112,7 @@ DEPENDENCIES
104
112
  gitlab-exporter!
105
113
  rspec (~> 3.12)
106
114
  rspec-expectations (~> 3.12.0)
107
- rubocop (~> 0.42)
115
+ rubocop (~> 1.52)
108
116
 
109
117
  BUNDLED WITH
110
118
  2.4.5
@@ -95,6 +95,8 @@ probes:
95
95
  opts:
96
96
  redis_url: "redis://localhost:6379"
97
97
  redis_enable_client: true
98
+ probe_namespaced: true
99
+ probe_non_namespaced: true
98
100
 
99
101
  ruby: &ruby
100
102
  class_name: RubyProber
@@ -10,6 +10,7 @@ Gem::Specification.new do |s|
10
10
  s.description = "GitLab metrics exporter to use with prometheus"
11
11
  s.authors = ["Pablo Carranza"]
12
12
  s.email = "pablo@gitlab.com"
13
+ s.required_ruby_version = ">= 3.0.0"
13
14
 
14
15
  s.files = `git ls-files -z`.split("\x0")
15
16
 
@@ -1,19 +1,5 @@
1
1
  require "yaml"
2
2
 
3
- # TODO: Remove this once we're on Ruby 3
4
- # https://gitlab.com/gitlab-org/gitlab/-/issues/393651
5
- unless YAML.respond_to?(:safe_load_file)
6
- module YAML # rubocop:disable Style/Documentation
7
- # Temporary Ruby 2 back-compat workaround.
8
- #
9
- # This method only exists as of stdlib 3.0.0:
10
- # https://ruby-doc.org/stdlib-3.0.0/libdoc/psych/rdoc/Psych.html
11
- def self.safe_load_file(path, **options)
12
- YAML.safe_load(File.read(path), **options)
13
- end
14
- end
15
- end
16
-
17
3
  module GitLab
18
4
  module Exporter
19
5
  # Stores runner classes in a single place
@@ -7,6 +7,7 @@ module GitLab
7
7
  # A prober for Sidekiq queues
8
8
  #
9
9
  # It takes the Redis URL Sidekiq is connected to
10
+ # rubocop:disable Metrics/ClassLength
10
11
  class SidekiqProber
11
12
  # The maximum depth (from the head) of each queue to probe. Probing the
12
13
  # entirety of a very large queue will take longer and run the risk of
@@ -37,35 +38,24 @@ module GitLab
37
38
  @opts = opts
38
39
  @metrics = metrics
39
40
  @logger = logger
41
+
42
+ @probe_namespaced = !!opts[:probe_namespaced] # rubocop:disable Style/DoubleNegation
43
+ @probe_non_namespaced = !!opts[:probe_non_namespaced] # rubocop:disable Style/DoubleNegation
44
+
45
+ # to maintain backward compatibility if the config is missing values
46
+ @probe_namespaced = true if opts[:probe_namespaced].nil? && opts[:probe_non_namespaced].nil?
40
47
  end
41
48
 
42
49
  def probe_stats
43
- with_sidekiq do
44
- stats = Sidekiq::Stats.new
45
-
46
- @metrics.add("sidekiq_jobs_processed_total", stats.processed)
47
- @metrics.add("sidekiq_jobs_failed_total", stats.failed)
48
- @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
49
- @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
50
- @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
51
- @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
52
-
53
- @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
54
- @metrics.add("sidekiq_processes_size", stats.processes_size)
55
- @metrics.add("sidekiq_workers_size", stats.workers_size)
56
- end
50
+ with_sidekiq { _probe_stats } if @probe_namespaced
51
+ with_sidekiq(false) { _probe_stats(false) } if @probe_non_namespaced
57
52
 
58
53
  self
59
54
  end
60
55
 
61
56
  def probe_queues
62
- with_sidekiq do
63
- Sidekiq::Queue.all.each do |queue|
64
- @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
65
- @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
66
- @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
67
- end
68
- end
57
+ with_sidekiq { _probe_queues } if @probe_namespaced
58
+ with_sidekiq(false) { _probe_queues(false) } if @probe_non_namespaced
69
59
 
70
60
  self
71
61
  end
@@ -78,24 +68,8 @@ module GitLab
78
68
  end
79
69
 
80
70
  def probe_future_sets
81
- now = Time.now.to_f
82
- with_sidekiq do
83
- Sidekiq.redis do |conn|
84
- Sidekiq::Scheduled::SETS.each do |set|
85
- # Default to 0; if all jobs are due in the future, there is no "negative" delay.
86
- delay = 0
87
-
88
- _job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
89
- delay = now - timestamp if timestamp
90
-
91
- @metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay)
92
-
93
- # zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
94
- @metrics.add("sidekiq_#{set}_set_backlog_count",
95
- conn.zcount(set, "-inf", now.to_s))
96
- end
97
- end
98
- end
71
+ with_sidekiq { _probe_future_sets } if @probe_namespaced
72
+ with_sidekiq(false) { _probe_future_sets(false) } if @probe_non_namespaced
99
73
  end
100
74
 
101
75
  # Count worker classes present in Sidekiq queues. This only looks at the
@@ -105,57 +79,22 @@ module GitLab
105
79
  # will not have completely accurate statistics, but the probe performance
106
80
  # will also not degrade as the queue gets larger.
107
81
  def probe_jobs_limit
108
- with_sidekiq do
109
- job_stats = Hash.new(0)
110
-
111
- Sidekiq::Queue.all.each do |queue|
112
- Sidekiq.redis do |conn|
113
- conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
114
- job_class = Sidekiq.load_json(job)["class"]
115
-
116
- job_stats[job_class] += 1
117
- end
118
- end
119
- end
120
-
121
- job_stats.each do |class_name, count|
122
- @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
123
- end
124
- end
82
+ with_sidekiq { _probe_jobs_limit } if @probe_namespaced
83
+ with_sidekiq(false) { _probe_jobs_limit(false) } if @probe_non_namespaced
125
84
 
126
85
  self
127
86
  end
128
87
 
129
88
  def probe_workers
130
- with_sidekiq do
131
- worker_stats = Hash.new(0)
132
-
133
- Sidekiq::Workers.new.map do |_pid, _tid, work|
134
- job_klass = work["payload"]["class"]
135
-
136
- worker_stats[job_klass] += 1
137
- end
138
-
139
- worker_stats.each do |class_name, count|
140
- @metrics.add("sidekiq_running_jobs", count, name: class_name)
141
- end
142
- end
89
+ with_sidekiq { _probe_workers } if @probe_namespaced
90
+ with_sidekiq(false) { _probe_workers(false) } if @probe_non_namespaced
143
91
 
144
92
  self
145
93
  end
146
94
 
147
95
  def probe_retries
148
- with_sidekiq do
149
- retry_stats = Hash.new(0)
150
-
151
- Sidekiq::RetrySet.new.map do |job|
152
- retry_stats[job.klass] += 1
153
- end
154
-
155
- retry_stats.each do |class_name, count|
156
- @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
157
- end
158
- end
96
+ with_sidekiq { _probe_retries } if @probe_namespaced
97
+ with_sidekiq(false) { _probe_retries(false) } if @probe_non_namespaced
159
98
 
160
99
  self
161
100
  end
@@ -177,12 +116,106 @@ module GitLab
177
116
 
178
117
  private
179
118
 
180
- def with_sidekiq
119
+ def _probe_workers(namespace = true)
120
+ labels = add_namespace_labels? ? { namespaced: namespace } : {}
121
+ worker_stats = Hash.new(0)
122
+
123
+ Sidekiq::Workers.new.map do |_pid, _tid, work|
124
+ job_klass = work["payload"]["class"]
125
+
126
+ worker_stats[job_klass] += 1
127
+ end
128
+
129
+ worker_stats.each do |class_name, count|
130
+ @metrics.add("sidekiq_running_jobs", count, **labels.merge({ name: class_name }))
131
+ end
132
+ end
133
+
134
+ def _probe_future_sets(namespace = true)
135
+ labels = add_namespace_labels? ? { namespaced: namespace } : {}
136
+ now = Time.now.to_f
137
+ Sidekiq.redis do |conn|
138
+ Sidekiq::Scheduled::SETS.each do |set|
139
+ # Default to 0; if all jobs are due in the future, there is no "negative" delay.
140
+ delay = 0
141
+
142
+ _job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
143
+ delay = now - timestamp if timestamp
144
+
145
+ @metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay, **labels)
146
+
147
+ # zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
148
+ @metrics.add("sidekiq_#{set}_set_backlog_count",
149
+ conn.zcount(set, "-inf", now.to_s), **labels)
150
+ end
151
+ end
152
+ end
153
+
154
+ def _probe_stats(namespace = true)
155
+ stats = Sidekiq::Stats.new
156
+ labels = add_namespace_labels? ? { namespaced: namespace } : {}
157
+
158
+ @metrics.add("sidekiq_jobs_processed_total", stats.processed, **labels)
159
+ @metrics.add("sidekiq_jobs_failed_total", stats.failed, **labels)
160
+ @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued, **labels)
161
+ @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size, **labels)
162
+ @metrics.add("sidekiq_jobs_retry_size", stats.retry_size, **labels)
163
+ @metrics.add("sidekiq_jobs_dead_size", stats.dead_size, **labels)
164
+
165
+ @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency, **labels)
166
+ @metrics.add("sidekiq_processes_size", stats.processes_size, **labels)
167
+ @metrics.add("sidekiq_workers_size", stats.workers_size, **labels)
168
+ end
169
+
170
+ def _probe_queues(namespace = true)
171
+ Sidekiq::Queue.all.each do |queue|
172
+ labels = { name: queue.name }
173
+ labels[:namespaced] = namespace if add_namespace_labels?
174
+
175
+ @metrics.add("sidekiq_queue_size", queue.size, **labels)
176
+ @metrics.add("sidekiq_queue_latency_seconds", queue.latency, **labels)
177
+ @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, **labels)
178
+ end
179
+ end
180
+
181
+ def _probe_jobs_limit(namespace = true)
182
+ labels = add_namespace_labels? ? { namespaced: namespace } : {}
183
+ job_stats = Hash.new(0)
184
+
185
+ Sidekiq::Queue.all.each do |queue|
186
+ Sidekiq.redis do |conn|
187
+ conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
188
+ job_class = Sidekiq.load_json(job)["class"]
189
+
190
+ job_stats[job_class] += 1
191
+ end
192
+ end
193
+ end
194
+
195
+ job_stats.each do |class_name, count|
196
+ @metrics.add("sidekiq_enqueued_jobs", count, **labels.merge({ name: class_name }))
197
+ end
198
+ end
199
+
200
+ def _probe_retries(namespace = true)
201
+ labels = add_namespace_labels? ? { namespaced: namespace } : {}
202
+ retry_stats = Hash.new(0)
203
+
204
+ Sidekiq::RetrySet.new.map do |job|
205
+ retry_stats[job.klass] += 1
206
+ end
207
+
208
+ retry_stats.each do |class_name, count|
209
+ @metrics.add("sidekiq_to_be_retried_jobs", count, **labels.merge({ name: class_name }))
210
+ end
211
+ end
212
+
213
+ def with_sidekiq(namespaced = true)
181
214
  # TODO: this is not concurrent safe as we change global context
182
215
  # It means that we are unable to use many different sidekiq's
183
216
  # which is not a problem as of now
184
217
  Sidekiq.configure_client do |config|
185
- config.redis = self.class.connection_pool[redis_options]
218
+ config.redis = self.class.connection_pool[redis_options(namespaced)]
186
219
  end
187
220
 
188
221
  return unless connected?
@@ -190,14 +223,14 @@ module GitLab
190
223
  yield
191
224
  end
192
225
 
193
- def redis_options
226
+ def redis_options(namespaced = true)
194
227
  options = {
195
228
  url: @opts[:redis_url],
196
- namespace: "resque:gitlab",
197
229
  connect_timeout: 1,
198
230
  reconnect_attempts: 0
199
231
  }
200
232
 
233
+ options[:namespace] = "resque:gitlab" if namespaced
201
234
  options[:id] = nil unless redis_enable_client?
202
235
  options
203
236
  end
@@ -218,6 +251,11 @@ module GitLab
218
251
  @logger&.error "Error connecting to the Redis: #{e}"
219
252
  @connected = false
220
253
  end
254
+
255
+ def add_namespace_labels?
256
+ @probe_namespaced && @probe_non_namespaced
257
+ end
221
258
  end
259
+ # rubocop:enable Metrics/ClassLength
222
260
  end
223
261
  end
@@ -1,5 +1,5 @@
1
1
  module GitLab
2
2
  module Exporter
3
- VERSION = "13.1.0".freeze
3
+ VERSION = "13.3.0".freeze
4
4
  end
5
5
  end
@@ -0,0 +1,64 @@
1
+ require "spec_helper"
2
+ require "gitlab_exporter/sidekiq"
3
+
4
+ describe GitLab::Exporter::SidekiqProber do
5
+ let(:probe_list) { %w[probe_stats probe_queues probe_future_sets probe_jobs_limit probe_workers probe_retries] }
6
+ let(:opt) { {} }
7
+ let(:probe) { described_class.new(**opt) }
8
+
9
+ before do
10
+ # stub connection as we want check if the probe methods are called
11
+ # with the right namespace argument
12
+ allow(probe).to receive(:connected?).and_return(true)
13
+ end
14
+
15
+ context "when no namespace options are defined" do
16
+ it "defaults to probing non-namespaced keys only" do
17
+ probe_list.each do |probe_type|
18
+ expect(probe).not_to receive("_#{probe_type}".to_sym).with(false)
19
+ expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
20
+
21
+ probe.send(probe_type)
22
+ end
23
+ end
24
+ end
25
+
26
+ context "when probing both namespaces" do
27
+ let(:opt) { { probe_non_namespaced: true, probe_namespaced: true } }
28
+
29
+ it "probes both namespaces" do
30
+ probe_list.each do |probe_type|
31
+ expect(probe).to receive("_#{probe_type}".to_sym).with(false)
32
+ expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
33
+
34
+ probe.send(probe_type)
35
+ end
36
+ end
37
+ end
38
+
39
+ context "when probing non-namespaced only" do
40
+ let(:opt) { { probe_non_namespaced: true } }
41
+
42
+ it "probes non-namespaced only" do
43
+ probe_list.each do |probe_type|
44
+ expect(probe).to receive("_#{probe_type}".to_sym).with(false)
45
+ expect(probe).not_to receive("_#{probe_type}".to_sym).with(no_args)
46
+
47
+ probe.send(probe_type)
48
+ end
49
+ end
50
+ end
51
+
52
+ context "when probing namespace only" do
53
+ let(:opt) { { probe_namespaced: true } }
54
+
55
+ it "probes namespaced only" do
56
+ probe_list.each do |probe_type|
57
+ expect(probe).not_to receive("_#{probe_type}".to_sym).with(false)
58
+ expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
59
+
60
+ probe.send(probe_type)
61
+ end
62
+ end
63
+ end
64
+ end
metadata CHANGED
@@ -1,11 +1,11 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-exporter
3
3
  version: !ruby/object:Gem::Version
4
- version: 13.1.0
4
+ version: 13.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Carranza
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
  date: 2016-07-27 00:00:00.000000000 Z
@@ -235,13 +235,14 @@ files:
235
235
  - spec/memstats_spec.rb
236
236
  - spec/prometheus_metrics_spec.rb
237
237
  - spec/ruby_spec.rb
238
+ - spec/sidekiq_spec.rb
238
239
  - spec/spec_helper.rb
239
240
  - spec/util_spec.rb
240
241
  homepage: http://gitlab.com
241
242
  licenses:
242
243
  - MIT
243
244
  metadata: {}
244
- post_install_message:
245
+ post_install_message:
245
246
  rdoc_options: []
246
247
  require_paths:
247
248
  - lib
@@ -249,15 +250,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
249
250
  requirements:
250
251
  - - ">="
251
252
  - !ruby/object:Gem::Version
252
- version: '0'
253
+ version: 3.0.0
253
254
  required_rubygems_version: !ruby/object:Gem::Requirement
254
255
  requirements:
255
256
  - - ">="
256
257
  - !ruby/object:Gem::Version
257
258
  version: '0'
258
259
  requirements: []
259
- rubygems_version: 3.1.6
260
- signing_key:
260
+ rubygems_version: 3.4.10
261
+ signing_key:
261
262
  specification_version: 4
262
263
  summary: GitLab metrics exporter
263
264
  test_files:
@@ -273,5 +274,6 @@ test_files:
273
274
  - spec/memstats_spec.rb
274
275
  - spec/prometheus_metrics_spec.rb
275
276
  - spec/ruby_spec.rb
277
+ - spec/sidekiq_spec.rb
276
278
  - spec/spec_helper.rb
277
279
  - spec/util_spec.rb