gitlab-exporter 9.0.0 → 10.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b3fb1c9119047242e695cb956c7a68f7c19f564f7a274c8d503b7d4b984eed03
4
- data.tar.gz: fb77e078e08dbc9f8f81a1a82dc070944fbf49af3944c90a8e30eac59ca1c39a
3
+ metadata.gz: 8e5ca8b32e5bce057dbfd998d16e3bc940440115a7540d44e4ab6a4c616c23b3
4
+ data.tar.gz: 616a825e49b9a0aee99f0684ec7df670c3ad1d09a2b182e317037ac30bca031a
5
5
  SHA512:
6
- metadata.gz: 0c75097275454229a38f351618bc7eef530534726fa16d3d06b1c8fb95d5fdacc43a84d4d9781963bf27b8afc7700699bfe32d5f0f8516403edf7d5fa5a6555a
7
- data.tar.gz: 1eb96b35d5a80166dc5f587dce7de896d7533cf5822d5a17b220c0d09d858b01e5d02cfc7bbd8710b6002591efd9181cfab063b8fa8bfee698beca672536e83a
6
+ metadata.gz: a346775749dce1e4e08311d42e7988dc470a46f1b52379b982bc11376b70ebf81fa4a8090447743a5624e16ec3377b638204b2d31bd4e41f2c175d8fdbe51336
7
+ data.tar.gz: d1fb453077499d6de7ea8fd434e839f390ce3052f4f50fc17c0ee041ed90dcafd83e0e0a9a16ad7a50ce862752701be5a2eecc39b5306458f8feb08b293c3868
data/Gemfile.lock CHANGED
@@ -1,9 +1,10 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab-exporter (9.0.0)
4
+ gitlab-exporter (10.3.0)
5
5
  connection_pool (~> 2.2.1)
6
6
  pg (~> 1.1)
7
+ puma (~> 5.3.2)
7
8
  quantile (~> 0.2.0)
8
9
  redis (~> 4.1.2)
9
10
  redis-namespace (~> 1.6.0)
@@ -14,14 +15,17 @@ GEM
14
15
  remote: https://rubygems.org/
15
16
  specs:
16
17
  ast (2.4.1)
17
- connection_pool (2.2.3)
18
+ connection_pool (2.2.5)
18
19
  diff-lcs (1.3)
19
20
  mustermann (1.1.1)
20
21
  ruby2_keywords (~> 0.0.1)
22
+ nio4r (2.5.7)
21
23
  parallel (1.20.1)
22
24
  parser (3.0.0.0)
23
25
  ast (~> 2.4.1)
24
26
  pg (1.2.3)
27
+ puma (5.3.2)
28
+ nio4r (~> 2.0)
25
29
  quantile (0.2.1)
26
30
  rack (2.2.3)
27
31
  rack-protection (2.0.8.1)
@@ -57,7 +61,7 @@ GEM
57
61
  rubocop-ast (1.4.0)
58
62
  parser (>= 2.7.1.5)
59
63
  ruby-progressbar (1.11.0)
60
- ruby2_keywords (0.0.2)
64
+ ruby2_keywords (0.0.4)
61
65
  sidekiq (5.2.9)
62
66
  connection_pool (~> 2.2, >= 2.2.2)
63
67
  rack (~> 2.0)
data/README.md CHANGED
@@ -33,23 +33,6 @@ metrics.
33
33
  * [git pull/push timings](lib/gitlab_exporter/git.rb) --
34
34
  `git_pull_time_milliseconds`, `git_push_time_milliseconds`
35
35
  * git processes stats (see Process below)
36
- 1. [Process](lib/gitlab_exporter/process.rb)
37
- * CPU time -- `process_cpu_seconds_total`
38
- * Start time -- `process_start_time_seconds`
39
- * Count -- `process_count`
40
- * Memory usage
41
- * Data from /proc/<pid>/cmdline:
42
- * `process_resident_memory_bytes`
43
- * `process_virtual_memory_bytes`
44
- * Data from /proc/<pid>/smaps -- `probe_smaps` (off by default):
45
- * `process_smaps_size_bytes`
46
- * `process_smaps_rss_bytes`
47
- * `process_smaps_shared_clean_bytes`
48
- * `process_smaps_shared_dirty_bytes`
49
- * `process_smaps_private_clean_bytes`
50
- * `process_smaps_private_dirty_bytes`
51
- * `process_smaps_swap_bytes`
52
- * `process_smaps_pss_bytes`
53
36
  1. [Sidekiq](lib/gitlab_exporter/sidekiq.rb)
54
37
  * Stats
55
38
  * `sidekiq_jobs_processed_total`
data/bin/gitlab-exporter CHANGED
@@ -2,6 +2,9 @@
2
2
 
3
3
  $LOAD_PATH.unshift File.expand_path("../lib", __dir__)
4
4
 
5
+ require "rubygems"
6
+ require "bundler"
7
+ require "bundler/setup"
5
8
  require "optparse"
6
9
  require "gitlab_exporter"
7
10
 
@@ -46,7 +46,6 @@ probes:
46
46
  <<: *db_common
47
47
  opts:
48
48
  <<: *db_common_opts
49
- allowed_repeated_commands_count: 2
50
49
  created_builds_counting_disabled: true
51
50
  unarchived_traces_offset_minutes: 1440
52
51
  tuple_stats:
@@ -95,12 +94,21 @@ probes:
95
94
  redis_url: "redis://localhost:6379"
96
95
  redis_enable_client: true
97
96
 
97
+ ruby: &ruby
98
+ class_name: RubyProber
99
+ methods:
100
+ - probe_gc
101
+ opts:
102
+ quantiles: false
103
+
98
104
  metrics:
99
105
  multiple: true
100
106
  git_process:
101
107
  <<: *git_process
102
108
  process:
103
109
  <<: *process
110
+ ruby:
111
+ <<: *ruby
104
112
  sidekiq:
105
113
  <<: *sidekiq
106
114
  ci_builds:
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
22
22
 
23
23
  s.add_runtime_dependency "connection_pool", "~> 2.2.1"
24
24
  s.add_runtime_dependency "pg", "~> 1.1"
25
+ s.add_runtime_dependency "puma", "~> 5.3.2"
25
26
  s.add_runtime_dependency "quantile", "~> 0.2.0"
26
27
  s.add_runtime_dependency "redis", "~> 4.1.2"
27
28
  s.add_runtime_dependency "redis-namespace", "~> 1.6.0"
@@ -14,5 +14,6 @@ module GitLab
14
14
  autoload :WebExporter, "gitlab_exporter/web_exporter"
15
15
  autoload :Prober, "gitlab_exporter/prober"
16
16
  autoload :SidekiqProber, "gitlab_exporter/sidekiq"
17
+ autoload :RubyProber, "gitlab_exporter/ruby"
17
18
  end
18
19
  end
@@ -8,14 +8,39 @@ module GitLab
8
8
  #
9
9
  # It takes a connection string (e.g. "dbname=test port=5432")
10
10
  class Base
11
+ POOL_SIZE = 3
12
+
13
+ # This timeout is configured to higher interval than scrapping
14
+ # of Prometheus to ensure that connection is kept instead of
15
+ # needed to be re-initialized
16
+ POOL_TIMEOUT = 90
17
+
11
18
  def self.connection_pool
12
- @connection_pool ||= Hash.new do |h, connection_string|
13
- h[connection_string] = ConnectionPool.new(size: 3, timeout: 5) do
14
- PG.connect(connection_string)
19
+ @@connection_pool ||= Hash.new do |h, connection_string| # rubocop:disable Style/ClassVars
20
+ h[connection_string] = ConnectionPool.new(size: POOL_SIZE, timeout: POOL_TIMEOUT) do
21
+ PG.connect(connection_string).tap do |conn|
22
+ configure_type_map_for_results(conn)
23
+ end
15
24
  end
16
25
  end
17
26
  end
18
27
 
28
+ def self.configure_type_map_for_results(conn)
29
+ tm = PG::BasicTypeMapForResults.new(conn)
30
+
31
+ # Remove warning message:
32
+ # Warning: no type cast defined for type "name" with oid 19.
33
+ # Please cast this type explicitly to TEXT to be safe for future changes.
34
+ # Warning: no type cast defined for type "regproc" with oid 24.
35
+ # Please cast this type explicitly to TEXT to be safe for future changes.
36
+ [{ "type": "text", "oid": 19 }, { "type": "int4", "oid": 24 }].each do |value|
37
+ old_coder = tm.coders.find { |c| c.name == value[:type] }
38
+ tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
39
+ end
40
+
41
+ conn.type_map_for_results = tm
42
+ end
43
+
19
44
  def initialize(args, logger: nil)
20
45
  @connection_string = args[:connection_string]
21
46
  @logger = logger
@@ -147,100 +147,6 @@ module GitLab
147
147
  SELECT COUNT(*) FROM licenses
148
148
  SQL
149
149
 
150
- REPEATED_COMMANDS_QUERY_EE =
151
- <<~SQL.freeze
152
- SELECT
153
- subquery.namespace_id,
154
- subquery.shared_runners_enabled,
155
- subquery.project_id,
156
- subquery.status,
157
- subquery.has_minutes,
158
- MAX(subquery.count) as count
159
- FROM (
160
- SELECT
161
- projects.namespace_id,
162
- projects.shared_runners_enabled,
163
- ci_builds.project_id,
164
- ci_builds.commit_id,
165
- ci_builds.status,
166
- (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
167
- COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
168
- COUNT(*) AS count
169
- FROM ci_builds
170
- JOIN projects
171
- ON projects.id = ci_builds.project_id
172
- JOIN namespaces
173
- ON namespaces.id = projects.namespace_id
174
- LEFT JOIN namespace_statistics
175
- ON namespace_statistics.namespace_id = namespaces.id
176
- JOIN application_settings ON (TRUE)
177
- WHERE ci_builds.type = 'Ci::Build'
178
- AND ci_builds.status IN ('running', 'pending')
179
- -- The created_at filter has been introduced for performance reasons only
180
- AND ci_builds.created_at > NOW() - INTERVAL '7 days'
181
- GROUP BY
182
- projects.namespace_id,
183
- projects.shared_runners_enabled,
184
- ci_builds.project_id,
185
- ci_builds.commit_id,
186
- ci_builds.status,
187
- ci_builds.commands,
188
- namespaces.shared_runners_minutes_limit,
189
- namespace_statistics.shared_runners_seconds,
190
- application_settings.shared_runners_minutes
191
- HAVING COUNT(*) > %d
192
- ) AS subquery
193
- GROUP BY
194
- subquery.namespace_id,
195
- subquery.shared_runners_enabled,
196
- subquery.project_id,
197
- subquery.commit_id,
198
- subquery.status,
199
- subquery.has_minutes
200
- SQL
201
-
202
- REPEATED_COMMANDS_QUERY_CE =
203
- <<~SQL.freeze
204
- SELECT
205
- subquery.namespace_id,
206
- subquery.shared_runners_enabled,
207
- subquery.project_id,
208
- subquery.status,
209
- MAX(subquery.count) as count
210
- FROM (
211
- SELECT
212
- projects.namespace_id,
213
- projects.shared_runners_enabled,
214
- ci_builds.project_id,
215
- ci_builds.commit_id,
216
- ci_builds.status,
217
- COUNT(*) AS count
218
- FROM ci_builds
219
- JOIN projects
220
- ON projects.id = ci_builds.project_id
221
- JOIN namespaces
222
- ON namespaces.id = projects.namespace_id
223
- WHERE ci_builds.type = 'Ci::Build'
224
- AND ci_builds.status IN ('running', 'pending')
225
- -- The created_at filter has been introduced for performance reasons only
226
- AND ci_builds.created_at > NOW() - INTERVAL '7 days'
227
- GROUP BY
228
- projects.namespace_id,
229
- projects.shared_runners_enabled,
230
- ci_builds.project_id,
231
- ci_builds.commit_id,
232
- ci_builds.status,
233
- ci_builds.commands
234
- HAVING COUNT(*) > %d
235
- ) AS subquery
236
- GROUP BY
237
- subquery.namespace_id,
238
- subquery.shared_runners_enabled,
239
- subquery.project_id,
240
- subquery.commit_id,
241
- subquery.status
242
- SQL
243
-
244
150
  UNARCHIVED_TRACES_QUERY =
245
151
  <<~SQL.freeze
246
152
  SELECT
@@ -265,7 +171,6 @@ module GitLab
265
171
  def initialize(opts, logger: nil)
266
172
  super(opts, logger: logger)
267
173
 
268
- @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
269
174
  @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
270
175
  @unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes]
271
176
  end
@@ -276,7 +181,6 @@ module GitLab
276
181
  results[:pending_builds] = builds(STATUS_PENDING)
277
182
  results[:stale_builds] = stale_builds
278
183
  results[:per_runner] = per_runner_builds
279
- results[:repeated_commands] = repeated_commands
280
184
  results[:unarchived_traces] = unarchived_traces
281
185
  results
282
186
  end
@@ -335,34 +239,6 @@ module GitLab
335
239
  include_ee_fields(values, row)
336
240
  end
337
241
 
338
- def repeated_commands
339
- results = []
340
-
341
- query = ee? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
342
- query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
343
- exec_query_with_custom_random_page_cost(query).each do |row|
344
- results << transform_repeated_commands_row_to_values(row)
345
- end
346
-
347
- results
348
- rescue PG::UndefinedTable, PG::UndefinedColumn
349
- []
350
- end
351
-
352
- def allowed_repeated_commands_count
353
- @allowed_repeated_commands_count ||= 2
354
- end
355
-
356
- def transform_repeated_commands_row_to_values(row)
357
- values = { namespace: row["namespace_id"].to_s,
358
- project: row["project_id"].to_s,
359
- shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
360
- status: row["status"].to_s,
361
- value: row["count"].to_i }
362
-
363
- include_has_minutes_field(values, row)
364
- end
365
-
366
242
  def unarchived_traces
367
243
  time = Time.now - (unarchived_traces_offset_minutes * 60)
368
244
  query = UNARCHIVED_TRACES_QUERY % [time.strftime("%F %T")] # rubocop:disable Style/FormatString
@@ -422,7 +298,6 @@ module GitLab
422
298
  @metrics = metrics
423
299
 
424
300
  collector_opts = { connection_string: opts[:connection_string],
425
- allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
426
301
  created_builds_counting_disabled: opts[:created_builds_counting_disabled],
427
302
  unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
428
303
  @collector = CiBuildsCollector.new(collector_opts, logger: logger)
@@ -435,7 +310,6 @@ module GitLab
435
310
  ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
436
311
  ci_stale_builds_metrics
437
312
  metrics_per_runner
438
- repeated_commands_metrics
439
313
  unarchived_traces_metrics
440
314
 
441
315
  self
@@ -520,14 +394,6 @@ module GitLab
520
394
  @metrics.add(metric_name, value.to_f, selected_labels)
521
395
  end
522
396
 
523
- def repeated_commands_metrics
524
- @results[:repeated_commands].each do |metric|
525
- value = metric.delete(:value)
526
-
527
- @metrics.add("ci_repeated_commands_builds", value.to_f, metric)
528
- end
529
- end
530
-
531
397
  def unarchived_traces_metrics
532
398
  @metrics.add("ci_unarchived_traces", @results[:unarchived_traces].to_f)
533
399
  end
@@ -175,30 +175,12 @@ module GitLab
175
175
 
176
176
  def execute(query)
177
177
  with_connection_pool do |conn|
178
- conn.exec(query).map_types!(type_map_for_results(conn))
178
+ conn.exec(query)
179
179
  end
180
180
  rescue PG::UndefinedTable, PG::UndefinedColumn
181
181
  nil
182
182
  end
183
183
 
184
- def type_map_for_results(conn)
185
- @type_map_for_results ||= begin
186
- tm = PG::BasicTypeMapForResults.new(conn)
187
-
188
- # Remove warning message:
189
- # Warning: no type cast defined for type "name" with oid 19.
190
- # Please cast this type explicitly to TEXT to be safe for future changes.
191
- # Warning: no type cast defined for type "regproc" with oid 24.
192
- # Please cast this type explicitly to TEXT to be safe for future changes.
193
- [{ "type": "text", "oid": 19 }, { "type": "int4", "oid": 24 }].each do |value|
194
- old_coder = tm.coders.find { |c| c.name == value[:type] }
195
- tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
196
- end
197
-
198
- tm
199
- end
200
- end
201
-
202
184
  # Not private so I can test it without meta programming tricks
203
185
  def construct_query(query)
204
186
  query_string = "SELECT COUNT(*)"
@@ -93,6 +93,10 @@ module GitLab
93
93
  end
94
94
 
95
95
  def probe_git # rubocop:disable Metrics/MethodLength
96
+ puts "[DEPRECATED] probe_git and GitProcessProber are now considered obsolete"\
97
+ " and will be removed in future major versions,"\
98
+ " please use git metrics produced by Gitaly instead"
99
+
96
100
  counts = Hash.new(0)
97
101
 
98
102
  Utils.pgrep("^git ").each do |pid|
@@ -71,6 +71,10 @@ module GitLab
71
71
  end
72
72
 
73
73
  def probe_stat
74
+ puts "[DEPRECATED] probe_stat and ProcessProber are now considered obsolete"\
75
+ " and will be removed in future major versions,"\
76
+ " please use metrics produced by application processes instead"
77
+
74
78
  @pids.each do |pid|
75
79
  stats = ProcessStats.new(pid)
76
80
  next unless stats.valid?
@@ -88,12 +92,20 @@ module GitLab
88
92
  end
89
93
 
90
94
  def probe_count
95
+ puts "[DEPRECATED] probe_count and ProcessProber are now considered obsolete"\
96
+ " and will be removed in future major versions,"\
97
+ " please use metrics produced by application processes instead"
98
+
91
99
  @metrics.add("process_count", @pids.count, name: @name.downcase)
92
100
 
93
101
  self
94
102
  end
95
103
 
96
104
  def probe_smaps
105
+ puts "[DEPRECATED] probe_smaps and ProcessProber are now considered obsolete"\
106
+ " and will be removed in future major versions,"\
107
+ " please use metrics produced by application processes instead"
108
+
97
109
  @pids.each do |pid|
98
110
  stats = ::GitLab::Exporter::MemStats::Aggregator.new(pid)
99
111
 
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module GitLab
4
+ module Exporter
5
+ # Probes a current process GC for info then writes metrics to a target
6
+ class RubyProber
7
+ def initialize(options, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
8
+ @metrics = metrics
9
+ @use_quantiles = options.fetch(:quantiles, false)
10
+ end
11
+
12
+ def probe_gc
13
+ GC.stat.each do |stat, value|
14
+ @metrics.add("ruby_gc_stat_#{stat}", value.to_i, @use_quantiles)
15
+ end
16
+
17
+ self
18
+ end
19
+ end
20
+ end
21
+ end
@@ -10,100 +10,109 @@ module GitLab
10
10
  QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
11
11
  QUEUE_JOB_STATS_SHA = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
12
12
 
13
+ POOL_SIZE = 3
14
+
15
+ # This timeout is configured to higher interval than scrapping
16
+ # of Prometheus to ensure that connection is kept instead of
17
+ # needed to be re-initialized
18
+ POOL_TIMEOUT = 90
19
+
20
+ def self.connection_pool
21
+ @@connection_pool ||= Hash.new do |h, connection_hash| # rubocop:disable Style/ClassVars
22
+ config = connection_hash.merge(pool_timeout: POOL_TIMEOUT, size: POOL_SIZE)
23
+
24
+ h[connection_hash] = Sidekiq::RedisConnection.create(config)
25
+ end
26
+ end
27
+
13
28
  def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
14
29
  @opts = opts
15
30
  @metrics = metrics
16
31
  @logger = logger
17
-
18
- Sidekiq.configure_client do |config|
19
- config.redis = redis_options
20
- end
21
-
22
- ensure_queue_job_stats_script_loaded
23
32
  end
24
33
 
25
34
  def probe_stats
26
- return self unless connected?
27
-
28
- stats = Sidekiq::Stats.new
29
-
30
- @metrics.add("sidekiq_jobs_processed_total", stats.processed)
31
- @metrics.add("sidekiq_jobs_failed_total", stats.failed)
32
- @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
33
- @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
34
- @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
35
- @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
36
-
37
- @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
38
- @metrics.add("sidekiq_processes_size", stats.processes_size)
39
- @metrics.add("sidekiq_workers_size", stats.workers_size)
35
+ with_sidekiq do
36
+ stats = Sidekiq::Stats.new
37
+
38
+ @metrics.add("sidekiq_jobs_processed_total", stats.processed)
39
+ @metrics.add("sidekiq_jobs_failed_total", stats.failed)
40
+ @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
41
+ @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
42
+ @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
43
+ @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
44
+
45
+ @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
46
+ @metrics.add("sidekiq_processes_size", stats.processes_size)
47
+ @metrics.add("sidekiq_workers_size", stats.workers_size)
48
+ end
40
49
 
41
50
  self
42
51
  end
43
52
 
44
53
  def probe_queues
45
- return self unless connected?
46
-
47
- Sidekiq::Queue.all.each do |queue|
48
- @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
49
- @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
50
- @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
54
+ with_sidekiq do
55
+ Sidekiq::Queue.all.each do |queue|
56
+ @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
57
+ @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
58
+ @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
59
+ end
51
60
  end
52
61
 
53
62
  self
54
63
  end
55
64
 
56
65
  def probe_jobs
57
- return self unless connected?
58
-
59
- job_stats = {}
60
-
61
- Sidekiq::Queue.all.each do |queue|
62
- Sidekiq.redis do |conn|
63
- stats = conn.evalsha(QUEUE_JOB_STATS_SHA, ["queue:#{queue.name}"])
64
- job_stats.merge!(stats.to_h)
66
+ with_sidekiq do
67
+ job_stats = {}
68
+
69
+ Sidekiq::Queue.all.each do |queue|
70
+ Sidekiq.redis do |conn|
71
+ stats = conn.evalsha(QUEUE_JOB_STATS_SHA, ["queue:#{queue.name}"])
72
+ job_stats.merge!(stats.to_h)
73
+ end
74
+ rescue Redis::CommandError # Could happen if the script exceeded the maximum run time (5 seconds by default)
75
+ # FIXME: Should we call SCRIPT KILL?
76
+ return self
65
77
  end
66
- rescue Redis::CommandError # Could happen if the script exceeded the maximum run time (5 seconds by default)
67
- # FIXME: Should we call SCRIPT KILL?
68
- return self
69
- end
70
78
 
71
- job_stats.each do |class_name, count|
72
- @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
79
+ job_stats.each do |class_name, count|
80
+ @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
81
+ end
73
82
  end
74
83
 
75
84
  self
76
85
  end
77
86
 
78
87
  def probe_workers
79
- return self unless connected?
80
-
81
- worker_stats = Hash.new(0)
88
+ with_sidekiq do
89
+ worker_stats = Hash.new(0)
82
90
 
83
- Sidekiq::Workers.new.map do |_pid, _tid, work|
84
- job_klass = work["payload"]["class"]
91
+ Sidekiq::Workers.new.map do |_pid, _tid, work|
92
+ job_klass = work["payload"]["class"]
85
93
 
86
- worker_stats[job_klass] += 1
87
- end
94
+ worker_stats[job_klass] += 1
95
+ end
88
96
 
89
- worker_stats.each do |class_name, count|
90
- @metrics.add("sidekiq_running_jobs", count, name: class_name)
97
+ worker_stats.each do |class_name, count|
98
+ @metrics.add("sidekiq_running_jobs", count, name: class_name)
99
+ end
91
100
  end
92
101
 
93
102
  self
94
103
  end
95
104
 
96
105
  def probe_retries
97
- return self unless connected?
106
+ with_sidekiq do
107
+ retry_stats = Hash.new(0)
98
108
 
99
- retry_stats = Hash.new(0)
100
-
101
- Sidekiq::RetrySet.new.map do |job|
102
- retry_stats[job.klass] += 1
103
- end
109
+ Sidekiq::RetrySet.new.map do |job|
110
+ retry_stats[job.klass] += 1
111
+ end
104
112
 
105
- retry_stats.each do |class_name, count|
106
- @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
113
+ retry_stats.each do |class_name, count|
114
+ @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
115
+ end
107
116
  end
108
117
 
109
118
  self
@@ -113,9 +122,9 @@ module GitLab
113
122
  puts "[DEPRECATED] probe_dead is now considered obsolete and will be removed in future major versions,"\
114
123
  " please use probe_stats instead"
115
124
 
116
- return self unless connected?
117
-
118
- @metrics.add("sidekiq_dead_jobs", Sidekiq::Stats.new.dead_size)
125
+ with_sidekiq do
126
+ @metrics.add("sidekiq_dead_jobs", Sidekiq::Stats.new.dead_size)
127
+ end
119
128
 
120
129
  self
121
130
  end
@@ -126,6 +135,19 @@ module GitLab
126
135
 
127
136
  private
128
137
 
138
+ def with_sidekiq
139
+ # TODO: this is not concurrent safe as we change global context
140
+ # It means that we are unable to use many different sidekiq's
141
+ # which is not a problem as of now
142
+ Sidekiq.configure_client do |config|
143
+ config.redis = self.class.connection_pool[redis_options]
144
+ end
145
+
146
+ return unless connected?
147
+
148
+ yield
149
+ end
150
+
129
151
  def redis_options
130
152
  options = {
131
153
  url: @opts[:redis_url],
@@ -145,26 +167,19 @@ module GitLab
145
167
  end
146
168
 
147
169
  def connected?
148
- @connected ||= begin
149
- Sidekiq.redis do |conn|
150
- conn.get("foo")
151
- end
152
- true
153
- end
154
- rescue Redis::CannotConnectError, Redis::TimeoutError
155
- # Maybe we're trying connecting to a slave
156
- end
157
-
158
- def ensure_queue_job_stats_script_loaded
159
- return unless connected?
170
+ return @connected unless @connected.nil?
160
171
 
172
+ # This is also a good "connected check"
161
173
  Sidekiq.redis do |conn|
162
174
  # Using administrative commands on conn directly (which is a Redis::Namespace)
163
175
  # will be removed in redis-namespace 2.0.
164
- next if conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
165
-
166
- conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT)
176
+ conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT) unless conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
167
177
  end
178
+
179
+ @connected = true
180
+ rescue Redis::BaseConnectionError => e
181
+ @logger&.error "Error connecting to the Redis: #{e}"
182
+ @connected = false
168
183
  end
169
184
  end
170
185
  end
@@ -1,5 +1,5 @@
1
1
  module GitLab
2
2
  module Exporter
3
- VERSION = "9.0.0".freeze
3
+ VERSION = "10.3.0".freeze
4
4
  end
5
5
  end
@@ -35,6 +35,21 @@ module GitLab
35
35
  end
36
36
  end
37
37
 
38
+ # Performs a major GC after each request. We found that this helps to free up
39
+ # several MB of memory in conjunction with sricter malloc config.
40
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/297241
41
+ class RunGC
42
+ def initialize(app)
43
+ @app = app
44
+ end
45
+
46
+ def call(env)
47
+ @app.call(env).tap do
48
+ GC.start
49
+ end
50
+ end
51
+ end
52
+
38
53
  class << self
39
54
  DEFAULT_WEB_SERVER = "webrick".freeze
40
55
 
@@ -45,6 +60,10 @@ module GitLab
45
60
  memory_threshold = (config[:server] && config[:server][:memory_threshold]) || 1024
46
61
  use MemoryKillerMiddleware, memory_threshold
47
62
  use Rack::Logger
63
+ use RunGC
64
+
65
+ # Defrag heap after everything is loaded into memory.
66
+ GC.compact
48
67
  end
49
68
 
50
69
  def logger
@@ -10,12 +10,9 @@ describe GitLab::Exporter::Database do
10
10
  let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
11
11
  let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
12
12
  let(:ee_check_query) { "SELECT COUNT(*) FROM licenses" }
13
- let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
14
- let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
15
13
  let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
16
14
  let(:connection_pool) { double("connection pool") }
17
15
  let(:connection) { double("connection") }
18
- let(:allowed_repeated_commands_count) { 5 }
19
16
  let(:created_builds_counting_disabled) { true }
20
17
  let(:time_now) { Time.new(2019, 4, 9, 6, 30, 0) }
21
18
  let(:unarchived_traces_query_time) { "2019-04-09 05:30:00" }
@@ -63,22 +60,6 @@ describe GitLab::Exporter::Database do
63
60
  end
64
61
  # rubocop:enable Metrics/ParameterLists
65
62
 
66
- # rubocop:disable Metrics/ParameterLists
67
- def repeated_commands_query_row_ee(namespace_id, shared_runners_enabled, project_id, status, has_minutes, count)
68
- row = repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
69
- row["has_minutes"] = has_minutes
70
- row
71
- end
72
- # rubocop:enable Metrics/ParameterLists
73
-
74
- def repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
75
- { "namespace_id" => namespace_id,
76
- "shared_runners_enabled" => shared_runners_enabled,
77
- "project_id" => project_id,
78
- "status" => status,
79
- "count" => count }
80
- end
81
-
82
63
  before do
83
64
  stub_const("GitLab::Exporter::Database::CiBuildsCollector::SET_RANDOM_PAGE_COST", set_random_page_cost_query)
84
65
  stub_const("GitLab::Exporter::Database::CiBuildsCollector::BUILDS_QUERY_EE", builds_query_ee)
@@ -87,8 +68,6 @@ describe GitLab::Exporter::Database do
87
68
  stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
88
69
  stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
89
70
  stub_const("GitLab::Exporter::Database::CiBuildsCollector::EE_CHECK_QUERY", ee_check_query)
90
- stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
91
- stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
92
71
  stub_const("GitLab::Exporter::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)
93
72
 
94
73
  allow_any_instance_of(GitLab::Exporter::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool)
@@ -130,24 +109,6 @@ describe GitLab::Exporter::Database do
130
109
  per_runner_query_row_ce(2, "project_type", 3, nil, 3, 5),
131
110
  per_runner_query_row_ce(3, "project_type", 4, nil, 3, 5)])
132
111
 
133
- # rubocop:disable Style/FormatString
134
- repeated_commands_query_ee_with_limit = repeated_commands_query_ee % [allowed_repeated_commands_count]
135
- repeated_commands_query_ce_with_limit = repeated_commands_query_ce % [allowed_repeated_commands_count]
136
- # rubocop:enable Style/FormatString
137
-
138
- allow(connection).to receive(:exec)
139
- .with(repeated_commands_query_ee_with_limit)
140
- .and_return([repeated_commands_query_row_ee(1, "t", 1, "pending", "t", 10),
141
- repeated_commands_query_row_ee(2, "f", 2, "running", "f", 20),
142
- repeated_commands_query_row_ee(1, "f", 3, "pending", "t", 30),
143
- repeated_commands_query_row_ee(2, "t", 4, "running", "f", 40)])
144
- allow(connection).to receive(:exec)
145
- .with(repeated_commands_query_ce_with_limit)
146
- .and_return([repeated_commands_query_row_ce(1, "t", 1, "pending", 10),
147
- repeated_commands_query_row_ce(2, "f", 2, "running", 20),
148
- repeated_commands_query_row_ce(1, "f", 3, "pending", 30),
149
- repeated_commands_query_row_ce(2, "t", 4, "running", 40)])
150
-
151
112
  unarchived_traces_query_with_time = unarchived_traces_query % [unarchived_traces_query_time] # rubocop:disable Style/FormatString
152
113
 
153
114
  allow(connection).to receive(:exec).with(unarchived_traces_query_with_time).and_return([{ "count" => 10 }])
@@ -156,7 +117,6 @@ describe GitLab::Exporter::Database do
156
117
  describe GitLab::Exporter::Database::CiBuildsCollector do
157
118
  let(:collector) do
158
119
  described_class.new(connection_string: "host=localhost",
159
- allowed_repeated_commands_count: allowed_repeated_commands_count,
160
120
  created_builds_counting_disabled: created_builds_counting_disabled,
161
121
  unarchived_traces_offset_minutes: unarchived_traces_offset_minutes)
162
122
  end
@@ -195,10 +155,6 @@ describe GitLab::Exporter::Database do
195
155
  expect(subject[:stale_builds]).to eq(expected_stale_builds)
196
156
  end
197
157
 
198
- it "returns raw repeated_commands data" do
199
- expect(subject[:repeated_commands]).to include(*expected_repeated_commands)
200
- end
201
-
202
158
  it "returns raw unarchived_traces data" do
203
159
  expect(subject[:unarchived_traces]).to eq(expected_unarchived_traces)
204
160
  end
@@ -222,12 +178,6 @@ describe GitLab::Exporter::Database do
222
178
  { runner: "2", runner_type: "project_type", namespace: "3", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "yes", value: 5.0 },
223
179
  { runner: "3", runner_type: "project_type", namespace: "4", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "no", value: 5.0 }]
224
180
  end
225
- let(:expected_repeated_commands) do
226
- [{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", has_minutes: "yes", value: 10.0 },
227
- { namespace: "2", project: "2", shared_runners: "no", status: "running", has_minutes: "no", value: 20.0 },
228
- { namespace: "1", project: "3", shared_runners: "no", status: "pending", has_minutes: "yes", value: 30.0 },
229
- { namespace: "2", project: "4", shared_runners: "yes", status: "running", has_minutes: "no", value: 40.0 }]
230
- end
231
181
 
232
182
  before do
233
183
  stub_ee
@@ -254,12 +204,6 @@ describe GitLab::Exporter::Database do
254
204
  { runner: "2", runner_type: "project_type", namespace: "3", scheduled: "no", triggered: "yes", value: 5 },
255
205
  { runner: "3", runner_type: "project_type", namespace: "4", scheduled: "no", triggered: "yes", value: 5 }]
256
206
  end
257
- let(:expected_repeated_commands) do
258
- [{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", value: 10 },
259
- { namespace: "2", project: "2", shared_runners: "no", status: "running", value: 20 },
260
- { namespace: "1", project: "3", shared_runners: "no", status: "pending", value: 30 },
261
- { namespace: "2", project: "4", shared_runners: "yes", status: "running", value: 40 }]
262
- end
263
207
 
264
208
  before do
265
209
  stub_ce
@@ -273,7 +217,6 @@ describe GitLab::Exporter::Database do
273
217
  let(:writer) { StringIO.new }
274
218
  let(:prober) do
275
219
  opts = { connection_string: "host=localhost",
276
- allowed_repeated_commands_count: allowed_repeated_commands_count,
277
220
  created_builds_counting_disabled: created_builds_counting_disabled,
278
221
  unarchived_traces_offset_minutes: unarchived_traces_offset_minutes }
279
222
  described_class.new(opts,
@@ -324,12 +267,6 @@ describe GitLab::Exporter::Database do
324
267
  end
325
268
  end
326
269
 
327
- it "responds with repeated commands Prometheus metrics" do
328
- ci_repeated_commands_builds_lines.each do |expected_line|
329
- expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
330
- end
331
- end
332
-
333
270
  it "responds with stale builds Prometheus metrics" do
334
271
  expect(subject).to match(/^ci_stale_builds 2.0$/m)
335
272
  end
@@ -372,12 +309,6 @@ describe GitLab::Exporter::Database do
372
309
  'ci_running_builds\{has_minutes="yes",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="2",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0',
373
310
  'ci_running_builds\{has_minutes="no",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="3",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0']
374
311
  end
375
- let(:ci_repeated_commands_builds_lines) do
376
- ['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending",has_minutes="yes"\} 10.0',
377
- 'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running",has_minutes="no"\} 20.0',
378
- 'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending",has_minutes="yes"\} 30.0',
379
- 'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running",has_minutes="no"\} 40.0']
380
- end
381
312
  let(:namespace_out_of_limit) { 2 }
382
313
 
383
314
  before do
@@ -403,12 +334,6 @@ describe GitLab::Exporter::Database do
403
334
  'ci_running_builds\{namespace="",runner="2",runner_type="project_type",scheduled="no",triggered="yes"\} 10.0',
404
335
  'ci_running_builds\{namespace="",runner="3",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0']
405
336
  end
406
- let(:ci_repeated_commands_builds_lines) do
407
- ['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending"\} 10.0',
408
- 'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running"\} 20.0',
409
- 'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending"\} 30.0',
410
- 'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running"\} 40.0']
411
- end
412
337
  let(:namespace_out_of_limit) { 0 }
413
338
 
414
339
  before do
data/spec/ruby_spec.rb ADDED
@@ -0,0 +1,25 @@
1
+ require "spec_helper"
2
+ require "gitlab_exporter/ruby"
3
+ require "gitlab_exporter/prober"
4
+
5
+ describe GitLab::Exporter::RubyProber do
6
+ let(:prober) { GitLab::Exporter::Prober.new(options) }
7
+
8
+ let(:options) do
9
+ {
10
+ ruby: {
11
+ class_name: described_class.to_s,
12
+ methods: %w[probe_gc],
13
+ opts: { quantiles: false }
14
+ }
15
+ }
16
+ end
17
+
18
+ it "probes and returns GC stats" do
19
+ prober.probe_all
20
+
21
+ output = StringIO.new
22
+ prober.write_to(output)
23
+ expect(output.string).to match(/ruby_gc_stat_count \d+ \d+/)
24
+ end
25
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-exporter
3
3
  version: !ruby/object:Gem::Version
4
- version: 9.0.0
4
+ version: 10.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Carranza
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.1'
41
+ - !ruby/object:Gem::Dependency
42
+ name: puma
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 5.3.2
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 5.3.2
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: quantile
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -173,6 +187,7 @@ files:
173
187
  - lib/gitlab_exporter/prober.rb
174
188
  - lib/gitlab_exporter/process.rb
175
189
  - lib/gitlab_exporter/prometheus.rb
190
+ - lib/gitlab_exporter/ruby.rb
176
191
  - lib/gitlab_exporter/sidekiq.rb
177
192
  - lib/gitlab_exporter/sidekiq_queue_job_stats.lua
178
193
  - lib/gitlab_exporter/util.rb
@@ -187,6 +202,7 @@ files:
187
202
  - spec/git_spec.rb
188
203
  - spec/memstats_spec.rb
189
204
  - spec/prometheus_metrics_spec.rb
205
+ - spec/ruby_spec.rb
190
206
  - spec/spec_helper.rb
191
207
  - spec/util_spec.rb
192
208
  homepage: http://gitlab.com
@@ -222,5 +238,6 @@ test_files:
222
238
  - spec/git_spec.rb
223
239
  - spec/memstats_spec.rb
224
240
  - spec/prometheus_metrics_spec.rb
241
+ - spec/ruby_spec.rb
225
242
  - spec/spec_helper.rb
226
243
  - spec/util_spec.rb