gitlab-exporter 7.2.0 → 10.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -29,7 +29,7 @@ module GitLab
29
29
 
30
30
  MIRROR_QUERY = {
31
31
  select: :projects,
32
- joins: <<~SQL,
32
+ joins: <<~SQL,
33
33
  INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id
34
34
  INNER JOIN namespaces AS root_namespaces ON root_namespaces.id = (
35
35
  WITH RECURSIVE "base_and_ancestors" AS (
@@ -112,9 +112,9 @@ module GitLab
112
112
  ON users.id = u.user_id",
113
113
  where: "user_type IS NULL",
114
114
  fields: {
115
- admin: {},
116
- external: {},
117
- state: {},
115
+ admin: {},
116
+ external: {},
117
+ state: {},
118
118
  access_level: { definition: "COALESCE(u.access_level, 0)" }
119
119
  }
120
120
  },
@@ -122,14 +122,14 @@ module GitLab
122
122
  select: :projects,
123
123
  fields: {
124
124
  visibility_level: {},
125
- archived: {}
125
+ archived: {}
126
126
  }
127
127
  },
128
128
  groups: {
129
129
  select: :namespaces,
130
130
  fields: {
131
131
  visibility_level: {},
132
- root: { definition: "(parent_id IS NULL)" }
132
+ root: { definition: "(parent_id IS NULL)" }
133
133
  }
134
134
  }
135
135
  }.freeze
@@ -175,30 +175,12 @@ module GitLab
175
175
 
176
176
  def execute(query)
177
177
  with_connection_pool do |conn|
178
- conn.exec(query).map_types!(type_map_for_results(conn))
178
+ conn.exec(query)
179
179
  end
180
180
  rescue PG::UndefinedTable, PG::UndefinedColumn
181
181
  nil
182
182
  end
183
183
 
184
- def type_map_for_results(conn)
185
- @type_map_for_results ||= begin
186
- tm = PG::BasicTypeMapForResults.new(conn)
187
-
188
- # Remove warning message:
189
- # Warning: no type cast defined for type "name" with oid 19.
190
- # Please cast this type explicitly to TEXT to be safe for future changes.
191
- # Warning: no type cast defined for type "regproc" with oid 24.
192
- # Please cast this type explicitly to TEXT to be safe for future changes.
193
- [{ "type": "text", "oid": 19 }, { "type": "int4", "oid": 24 }].each do |value|
194
- old_coder = tm.coders.find { |c| c.name == value[:type] }
195
- tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
196
- end
197
-
198
- tm
199
- end
200
- end
201
-
202
184
  # Not private so I can test it without meta programming tricks
203
185
  def construct_query(query)
204
186
  query_string = "SELECT COUNT(*)"
@@ -5,7 +5,7 @@ module GitLab
5
5
  #
6
6
  # It takes a connection string (e.g. "dbname=test port=5432")
7
7
  class TupleStatsCollector < Base
8
- COLUMNS = %w(relname seq_tup_read idx_tup_fetch n_tup_ins n_tup_upd n_tup_del n_tup_hot_upd n_dead_tup seq_scan)
8
+ COLUMNS = %w[relname seq_tup_read idx_tup_fetch n_tup_ins n_tup_upd n_tup_del n_tup_hot_upd n_dead_tup seq_scan]
9
9
  .join(",")
10
10
  QUERY = <<-SQL.freeze
11
11
  SELECT #{COLUMNS}
@@ -35,9 +35,11 @@ module GitLab
35
35
 
36
36
  result.each do |table_name, tuple_stats|
37
37
  tuple_stats.each do |column_name, value|
38
+ next if value.is_a?(Numeric)
39
+
38
40
  @metrics.add("gitlab_database_stat_table_#{column_name}",
39
41
  value.to_f,
40
- table_name: table_name) unless value.is_a?(Numeric)
42
+ table_name: table_name)
41
43
  end
42
44
  end
43
45
 
@@ -13,6 +13,7 @@ module GitLab
13
13
  class Git
14
14
  def initialize(repo)
15
15
  fail "Repository #{repo} does not exists" unless Dir.exist? repo
16
+
16
17
  @repo = repo
17
18
  @tracker = TimeTracker.new
18
19
  end
@@ -35,6 +36,7 @@ module GitLab
35
36
  def execute(command)
36
37
  result = CommandResult.new(*Open3.capture2e(command, chdir: @repo))
37
38
  fail "Command #{command} failed with status #{result.status}\n#{result.stdout}" if result.failed?
39
+
38
40
  result
39
41
  end
40
42
  end
@@ -91,12 +93,16 @@ module GitLab
91
93
  end
92
94
 
93
95
  def probe_git # rubocop:disable Metrics/MethodLength
96
+ puts "[DEPRECATED] probe_git and GitProcessProber are now considered obsolete"\
97
+ " and will be removed in future major versions,"\
98
+ " please use git metrics produced by Gitaly instead"
99
+
94
100
  counts = Hash.new(0)
95
101
 
96
102
  Utils.pgrep("^git ").each do |pid|
97
103
  process_cmd = begin
98
104
  File.read("/proc/#{pid}/cmdline")
99
- rescue
105
+ rescue StandardError
100
106
  "" # Process file is gone (race condition)
101
107
  end
102
108
  subcommand = self.class.extract_subcommand(process_cmd)
@@ -131,6 +137,7 @@ module GitLab
131
137
 
132
138
  def self.extract_subcommand(cmd)
133
139
  return if cmd.empty?
140
+
134
141
  cmd_splitted = cmd.split("\u0000") # cmdline does not return it space-separated
135
142
 
136
143
  cmd_splitted.shift # Because it's "git"
@@ -87,7 +87,7 @@ module GitLab
87
87
  end
88
88
  end
89
89
  end
90
- rescue => e
90
+ rescue StandardError => e
91
91
  puts "Error: #{e}"
92
92
  @valid = false
93
93
  create_memstats_not_available(totals)
@@ -25,7 +25,7 @@ module GitLab
25
25
  # Locked: 0 kB
26
26
  # VmFlags: rd ex mr mw me dw sd
27
27
  class Mapping
28
- FIELDS = %w(size rss shared_clean shared_dirty private_clean private_dirty swap pss).freeze
28
+ FIELDS = %w[size rss shared_clean shared_dirty private_clean private_dirty swap pss].freeze
29
29
 
30
30
  attr_reader :address_start
31
31
  attr_reader :address_end
@@ -71,6 +71,10 @@ module GitLab
71
71
  end
72
72
 
73
73
  def probe_stat
74
+ puts "[DEPRECATED] probe_stat and ProcessProber are now considered obsolete"\
75
+ " and will be removed in future major versions,"\
76
+ " please use metrics produced by application processes instead"
77
+
74
78
  @pids.each do |pid|
75
79
  stats = ProcessStats.new(pid)
76
80
  next unless stats.valid?
@@ -88,12 +92,20 @@ module GitLab
88
92
  end
89
93
 
90
94
  def probe_count
95
+ puts "[DEPRECATED] probe_count and ProcessProber are now considered obsolete"\
96
+ " and will be removed in future major versions,"\
97
+ " please use metrics produced by application processes instead"
98
+
91
99
  @metrics.add("process_count", @pids.count, name: @name.downcase)
92
100
 
93
101
  self
94
102
  end
95
103
 
96
104
  def probe_smaps
105
+ puts "[DEPRECATED] probe_smaps and ProcessProber are now considered obsolete"\
106
+ " and will be removed in future major versions,"\
107
+ " please use metrics produced by application processes instead"
108
+
97
109
  @pids.each do |pid|
98
110
  stats = ::GitLab::Exporter::MemStats::Aggregator.new(pid)
99
111
 
@@ -105,9 +117,7 @@ module GitLab
105
117
  ::GitLab::Exporter::MemStats::Mapping::FIELDS.each do |field|
106
118
  value = stats.totals[field]
107
119
 
108
- if value >= 0
109
- @metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels)
110
- end
120
+ @metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels) if value >= 0
111
121
  end
112
122
  end
113
123
 
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module GitLab
4
+ module Exporter
5
+ # Probes a current process GC for info then writes metrics to a target
6
+ class RubyProber
7
+ def initialize(options, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
8
+ @metrics = metrics
9
+ @use_quantiles = options.fetch(:quantiles, false)
10
+ end
11
+
12
+ def probe_gc
13
+ GC.stat.each do |stat, value|
14
+ @metrics.add("ruby_gc_stat_#{stat}", value.to_i, @use_quantiles)
15
+ end
16
+
17
+ self
18
+ end
19
+ end
20
+ end
21
+ end
@@ -10,56 +10,63 @@ module GitLab
10
10
  QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
11
11
  QUEUE_JOB_STATS_SHA = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
12
12
 
13
+ POOL_SIZE = 3
14
+
15
+ # This timeout is configured to higher interval than scrapping
16
+ # of Prometheus to ensure that connection is kept instead of
17
+ # needed to be re-initialized
18
+ POOL_TIMEOUT = 90
19
+
20
+ def self.connection_pool
21
+ @@connection_pool ||= Hash.new do |h, connection_hash| # rubocop:disable Style/ClassVars
22
+ config = connection_hash.merge(pool_timeout: POOL_TIMEOUT, size: POOL_SIZE)
23
+
24
+ h[connection_hash] = Sidekiq::RedisConnection.create(config)
25
+ end
26
+ end
27
+
13
28
  def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
14
29
  @opts = opts
15
30
  @metrics = metrics
16
31
  @logger = logger
17
-
18
- Sidekiq.configure_client do |config|
19
- config.redis = redis_options
20
- end
21
-
22
- ensure_queue_job_stats_script_loaded
23
32
  end
24
33
 
25
34
  def probe_stats
26
- return self unless connected?
27
-
28
- stats = Sidekiq::Stats.new
29
-
30
- @metrics.add("sidekiq_jobs_processed_total", stats.processed)
31
- @metrics.add("sidekiq_jobs_failed_total", stats.failed)
32
- @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
33
- @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
34
- @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
35
- @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
36
-
37
- @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
38
- @metrics.add("sidekiq_processes_size", stats.processes_size)
39
- @metrics.add("sidekiq_workers_size", stats.workers_size)
35
+ with_sidekiq do
36
+ stats = Sidekiq::Stats.new
37
+
38
+ @metrics.add("sidekiq_jobs_processed_total", stats.processed)
39
+ @metrics.add("sidekiq_jobs_failed_total", stats.failed)
40
+ @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
41
+ @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
42
+ @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
43
+ @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
44
+
45
+ @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
46
+ @metrics.add("sidekiq_processes_size", stats.processes_size)
47
+ @metrics.add("sidekiq_workers_size", stats.workers_size)
48
+ end
40
49
 
41
50
  self
42
51
  end
43
52
 
44
53
  def probe_queues
45
- return self unless connected?
46
-
47
- Sidekiq::Queue.all.each do |queue|
48
- @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
49
- @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
50
- @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
54
+ with_sidekiq do
55
+ Sidekiq::Queue.all.each do |queue|
56
+ @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
57
+ @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
58
+ @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
59
+ end
51
60
  end
52
61
 
53
62
  self
54
63
  end
55
64
 
56
- def probe_jobs # rubocop:disable Metrics/MethodLength
57
- return self unless connected?
58
-
59
- job_stats = {}
65
+ def probe_jobs
66
+ with_sidekiq do
67
+ job_stats = {}
60
68
 
61
- Sidekiq::Queue.all.each do |queue|
62
- begin
69
+ Sidekiq::Queue.all.each do |queue|
63
70
  Sidekiq.redis do |conn|
64
71
  stats = conn.evalsha(QUEUE_JOB_STATS_SHA, ["queue:#{queue.name}"])
65
72
  job_stats.merge!(stats.to_h)
@@ -68,44 +75,44 @@ module GitLab
68
75
  # FIXME: Should we call SCRIPT KILL?
69
76
  return self
70
77
  end
71
- end
72
78
 
73
- job_stats.each do |class_name, count|
74
- @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
79
+ job_stats.each do |class_name, count|
80
+ @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
81
+ end
75
82
  end
76
83
 
77
84
  self
78
85
  end
79
86
 
80
87
  def probe_workers
81
- return self unless connected?
82
-
83
- worker_stats = Hash.new(0)
88
+ with_sidekiq do
89
+ worker_stats = Hash.new(0)
84
90
 
85
- Sidekiq::Workers.new.map do |_pid, _tid, work|
86
- job_klass = work["payload"]["class"]
91
+ Sidekiq::Workers.new.map do |_pid, _tid, work|
92
+ job_klass = work["payload"]["class"]
87
93
 
88
- worker_stats[job_klass] += 1
89
- end
94
+ worker_stats[job_klass] += 1
95
+ end
90
96
 
91
- worker_stats.each do |class_name, count|
92
- @metrics.add("sidekiq_running_jobs", count, name: class_name)
97
+ worker_stats.each do |class_name, count|
98
+ @metrics.add("sidekiq_running_jobs", count, name: class_name)
99
+ end
93
100
  end
94
101
 
95
102
  self
96
103
  end
97
104
 
98
105
  def probe_retries
99
- return self unless connected?
106
+ with_sidekiq do
107
+ retry_stats = Hash.new(0)
100
108
 
101
- retry_stats = Hash.new(0)
102
-
103
- Sidekiq::RetrySet.new.map do |job|
104
- retry_stats[job.klass] += 1
105
- end
109
+ Sidekiq::RetrySet.new.map do |job|
110
+ retry_stats[job.klass] += 1
111
+ end
106
112
 
107
- retry_stats.each do |class_name, count|
108
- @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
113
+ retry_stats.each do |class_name, count|
114
+ @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
115
+ end
109
116
  end
110
117
 
111
118
  self
@@ -115,9 +122,9 @@ module GitLab
115
122
  puts "[DEPRECATED] probe_dead is now considered obsolete and will be removed in future major versions,"\
116
123
  " please use probe_stats instead"
117
124
 
118
- return self unless connected?
119
-
120
- @metrics.add("sidekiq_dead_jobs", Sidekiq::Stats.new.dead_size)
125
+ with_sidekiq do
126
+ @metrics.add("sidekiq_dead_jobs", Sidekiq::Stats.new.dead_size)
127
+ end
121
128
 
122
129
  self
123
130
  end
@@ -128,6 +135,19 @@ module GitLab
128
135
 
129
136
  private
130
137
 
138
+ def with_sidekiq
139
+ # TODO: this is not concurrent safe as we change global context
140
+ # It means that we are unable to use many different sidekiq's
141
+ # which is not a problem as of now
142
+ Sidekiq.configure_client do |config|
143
+ config.redis = self.class.connection_pool[redis_options]
144
+ end
145
+
146
+ return unless connected?
147
+
148
+ yield
149
+ end
150
+
131
151
  def redis_options
132
152
  options = {
133
153
  url: @opts[:redis_url],
@@ -147,25 +167,19 @@ module GitLab
147
167
  end
148
168
 
149
169
  def connected?
150
- @connected ||= begin
151
- Sidekiq.redis do |conn|
152
- conn.get("foo")
153
- end
154
- true
155
- end
156
- rescue Redis::CannotConnectError, Redis::TimeoutError # rubocop:disable Lint/HandleExceptions
157
- # Maybe we're trying connecting to a slave
158
- end
159
-
160
- def ensure_queue_job_stats_script_loaded
161
- return unless connected?
170
+ return @connected unless @connected.nil?
162
171
 
172
+ # This is also a good "connected check"
163
173
  Sidekiq.redis do |conn|
164
174
  # Using administrative commands on conn directly (which is a Redis::Namespace)
165
175
  # will be removed in redis-namespace 2.0.
166
- next if conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
167
- conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT)
176
+ conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT) unless conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
168
177
  end
178
+
179
+ @connected = true
180
+ rescue Redis::BaseConnectionError => e
181
+ @logger&.error "Error connecting to the Redis: #{e}"
182
+ @connected = false
169
183
  end
170
184
  end
171
185
  end
@@ -24,7 +24,7 @@ module GitLab
24
24
  # Helper methods, some stuff was copied from ActiveSupport
25
25
  module Utils
26
26
  def camel_case_string(str)
27
- str.gsub(/(?:_|^)([a-z\d]*)/i) { $1.capitalize } # rubocop:disable PerlBackrefs
27
+ str.gsub(/(?:_|^)([a-z\d]*)/i) { $1.capitalize } # rubocop:disable Style/PerlBackrefs
28
28
  end
29
29
  module_function :camel_case_string
30
30