gitlab-exporter 7.2.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +1 -1
- data/.rubocop.yml +7 -5
- data/.rubocop_todo.yml +64 -0
- data/.ruby-version +1 -0
- data/Gemfile.lock +26 -15
- data/README.md +0 -17
- data/bin/gitlab-exporter +3 -1
- data/config/gitlab-exporter.yml.example +10 -1
- data/gitlab-exporter.gemspec +5 -4
- data/lib/gitlab_exporter.rb +1 -0
- data/lib/gitlab_exporter/cli.rb +1 -2
- data/lib/gitlab_exporter/database/base.rb +33 -10
- data/lib/gitlab_exporter/database/bloat.rb +2 -2
- data/lib/gitlab_exporter/database/ci_builds.rb +4 -137
- data/lib/gitlab_exporter/database/row_count.rb +7 -25
- data/lib/gitlab_exporter/database/tuple_stats.rb +4 -2
- data/lib/gitlab_exporter/git.rb +8 -1
- data/lib/gitlab_exporter/memstats.rb +1 -1
- data/lib/gitlab_exporter/memstats/mapping.rb +1 -1
- data/lib/gitlab_exporter/process.rb +13 -3
- data/lib/gitlab_exporter/ruby.rb +21 -0
- data/lib/gitlab_exporter/sidekiq.rb +83 -69
- data/lib/gitlab_exporter/util.rb +1 -1
- data/lib/gitlab_exporter/version.rb +1 -1
- data/lib/gitlab_exporter/web_exporter.rb +23 -1
- data/spec/database/bloat_spec.rb +1 -1
- data/spec/database/ci_builds_spec.rb +23 -98
- data/spec/git_process_proper_spec.rb +1 -1
- data/spec/memstats_spec.rb +3 -3
- data/spec/ruby_spec.rb +25 -0
- data/spec/spec_helper.rb +2 -4
- metadata +36 -17
@@ -29,7 +29,7 @@ module GitLab
|
|
29
29
|
|
30
30
|
MIRROR_QUERY = {
|
31
31
|
select: :projects,
|
32
|
-
joins:
|
32
|
+
joins: <<~SQL,
|
33
33
|
INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id
|
34
34
|
INNER JOIN namespaces AS root_namespaces ON root_namespaces.id = (
|
35
35
|
WITH RECURSIVE "base_and_ancestors" AS (
|
@@ -112,9 +112,9 @@ module GitLab
|
|
112
112
|
ON users.id = u.user_id",
|
113
113
|
where: "user_type IS NULL",
|
114
114
|
fields: {
|
115
|
-
admin:
|
116
|
-
external:
|
117
|
-
state:
|
115
|
+
admin: {},
|
116
|
+
external: {},
|
117
|
+
state: {},
|
118
118
|
access_level: { definition: "COALESCE(u.access_level, 0)" }
|
119
119
|
}
|
120
120
|
},
|
@@ -122,14 +122,14 @@ module GitLab
|
|
122
122
|
select: :projects,
|
123
123
|
fields: {
|
124
124
|
visibility_level: {},
|
125
|
-
archived:
|
125
|
+
archived: {}
|
126
126
|
}
|
127
127
|
},
|
128
128
|
groups: {
|
129
129
|
select: :namespaces,
|
130
130
|
fields: {
|
131
131
|
visibility_level: {},
|
132
|
-
root:
|
132
|
+
root: { definition: "(parent_id IS NULL)" }
|
133
133
|
}
|
134
134
|
}
|
135
135
|
}.freeze
|
@@ -175,30 +175,12 @@ module GitLab
|
|
175
175
|
|
176
176
|
def execute(query)
|
177
177
|
with_connection_pool do |conn|
|
178
|
-
conn.exec(query)
|
178
|
+
conn.exec(query)
|
179
179
|
end
|
180
180
|
rescue PG::UndefinedTable, PG::UndefinedColumn
|
181
181
|
nil
|
182
182
|
end
|
183
183
|
|
184
|
-
def type_map_for_results(conn)
|
185
|
-
@type_map_for_results ||= begin
|
186
|
-
tm = PG::BasicTypeMapForResults.new(conn)
|
187
|
-
|
188
|
-
# Remove warning message:
|
189
|
-
# Warning: no type cast defined for type "name" with oid 19.
|
190
|
-
# Please cast this type explicitly to TEXT to be safe for future changes.
|
191
|
-
# Warning: no type cast defined for type "regproc" with oid 24.
|
192
|
-
# Please cast this type explicitly to TEXT to be safe for future changes.
|
193
|
-
[{ "type": "text", "oid": 19 }, { "type": "int4", "oid": 24 }].each do |value|
|
194
|
-
old_coder = tm.coders.find { |c| c.name == value[:type] }
|
195
|
-
tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
|
196
|
-
end
|
197
|
-
|
198
|
-
tm
|
199
|
-
end
|
200
|
-
end
|
201
|
-
|
202
184
|
# Not private so I can test it without meta programming tricks
|
203
185
|
def construct_query(query)
|
204
186
|
query_string = "SELECT COUNT(*)"
|
@@ -5,7 +5,7 @@ module GitLab
|
|
5
5
|
#
|
6
6
|
# It takes a connection string (e.g. "dbname=test port=5432")
|
7
7
|
class TupleStatsCollector < Base
|
8
|
-
COLUMNS = %w
|
8
|
+
COLUMNS = %w[relname seq_tup_read idx_tup_fetch n_tup_ins n_tup_upd n_tup_del n_tup_hot_upd n_dead_tup seq_scan]
|
9
9
|
.join(",")
|
10
10
|
QUERY = <<-SQL.freeze
|
11
11
|
SELECT #{COLUMNS}
|
@@ -35,9 +35,11 @@ module GitLab
|
|
35
35
|
|
36
36
|
result.each do |table_name, tuple_stats|
|
37
37
|
tuple_stats.each do |column_name, value|
|
38
|
+
next if value.is_a?(Numeric)
|
39
|
+
|
38
40
|
@metrics.add("gitlab_database_stat_table_#{column_name}",
|
39
41
|
value.to_f,
|
40
|
-
table_name: table_name)
|
42
|
+
table_name: table_name)
|
41
43
|
end
|
42
44
|
end
|
43
45
|
|
data/lib/gitlab_exporter/git.rb
CHANGED
@@ -13,6 +13,7 @@ module GitLab
|
|
13
13
|
class Git
|
14
14
|
def initialize(repo)
|
15
15
|
fail "Repository #{repo} does not exists" unless Dir.exist? repo
|
16
|
+
|
16
17
|
@repo = repo
|
17
18
|
@tracker = TimeTracker.new
|
18
19
|
end
|
@@ -35,6 +36,7 @@ module GitLab
|
|
35
36
|
def execute(command)
|
36
37
|
result = CommandResult.new(*Open3.capture2e(command, chdir: @repo))
|
37
38
|
fail "Command #{command} failed with status #{result.status}\n#{result.stdout}" if result.failed?
|
39
|
+
|
38
40
|
result
|
39
41
|
end
|
40
42
|
end
|
@@ -91,12 +93,16 @@ module GitLab
|
|
91
93
|
end
|
92
94
|
|
93
95
|
def probe_git # rubocop:disable Metrics/MethodLength
|
96
|
+
puts "[DEPRECATED] probe_git and GitProcessProber are now considered obsolete"\
|
97
|
+
" and will be removed in future major versions,"\
|
98
|
+
" please use git metrics produced by Gitaly instead"
|
99
|
+
|
94
100
|
counts = Hash.new(0)
|
95
101
|
|
96
102
|
Utils.pgrep("^git ").each do |pid|
|
97
103
|
process_cmd = begin
|
98
104
|
File.read("/proc/#{pid}/cmdline")
|
99
|
-
rescue
|
105
|
+
rescue StandardError
|
100
106
|
"" # Process file is gone (race condition)
|
101
107
|
end
|
102
108
|
subcommand = self.class.extract_subcommand(process_cmd)
|
@@ -131,6 +137,7 @@ module GitLab
|
|
131
137
|
|
132
138
|
def self.extract_subcommand(cmd)
|
133
139
|
return if cmd.empty?
|
140
|
+
|
134
141
|
cmd_splitted = cmd.split("\u0000") # cmdline does not return it space-separated
|
135
142
|
|
136
143
|
cmd_splitted.shift # Because it's "git"
|
@@ -25,7 +25,7 @@ module GitLab
|
|
25
25
|
# Locked: 0 kB
|
26
26
|
# VmFlags: rd ex mr mw me dw sd
|
27
27
|
class Mapping
|
28
|
-
FIELDS = %w
|
28
|
+
FIELDS = %w[size rss shared_clean shared_dirty private_clean private_dirty swap pss].freeze
|
29
29
|
|
30
30
|
attr_reader :address_start
|
31
31
|
attr_reader :address_end
|
@@ -71,6 +71,10 @@ module GitLab
|
|
71
71
|
end
|
72
72
|
|
73
73
|
def probe_stat
|
74
|
+
puts "[DEPRECATED] probe_stat and ProcessProber are now considered obsolete"\
|
75
|
+
" and will be removed in future major versions,"\
|
76
|
+
" please use metrics produced by application processes instead"
|
77
|
+
|
74
78
|
@pids.each do |pid|
|
75
79
|
stats = ProcessStats.new(pid)
|
76
80
|
next unless stats.valid?
|
@@ -88,12 +92,20 @@ module GitLab
|
|
88
92
|
end
|
89
93
|
|
90
94
|
def probe_count
|
95
|
+
puts "[DEPRECATED] probe_count and ProcessProber are now considered obsolete"\
|
96
|
+
" and will be removed in future major versions,"\
|
97
|
+
" please use metrics produced by application processes instead"
|
98
|
+
|
91
99
|
@metrics.add("process_count", @pids.count, name: @name.downcase)
|
92
100
|
|
93
101
|
self
|
94
102
|
end
|
95
103
|
|
96
104
|
def probe_smaps
|
105
|
+
puts "[DEPRECATED] probe_smaps and ProcessProber are now considered obsolete"\
|
106
|
+
" and will be removed in future major versions,"\
|
107
|
+
" please use metrics produced by application processes instead"
|
108
|
+
|
97
109
|
@pids.each do |pid|
|
98
110
|
stats = ::GitLab::Exporter::MemStats::Aggregator.new(pid)
|
99
111
|
|
@@ -105,9 +117,7 @@ module GitLab
|
|
105
117
|
::GitLab::Exporter::MemStats::Mapping::FIELDS.each do |field|
|
106
118
|
value = stats.totals[field]
|
107
119
|
|
108
|
-
if value >= 0
|
109
|
-
@metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels)
|
110
|
-
end
|
120
|
+
@metrics.add("process_smaps_#{field}_bytes", value * 1024, @use_quantiles, **labels) if value >= 0
|
111
121
|
end
|
112
122
|
end
|
113
123
|
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitLab
|
4
|
+
module Exporter
|
5
|
+
# Probes a current process GC for info then writes metrics to a target
|
6
|
+
class RubyProber
|
7
|
+
def initialize(options, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
|
8
|
+
@metrics = metrics
|
9
|
+
@use_quantiles = options.fetch(:quantiles, false)
|
10
|
+
end
|
11
|
+
|
12
|
+
def probe_gc
|
13
|
+
GC.stat.each do |stat, value|
|
14
|
+
@metrics.add("ruby_gc_stat_#{stat}", value.to_i, @use_quantiles)
|
15
|
+
end
|
16
|
+
|
17
|
+
self
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -10,56 +10,63 @@ module GitLab
|
|
10
10
|
QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
|
11
11
|
QUEUE_JOB_STATS_SHA = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
|
12
12
|
|
13
|
+
POOL_SIZE = 3
|
14
|
+
|
15
|
+
# This timeout is configured to higher interval than scrapping
|
16
|
+
# of Prometheus to ensure that connection is kept instead of
|
17
|
+
# needed to be re-initialized
|
18
|
+
POOL_TIMEOUT = 90
|
19
|
+
|
20
|
+
def self.connection_pool
|
21
|
+
@@connection_pool ||= Hash.new do |h, connection_hash| # rubocop:disable Style/ClassVars
|
22
|
+
config = connection_hash.merge(pool_timeout: POOL_TIMEOUT, size: POOL_SIZE)
|
23
|
+
|
24
|
+
h[connection_hash] = Sidekiq::RedisConnection.create(config)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
13
28
|
def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
|
14
29
|
@opts = opts
|
15
30
|
@metrics = metrics
|
16
31
|
@logger = logger
|
17
|
-
|
18
|
-
Sidekiq.configure_client do |config|
|
19
|
-
config.redis = redis_options
|
20
|
-
end
|
21
|
-
|
22
|
-
ensure_queue_job_stats_script_loaded
|
23
32
|
end
|
24
33
|
|
25
34
|
def probe_stats
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
35
|
+
with_sidekiq do
|
36
|
+
stats = Sidekiq::Stats.new
|
37
|
+
|
38
|
+
@metrics.add("sidekiq_jobs_processed_total", stats.processed)
|
39
|
+
@metrics.add("sidekiq_jobs_failed_total", stats.failed)
|
40
|
+
@metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
|
41
|
+
@metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
|
42
|
+
@metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
|
43
|
+
@metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
|
44
|
+
|
45
|
+
@metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
|
46
|
+
@metrics.add("sidekiq_processes_size", stats.processes_size)
|
47
|
+
@metrics.add("sidekiq_workers_size", stats.workers_size)
|
48
|
+
end
|
40
49
|
|
41
50
|
self
|
42
51
|
end
|
43
52
|
|
44
53
|
def probe_queues
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
54
|
+
with_sidekiq do
|
55
|
+
Sidekiq::Queue.all.each do |queue|
|
56
|
+
@metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
|
57
|
+
@metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
|
58
|
+
@metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
|
59
|
+
end
|
51
60
|
end
|
52
61
|
|
53
62
|
self
|
54
63
|
end
|
55
64
|
|
56
|
-
def probe_jobs
|
57
|
-
|
58
|
-
|
59
|
-
job_stats = {}
|
65
|
+
def probe_jobs
|
66
|
+
with_sidekiq do
|
67
|
+
job_stats = {}
|
60
68
|
|
61
|
-
|
62
|
-
begin
|
69
|
+
Sidekiq::Queue.all.each do |queue|
|
63
70
|
Sidekiq.redis do |conn|
|
64
71
|
stats = conn.evalsha(QUEUE_JOB_STATS_SHA, ["queue:#{queue.name}"])
|
65
72
|
job_stats.merge!(stats.to_h)
|
@@ -68,44 +75,44 @@ module GitLab
|
|
68
75
|
# FIXME: Should we call SCRIPT KILL?
|
69
76
|
return self
|
70
77
|
end
|
71
|
-
end
|
72
78
|
|
73
|
-
|
74
|
-
|
79
|
+
job_stats.each do |class_name, count|
|
80
|
+
@metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
|
81
|
+
end
|
75
82
|
end
|
76
83
|
|
77
84
|
self
|
78
85
|
end
|
79
86
|
|
80
87
|
def probe_workers
|
81
|
-
|
82
|
-
|
83
|
-
worker_stats = Hash.new(0)
|
88
|
+
with_sidekiq do
|
89
|
+
worker_stats = Hash.new(0)
|
84
90
|
|
85
|
-
|
86
|
-
|
91
|
+
Sidekiq::Workers.new.map do |_pid, _tid, work|
|
92
|
+
job_klass = work["payload"]["class"]
|
87
93
|
|
88
|
-
|
89
|
-
|
94
|
+
worker_stats[job_klass] += 1
|
95
|
+
end
|
90
96
|
|
91
|
-
|
92
|
-
|
97
|
+
worker_stats.each do |class_name, count|
|
98
|
+
@metrics.add("sidekiq_running_jobs", count, name: class_name)
|
99
|
+
end
|
93
100
|
end
|
94
101
|
|
95
102
|
self
|
96
103
|
end
|
97
104
|
|
98
105
|
def probe_retries
|
99
|
-
|
106
|
+
with_sidekiq do
|
107
|
+
retry_stats = Hash.new(0)
|
100
108
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
retry_stats[job.klass] += 1
|
105
|
-
end
|
109
|
+
Sidekiq::RetrySet.new.map do |job|
|
110
|
+
retry_stats[job.klass] += 1
|
111
|
+
end
|
106
112
|
|
107
|
-
|
108
|
-
|
113
|
+
retry_stats.each do |class_name, count|
|
114
|
+
@metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
|
115
|
+
end
|
109
116
|
end
|
110
117
|
|
111
118
|
self
|
@@ -115,9 +122,9 @@ module GitLab
|
|
115
122
|
puts "[DEPRECATED] probe_dead is now considered obsolete and will be removed in future major versions,"\
|
116
123
|
" please use probe_stats instead"
|
117
124
|
|
118
|
-
|
119
|
-
|
120
|
-
|
125
|
+
with_sidekiq do
|
126
|
+
@metrics.add("sidekiq_dead_jobs", Sidekiq::Stats.new.dead_size)
|
127
|
+
end
|
121
128
|
|
122
129
|
self
|
123
130
|
end
|
@@ -128,6 +135,19 @@ module GitLab
|
|
128
135
|
|
129
136
|
private
|
130
137
|
|
138
|
+
def with_sidekiq
|
139
|
+
# TODO: this is not concurrent safe as we change global context
|
140
|
+
# It means that we are unable to use many different sidekiq's
|
141
|
+
# which is not a problem as of now
|
142
|
+
Sidekiq.configure_client do |config|
|
143
|
+
config.redis = self.class.connection_pool[redis_options]
|
144
|
+
end
|
145
|
+
|
146
|
+
return unless connected?
|
147
|
+
|
148
|
+
yield
|
149
|
+
end
|
150
|
+
|
131
151
|
def redis_options
|
132
152
|
options = {
|
133
153
|
url: @opts[:redis_url],
|
@@ -147,25 +167,19 @@ module GitLab
|
|
147
167
|
end
|
148
168
|
|
149
169
|
def connected?
|
150
|
-
@connected
|
151
|
-
Sidekiq.redis do |conn|
|
152
|
-
conn.get("foo")
|
153
|
-
end
|
154
|
-
true
|
155
|
-
end
|
156
|
-
rescue Redis::CannotConnectError, Redis::TimeoutError # rubocop:disable Lint/HandleExceptions
|
157
|
-
# Maybe we're trying connecting to a slave
|
158
|
-
end
|
159
|
-
|
160
|
-
def ensure_queue_job_stats_script_loaded
|
161
|
-
return unless connected?
|
170
|
+
return @connected unless @connected.nil?
|
162
171
|
|
172
|
+
# This is also a good "connected check"
|
163
173
|
Sidekiq.redis do |conn|
|
164
174
|
# Using administrative commands on conn directly (which is a Redis::Namespace)
|
165
175
|
# will be removed in redis-namespace 2.0.
|
166
|
-
|
167
|
-
conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT)
|
176
|
+
conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT) unless conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
|
168
177
|
end
|
178
|
+
|
179
|
+
@connected = true
|
180
|
+
rescue Redis::BaseConnectionError => e
|
181
|
+
@logger&.error "Error connecting to the Redis: #{e}"
|
182
|
+
@connected = false
|
169
183
|
end
|
170
184
|
end
|
171
185
|
end
|
data/lib/gitlab_exporter/util.rb
CHANGED
@@ -24,7 +24,7 @@ module GitLab
|
|
24
24
|
# Helper methods, some stuff was copied from ActiveSupport
|
25
25
|
module Utils
|
26
26
|
def camel_case_string(str)
|
27
|
-
str.gsub(/(?:_|^)([a-z\d]*)/i) { $1.capitalize } # rubocop:disable PerlBackrefs
|
27
|
+
str.gsub(/(?:_|^)([a-z\d]*)/i) { $1.capitalize } # rubocop:disable Style/PerlBackrefs
|
28
28
|
end
|
29
29
|
module_function :camel_case_string
|
30
30
|
|