gitlab-exporter 10.5.0 → 11.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +8 -1
- data/Gemfile +2 -0
- data/Gemfile.lock +9 -7
- data/README.md +10 -3
- data/gitlab-exporter.gemspec +1 -1
- data/lib/gitlab_exporter/cli.rb +1 -1
- data/lib/gitlab_exporter/database/base.rb +2 -2
- data/lib/gitlab_exporter/database/bloat.rb +4 -5
- data/lib/gitlab_exporter/database/ci_builds.rb +5 -9
- data/lib/gitlab_exporter/database/remote_mirrors.rb +3 -3
- data/lib/gitlab_exporter/database/row_count.rb +5 -8
- data/lib/gitlab_exporter/database/tuple_stats.rb +2 -2
- data/lib/gitlab_exporter/git.rb +6 -6
- data/lib/gitlab_exporter/prober.rb +2 -2
- data/lib/gitlab_exporter/process.rb +3 -3
- data/lib/gitlab_exporter/ruby.rb +2 -2
- data/lib/gitlab_exporter/sidekiq.rb +28 -40
- data/lib/gitlab_exporter/version.rb +1 -1
- data/lib/gitlab_exporter/web_exporter.rb +1 -1
- data/spec/database/bloat_spec.rb +3 -3
- data/spec/database/ci_builds_spec.rb +6 -5
- data/spec/git_spec.rb +3 -3
- data/spec/ruby_spec.rb +1 -1
- data/spec/spec_helper.rb +0 -2
- metadata +5 -6
- data/lib/gitlab_exporter/sidekiq_queue_job_stats.lua +0 -42
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2d94bb457b87a9b44d9147c8bb0f504d43ca4d6516d0995b5f02fca2f2f19f71
|
4
|
+
data.tar.gz: 00bf2b35ff8759f55485026ff21c72adc472ad750001450084bbfee5f4e5f59d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 19769e5db26bf3be5a5dfe4109d4e7e36830d7e2bcc9a471398936cf5a83eb99a7b63082cacf826219f6e1d6ab229e6b97504670b402d74326e7e984ed9034c3
|
7
|
+
data.tar.gz: c65749449b07cc73b777c79d93091e6720392663574b627b48f0dc56ac44975903402cddaa9c081e2b064fe17252e0ecb35260b354e8dad18c5f9fbd76b96ffb
|
data/.gitlab-ci.yml
CHANGED
@@ -6,12 +6,15 @@ include:
|
|
6
6
|
- template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
|
7
7
|
- template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
|
8
8
|
|
9
|
+
variables:
|
10
|
+
RUBY_VERSION: "2.7"
|
11
|
+
|
9
12
|
stages:
|
10
13
|
- test
|
11
14
|
- dast
|
12
15
|
|
13
16
|
default:
|
14
|
-
image: ruby
|
17
|
+
image: ruby:${RUBY_VERSION}
|
15
18
|
cache:
|
16
19
|
paths:
|
17
20
|
- vendor
|
@@ -36,6 +39,10 @@ rspec:
|
|
36
39
|
script:
|
37
40
|
- bundle exec rspec spec -f d -c
|
38
41
|
before_script: *before_scripts
|
42
|
+
parallel:
|
43
|
+
matrix:
|
44
|
+
- RUBY_VERSION: "2.7"
|
45
|
+
- RUBY_VERSION: "3.0"
|
39
46
|
|
40
47
|
rubocop:
|
41
48
|
script:
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
gitlab-exporter (
|
4
|
+
gitlab-exporter (11.2.0)
|
5
5
|
connection_pool (= 2.2.5)
|
6
6
|
pg (= 1.2.3)
|
7
7
|
puma (= 5.3.2)
|
@@ -9,7 +9,7 @@ PATH
|
|
9
9
|
redis (= 4.1.4)
|
10
10
|
redis-namespace (= 1.6.0)
|
11
11
|
sidekiq (= 5.2.9)
|
12
|
-
sinatra (
|
12
|
+
sinatra (~> 2.1.0)
|
13
13
|
|
14
14
|
GEM
|
15
15
|
remote: https://rubygems.org/
|
@@ -28,7 +28,7 @@ GEM
|
|
28
28
|
nio4r (~> 2.0)
|
29
29
|
quantile (0.2.1)
|
30
30
|
rack (2.2.3)
|
31
|
-
rack-protection (2.0
|
31
|
+
rack-protection (2.1.0)
|
32
32
|
rack
|
33
33
|
rainbow (3.0.0)
|
34
34
|
redis (4.1.4)
|
@@ -67,13 +67,14 @@ GEM
|
|
67
67
|
rack (~> 2.0)
|
68
68
|
rack-protection (>= 1.5.0)
|
69
69
|
redis (>= 3.3.5, < 4.2)
|
70
|
-
sinatra (2.0
|
70
|
+
sinatra (2.1.0)
|
71
71
|
mustermann (~> 1.0)
|
72
|
-
rack (~> 2.
|
73
|
-
rack-protection (= 2.0
|
72
|
+
rack (~> 2.2)
|
73
|
+
rack-protection (= 2.1.0)
|
74
74
|
tilt (~> 2.0)
|
75
75
|
tilt (2.0.10)
|
76
76
|
unicode-display_width (1.7.0)
|
77
|
+
webrick (1.7.0)
|
77
78
|
|
78
79
|
PLATFORMS
|
79
80
|
ruby
|
@@ -83,6 +84,7 @@ DEPENDENCIES
|
|
83
84
|
rspec (~> 3.5)
|
84
85
|
rspec-expectations (~> 3.7.0)
|
85
86
|
rubocop (~> 0.42)
|
87
|
+
webrick (~> 1.7)
|
86
88
|
|
87
89
|
BUNDLED WITH
|
88
|
-
2.
|
90
|
+
2.2.22
|
data/README.md
CHANGED
@@ -34,7 +34,7 @@ metrics.
|
|
34
34
|
`git_pull_time_milliseconds`, `git_push_time_milliseconds`
|
35
35
|
* git processes stats (see Process below)
|
36
36
|
1. [Sidekiq](lib/gitlab_exporter/sidekiq.rb)
|
37
|
-
* Stats
|
37
|
+
* Stats (probe_stats)
|
38
38
|
* `sidekiq_jobs_processed_total`
|
39
39
|
* `sidekiq_jobs_failed_total`
|
40
40
|
* `sidekiq_jobs_enqueued_size`
|
@@ -44,14 +44,21 @@ metrics.
|
|
44
44
|
* `sidekiq_default_queue_latency_seconds`
|
45
45
|
* `sidekiq_processes_size`
|
46
46
|
* `sidekiq_workers_size`
|
47
|
-
* Queues
|
47
|
+
* Queues (probe_queues)
|
48
48
|
* `sidekiq_queue_size`
|
49
49
|
* `sidekiq_queue_paused`
|
50
50
|
* `sidekiq_queue_latency_seconds`
|
51
|
-
* Jobs
|
51
|
+
* Jobs (probe_jobs_limit)
|
52
52
|
* `sidekiq_enqueued_jobs`
|
53
|
+
* Workers (probe_workers)
|
53
54
|
* `sidekiq_running_jobs`
|
55
|
+
* Retries (probe_retries)
|
54
56
|
* `sidekiq_to_be_retried_jobs`
|
57
|
+
* Future Sets (probe_future_sets)
|
58
|
+
* `sidekiq_schedule_set_processing_delay_seconds`
|
59
|
+
* `sidekiq_schedule_set_backlog_count`
|
60
|
+
* `sidekiq_retry_set_processing_delay_seconds`
|
61
|
+
* `sidekiq_retry_set_backlog_count`
|
55
62
|
|
56
63
|
### Setup with GitLab Development Kit
|
57
64
|
|
data/gitlab-exporter.gemspec
CHANGED
@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
|
|
27
27
|
s.add_runtime_dependency "redis", "4.1.4"
|
28
28
|
s.add_runtime_dependency "redis-namespace", "1.6.0"
|
29
29
|
s.add_runtime_dependency "sidekiq", "5.2.9"
|
30
|
-
s.add_runtime_dependency "sinatra", "2.0
|
30
|
+
s.add_runtime_dependency "sinatra", "~> 2.1.0"
|
31
31
|
|
32
32
|
s.add_development_dependency "rspec", "~> 3.7.0"
|
33
33
|
s.add_development_dependency "rspec-expectations", "~> 3.7.0"
|
data/lib/gitlab_exporter/cli.rb
CHANGED
@@ -41,8 +41,8 @@ module GitLab
|
|
41
41
|
conn.type_map_for_results = tm
|
42
42
|
end
|
43
43
|
|
44
|
-
def initialize(
|
45
|
-
@connection_string =
|
44
|
+
def initialize(connection_string:, logger: nil, **opts) # rubocop:disable Lint/UnusedMethodArgument
|
45
|
+
@connection_string = connection_string
|
46
46
|
@logger = logger
|
47
47
|
end
|
48
48
|
|
@@ -39,12 +39,11 @@ module GitLab
|
|
39
39
|
|
40
40
|
attr_reader :metrics, :collector, :bloat_types
|
41
41
|
|
42
|
-
def initialize(
|
43
|
-
|
44
|
-
|
45
|
-
logger: nil)
|
42
|
+
def initialize(metrics: PrometheusMetrics.new,
|
43
|
+
logger: nil,
|
44
|
+
**opts)
|
46
45
|
@metrics = metrics
|
47
|
-
@collector = collector
|
46
|
+
@collector = opts[:collector] || BloatCollector.new(**opts)
|
48
47
|
@collector.logger = logger
|
49
48
|
@bloat_types = opts[:bloat_types] || %i[btree table]
|
50
49
|
end
|
@@ -168,8 +168,8 @@ module GitLab
|
|
168
168
|
|
169
169
|
DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440
|
170
170
|
|
171
|
-
def initialize(opts
|
172
|
-
super
|
171
|
+
def initialize(**opts)
|
172
|
+
super
|
173
173
|
|
174
174
|
@created_builds_counting_disabled = opts[:created_builds_counting_disabled]
|
175
175
|
@unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes]
|
@@ -294,13 +294,9 @@ module GitLab
|
|
294
294
|
|
295
295
|
# The prober which is called when gathering metrics
|
296
296
|
class CiBuildsProber
|
297
|
-
def initialize(
|
297
|
+
def initialize(metrics: PrometheusMetrics.new, **opts)
|
298
298
|
@metrics = metrics
|
299
|
-
|
300
|
-
collector_opts = { connection_string: opts[:connection_string],
|
301
|
-
created_builds_counting_disabled: opts[:created_builds_counting_disabled],
|
302
|
-
unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
|
303
|
-
@collector = CiBuildsCollector.new(collector_opts, logger: logger)
|
299
|
+
@collector = CiBuildsCollector.new(**opts)
|
304
300
|
end
|
305
301
|
|
306
302
|
def probe_db
|
@@ -391,7 +387,7 @@ module GitLab
|
|
391
387
|
labels[:namespace] = "" unless labels[:namespace]
|
392
388
|
|
393
389
|
selected_labels = labels.select { |k, _| allowed_labels.include?(k) }.sort.to_h
|
394
|
-
@metrics.add(metric_name, value.to_f, selected_labels)
|
390
|
+
@metrics.add(metric_name, value.to_f, **selected_labels)
|
395
391
|
end
|
396
392
|
|
397
393
|
def unarchived_traces_metrics
|
@@ -10,8 +10,8 @@ module GitLab
|
|
10
10
|
FROM remote_mirrors WHERE project_id IN (%s) AND enabled = 't'
|
11
11
|
SQL
|
12
12
|
|
13
|
-
def initialize(args)
|
14
|
-
super(args)
|
13
|
+
def initialize(**args)
|
14
|
+
super(**args)
|
15
15
|
|
16
16
|
@project_ids = args[:project_ids]
|
17
17
|
end
|
@@ -35,7 +35,7 @@ module GitLab
|
|
35
35
|
|
36
36
|
# The prober which is called when gathering metrics
|
37
37
|
class RemoteMirrorsProber
|
38
|
-
def initialize(
|
38
|
+
def initialize(metrics: PrometheusMetrics.new, **opts)
|
39
39
|
@metrics = metrics
|
40
40
|
@collector = RemoteMirrorsCollector.new(
|
41
41
|
connection_string: opts[:connection_string],
|
@@ -134,10 +134,10 @@ module GitLab
|
|
134
134
|
}
|
135
135
|
}.freeze
|
136
136
|
|
137
|
-
def initialize(args)
|
138
|
-
super(args)
|
137
|
+
def initialize(selected_queries: nil, **args)
|
138
|
+
super(**args)
|
139
139
|
|
140
|
-
@selected_queries = Set.new(
|
140
|
+
@selected_queries = Set.new(selected_queries.map(&:to_sym)) unless selected_queries.nil?
|
141
141
|
end
|
142
142
|
|
143
143
|
def run
|
@@ -200,12 +200,9 @@ module GitLab
|
|
200
200
|
|
201
201
|
# The prober which is called when gathering metrics
|
202
202
|
class RowCountProber
|
203
|
-
def initialize(
|
203
|
+
def initialize(metrics: PrometheusMetrics.new, **opts)
|
204
204
|
@metrics = metrics
|
205
|
-
@collector = RowCountCollector.new(
|
206
|
-
connection_string: opts[:connection_string],
|
207
|
-
selected_queries: opts[:selected_queries]
|
208
|
-
)
|
205
|
+
@collector = RowCountCollector.new(**opts)
|
209
206
|
end
|
210
207
|
|
211
208
|
def probe_db
|
@@ -25,9 +25,9 @@ module GitLab
|
|
25
25
|
|
26
26
|
# Probes the DB specified by opts[:connection_string] for tuple stats, then converts them to metrics
|
27
27
|
class TuplesProber
|
28
|
-
def initialize(
|
28
|
+
def initialize(metrics: PrometheusMetrics.new, **opts)
|
29
29
|
@metrics = metrics
|
30
|
-
@collector = TupleStatsCollector.new(
|
30
|
+
@collector = TupleStatsCollector.new(**opts)
|
31
31
|
end
|
32
32
|
|
33
33
|
def probe_db
|
data/lib/gitlab_exporter/git.rb
CHANGED
@@ -64,10 +64,10 @@ module GitLab
|
|
64
64
|
# Optionally takes a metrics object which by default is a PrometheusMetrics, useful to change the
|
65
65
|
# metrics writer to something else.
|
66
66
|
class GitProber
|
67
|
-
def initialize(
|
67
|
+
def initialize(source:, metrics: PrometheusMetrics.new, labels: {}, **opts) # rubocop:disable Lint/UnusedMethodArgument
|
68
68
|
@metrics = metrics
|
69
|
-
@labels =
|
70
|
-
@git = Git.new(
|
69
|
+
@labels = labels
|
70
|
+
@git = Git.new(source)
|
71
71
|
end
|
72
72
|
|
73
73
|
def probe_pull
|
@@ -87,9 +87,9 @@ module GitLab
|
|
87
87
|
|
88
88
|
# A special prober for git processes
|
89
89
|
class GitProcessProber
|
90
|
-
def initialize(
|
91
|
-
@opts = opts
|
90
|
+
def initialize(metrics: PrometheusMetrics.new, quantiles: nil, **opts) # rubocop:disable Lint/UnusedMethodArgument
|
92
91
|
@metrics = metrics
|
92
|
+
@quantiles = quantiles
|
93
93
|
end
|
94
94
|
|
95
95
|
def probe_git # rubocop:disable Metrics/MethodLength
|
@@ -115,7 +115,7 @@ module GitLab
|
|
115
115
|
{
|
116
116
|
name: name,
|
117
117
|
pid_or_pattern: pid,
|
118
|
-
quantiles: @
|
118
|
+
quantiles: @quantiles
|
119
119
|
},
|
120
120
|
metrics: @metrics
|
121
121
|
)
|
@@ -2,7 +2,7 @@ module GitLab
|
|
2
2
|
module Exporter
|
3
3
|
# A class to combine multiple probers into one
|
4
4
|
class Prober
|
5
|
-
def initialize(
|
5
|
+
def initialize(metrics: PrometheusMetrics.new, logger: nil, **prober_opts)
|
6
6
|
@prober_opts = prober_opts
|
7
7
|
@metrics = metrics
|
8
8
|
@logger = logger
|
@@ -13,7 +13,7 @@ module GitLab
|
|
13
13
|
def probe_all
|
14
14
|
@prober_opts.each do |_probe_name, params|
|
15
15
|
Utils.wrap_in_array(params[:opts]).each do |opts|
|
16
|
-
prober = params[:class].new(
|
16
|
+
prober = params[:class].new(metrics: @metrics, logger: @logger, **opts)
|
17
17
|
params[:methods].each do |meth|
|
18
18
|
prober.send(meth)
|
19
19
|
end
|
@@ -59,15 +59,15 @@ module GitLab
|
|
59
59
|
|
60
60
|
# Probes a process for info then writes metrics to a target
|
61
61
|
class ProcessProber
|
62
|
-
def initialize(
|
62
|
+
def initialize(name:, metrics: PrometheusMetrics.new, quantiles: false, **options)
|
63
63
|
@metrics = metrics
|
64
|
-
@name =
|
64
|
+
@name = name
|
65
65
|
@pids = if options[:pid_or_pattern] =~ /^\d+$/
|
66
66
|
[options[:pid_or_pattern]]
|
67
67
|
else
|
68
68
|
Utils.pgrep(options[:pid_or_pattern])
|
69
69
|
end
|
70
|
-
@use_quantiles =
|
70
|
+
@use_quantiles = quantiles
|
71
71
|
end
|
72
72
|
|
73
73
|
def probe_stat
|
data/lib/gitlab_exporter/ruby.rb
CHANGED
@@ -4,9 +4,9 @@ module GitLab
|
|
4
4
|
module Exporter
|
5
5
|
# Probes a current process GC for info then writes metrics to a target
|
6
6
|
class RubyProber
|
7
|
-
def initialize(
|
7
|
+
def initialize(metrics: PrometheusMetrics.new, quantiles: false, **opts) # rubocop:disable Lint/UnusedMethodArgument
|
8
8
|
@metrics = metrics
|
9
|
-
@use_quantiles =
|
9
|
+
@use_quantiles = quantiles
|
10
10
|
end
|
11
11
|
|
12
12
|
def probe_gc
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require "sidekiq/api"
|
2
|
+
require "sidekiq/scheduled"
|
2
3
|
require "digest"
|
3
4
|
|
4
5
|
module GitLab
|
@@ -7,9 +8,6 @@ module GitLab
|
|
7
8
|
#
|
8
9
|
# It takes the Redis URL Sidekiq is connected to
|
9
10
|
class SidekiqProber
|
10
|
-
QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
|
11
|
-
QUEUE_JOB_STATS_SHA = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
|
12
|
-
|
13
11
|
# The maximum depth (from the head) of each queue to probe. Probing the
|
14
12
|
# entirety of a very large queue will take longer and run the risk of
|
15
13
|
# timing out. But when we have a very large queue, we are most in need of
|
@@ -35,7 +33,7 @@ module GitLab
|
|
35
33
|
end
|
36
34
|
end
|
37
35
|
|
38
|
-
def initialize(
|
36
|
+
def initialize(metrics: PrometheusMetrics.new, logger: nil, **opts)
|
39
37
|
@opts = opts
|
40
38
|
@metrics = metrics
|
41
39
|
@logger = logger
|
@@ -72,45 +70,40 @@ module GitLab
|
|
72
70
|
self
|
73
71
|
end
|
74
72
|
|
75
|
-
# Count worker classes present in Sidekiq queues. This uses a Lua
|
76
|
-
# script to find all jobs in all queues. That script will block
|
77
|
-
# all other Redis commands:
|
78
|
-
# https://redis.io/commands/eval#atomicity-of-scripts
|
79
|
-
#
|
80
|
-
# The script is generally fast, but may be slower with very large
|
81
|
-
# queues, which is why this is not enabled by default.
|
82
73
|
def probe_jobs
|
74
|
+
puts "[REMOVED] probe_jobs is now considered obsolete and does not emit any metrics,"\
|
75
|
+
" please use probe_jobs_limit instead"
|
76
|
+
|
77
|
+
self
|
78
|
+
end
|
79
|
+
|
80
|
+
def probe_future_sets
|
81
|
+
now = Time.now.to_f
|
83
82
|
with_sidekiq do
|
84
|
-
|
83
|
+
Sidekiq.redis do |conn|
|
84
|
+
Sidekiq::Scheduled::SETS.each do |set|
|
85
|
+
# Default to 0; if all jobs are due in the future, there is no "negative" delay.
|
86
|
+
delay = 0
|
85
87
|
|
86
|
-
|
87
|
-
|
88
|
-
stats = conn.evalsha(QUEUE_JOB_STATS_SHA, ["queue:#{queue.name}"])
|
89
|
-
job_stats.merge!(stats.to_h)
|
90
|
-
end
|
91
|
-
rescue Redis::CommandError # Could happen if the script exceeded the maximum run time (5 seconds by default)
|
92
|
-
# FIXME: Should we call SCRIPT KILL?
|
93
|
-
return self
|
94
|
-
end
|
88
|
+
_job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
|
89
|
+
delay = now - timestamp if timestamp
|
95
90
|
|
96
|
-
|
97
|
-
|
91
|
+
@metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay)
|
92
|
+
|
93
|
+
# zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
|
94
|
+
@metrics.add("sidekiq_#{set}_set_backlog_count",
|
95
|
+
conn.zcount(set, "-inf", now.to_s))
|
96
|
+
end
|
98
97
|
end
|
99
98
|
end
|
100
|
-
|
101
|
-
self
|
102
99
|
end
|
103
100
|
|
104
|
-
#
|
105
|
-
# PROBE_JOBS_LIMIT jobs in each queue. This means that we run a
|
101
|
+
# Count worker classes present in Sidekiq queues. This only looks at the
|
102
|
+
# first PROBE_JOBS_LIMIT jobs in each queue. This means that we run a
|
106
103
|
# single LRANGE command for each queue, which does not block other
|
107
|
-
# commands. For queues over PROBE_JOBS_LIMIT in size, this means
|
108
|
-
#
|
109
|
-
#
|
110
|
-
# larger.
|
111
|
-
#
|
112
|
-
# DO NOT USE this and probe_jobs together, as they export the same
|
113
|
-
# metric (sidekiq_enqueued_jobs).
|
104
|
+
# commands. For queues over PROBE_JOBS_LIMIT in size, this means that we
|
105
|
+
# will not have completely accurate statistics, but the probe performance
|
106
|
+
# will also not degrade as the queue gets larger.
|
114
107
|
def probe_jobs_limit
|
115
108
|
with_sidekiq do
|
116
109
|
job_stats = Hash.new(0)
|
@@ -218,14 +211,9 @@ module GitLab
|
|
218
211
|
def connected?
|
219
212
|
return @connected unless @connected.nil?
|
220
213
|
|
221
|
-
# This is also a good "connected check"
|
222
214
|
Sidekiq.redis do |conn|
|
223
|
-
|
224
|
-
# will be removed in redis-namespace 2.0.
|
225
|
-
conn.redis.script(:load, QUEUE_JOB_STATS_SCRIPT) unless conn.redis.script(:exists, QUEUE_JOB_STATS_SHA)
|
215
|
+
@connected = (conn.ping == "PONG")
|
226
216
|
end
|
227
|
-
|
228
|
-
@connected = true
|
229
217
|
rescue Redis::BaseConnectionError => e
|
230
218
|
@logger&.error "Error connecting to the Redis: #{e}"
|
231
219
|
@connected = false
|
@@ -89,7 +89,7 @@ module GitLab
|
|
89
89
|
|
90
90
|
get "/#{probe_name}" do
|
91
91
|
content_type "text/plain; version=0.0.4"
|
92
|
-
prober = Prober.new(
|
92
|
+
prober = Prober.new(metrics: PrometheusMetrics.new(include_timestamp: false), logger: logger, **opts)
|
93
93
|
|
94
94
|
prober.probe_all
|
95
95
|
prober.write_to(response)
|
data/spec/database/bloat_spec.rb
CHANGED
@@ -26,7 +26,7 @@ describe GitLab::Exporter::Database::BloatCollector do
|
|
26
26
|
end
|
27
27
|
|
28
28
|
describe GitLab::Exporter::Database::BloatProber do
|
29
|
-
let(:opts) { { bloat_types: %i[btree table] } }
|
29
|
+
let(:opts) { { bloat_types: %i[btree table], connection_string: "" } }
|
30
30
|
let(:metrics) { double("PrometheusMetrics", add: nil) }
|
31
31
|
let(:collector) { double("BloatCollector", run: data) }
|
32
32
|
|
@@ -43,7 +43,7 @@ describe GitLab::Exporter::Database::BloatProber do
|
|
43
43
|
end
|
44
44
|
|
45
45
|
describe "#probe_db" do
|
46
|
-
subject { described_class.new(
|
46
|
+
subject { described_class.new(metrics: metrics, collector: collector, logger: STDOUT, **opts).probe_db }
|
47
47
|
|
48
48
|
before do
|
49
49
|
expect(collector).to receive(:logger=).with(STDOUT)
|
@@ -92,7 +92,7 @@ describe GitLab::Exporter::Database::BloatProber do
|
|
92
92
|
describe "#write_to" do
|
93
93
|
let(:target) { double }
|
94
94
|
let(:metrics) { double("PrometheusMetrics", to_s: double) }
|
95
|
-
subject { described_class.new(
|
95
|
+
subject { described_class.new(metrics: metrics, **opts).write_to(target) }
|
96
96
|
|
97
97
|
it "writes to given target" do
|
98
98
|
expect(target).to receive(:write).with(metrics.to_s)
|
@@ -216,11 +216,12 @@ describe GitLab::Exporter::Database do
|
|
216
216
|
describe GitLab::Exporter::Database::CiBuildsProber do
|
217
217
|
let(:writer) { StringIO.new }
|
218
218
|
let(:prober) do
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
219
|
+
described_class.new(
|
220
|
+
connection_string: "host=localhost",
|
221
|
+
created_builds_counting_disabled: created_builds_counting_disabled,
|
222
|
+
unarchived_traces_offset_minutes: unarchived_traces_offset_minutes,
|
223
|
+
metrics: GitLab::Exporter::PrometheusMetrics.new(include_timestamp: false)
|
224
|
+
)
|
224
225
|
end
|
225
226
|
|
226
227
|
before do
|
data/spec/git_spec.rb
CHANGED
@@ -34,17 +34,17 @@ context "With valid pair of repositories" do
|
|
34
34
|
end
|
35
35
|
|
36
36
|
describe GitLab::Exporter::GitProber do
|
37
|
-
let(:options) {
|
37
|
+
let(:options) { { source: repos.cloned_repo, labels: {} } }
|
38
38
|
let(:output) { StringIO.new }
|
39
39
|
|
40
40
|
it "probes and monitors a pull" do
|
41
|
-
prober = GitLab::Exporter::GitProber.new(options)
|
41
|
+
prober = GitLab::Exporter::GitProber.new(**options)
|
42
42
|
prober.probe_pull.write_to(output)
|
43
43
|
expect(output.string).to match(/git_pull_time_milliseconds \d+ \d+/)
|
44
44
|
end
|
45
45
|
|
46
46
|
it "probes and monitors a push" do
|
47
|
-
prober = GitLab::Exporter::GitProber.new(options)
|
47
|
+
prober = GitLab::Exporter::GitProber.new(**options)
|
48
48
|
prober.probe_push.write_to(output)
|
49
49
|
expect(output.string).to match(/git_push_time_milliseconds \d+ \d+/)
|
50
50
|
end
|
data/spec/ruby_spec.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-exporter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 11.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Carranza
|
@@ -112,16 +112,16 @@ dependencies:
|
|
112
112
|
name: sinatra
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- -
|
115
|
+
- - "~>"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 2.0
|
117
|
+
version: 2.1.0
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- -
|
122
|
+
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: 2.0
|
124
|
+
version: 2.1.0
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: rspec
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -189,7 +189,6 @@ files:
|
|
189
189
|
- lib/gitlab_exporter/prometheus.rb
|
190
190
|
- lib/gitlab_exporter/ruby.rb
|
191
191
|
- lib/gitlab_exporter/sidekiq.rb
|
192
|
-
- lib/gitlab_exporter/sidekiq_queue_job_stats.lua
|
193
192
|
- lib/gitlab_exporter/util.rb
|
194
193
|
- lib/gitlab_exporter/version.rb
|
195
194
|
- lib/gitlab_exporter/web_exporter.rb
|
@@ -1,42 +0,0 @@
|
|
1
|
-
--
|
2
|
-
-- Adapted from https://github.com/mperham/sidekiq/blob/2f9258e4fe77991c526f7a65c92bcf792eef8338/lib/sidekiq/api.rb#L231
|
3
|
-
--
|
4
|
-
local queue_name = KEYS[1]
|
5
|
-
local initial_size = redis.call('llen', queue_name)
|
6
|
-
local deleted_size = 0
|
7
|
-
local page = 0
|
8
|
-
local page_size = 2000
|
9
|
-
local temp_job_stats = {}
|
10
|
-
local final_job_stats = {}
|
11
|
-
|
12
|
-
while true do
|
13
|
-
local range_start = page * page_size - deleted_size
|
14
|
-
local range_end = range_start + page_size - 1
|
15
|
-
local entries = redis.call('lrange', queue_name, range_start, range_end)
|
16
|
-
|
17
|
-
if #entries == 0 then
|
18
|
-
break
|
19
|
-
end
|
20
|
-
|
21
|
-
page = page + 1
|
22
|
-
|
23
|
-
for index, entry in next, entries do
|
24
|
-
local class = cjson.decode(entry)['class']
|
25
|
-
if class ~= nil then
|
26
|
-
if temp_job_stats[class] ~= nil then
|
27
|
-
temp_job_stats[class] = temp_job_stats[class] + 1
|
28
|
-
else
|
29
|
-
temp_job_stats[class] = 1
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
deleted_size = initial_size - redis.call('llen', queue_name)
|
35
|
-
end
|
36
|
-
|
37
|
-
for class, count in next, temp_job_stats do
|
38
|
-
local stat_entry = {class, count}
|
39
|
-
table.insert(final_job_stats, stat_entry)
|
40
|
-
end
|
41
|
-
|
42
|
-
return final_job_stats
|