gitlab-exporter 10.4.0 → 10.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ace0ad673d8d36d1a6dbbacefb7f6aefde7d795e76eda8d3788c4cb4f5eb5247
4
- data.tar.gz: ac42d2e6deaaa3fee45ed74ec1cec81f402abba0c4f14798878b2b006c72400b
3
+ metadata.gz: 4c392268186f090b025d290c818f821120f5160a9a59f2e3ba228f26da0b8505
4
+ data.tar.gz: 3771d5912cb2a310183a1f78a8e24d191eb502d20c2beab3c327994243ae7077
5
5
  SHA512:
6
- metadata.gz: d3ac36c2b2ec584cb8ac60355649eb79d5a3d96e85a266f14794627fc3a9fa61a5cfb231f6c33e044cbb2e71d7965c50463b6673f80c69de4ebca58d658280e6
7
- data.tar.gz: 43323ec5663db9dd865e84b3c7907ba43f805a958904cfef57d7566b6259a31cc1a7aa96c494ffb976b4b04148c2fbfe63bd688844672ad1938c41bf05181a4b
6
+ metadata.gz: c9ee25b554e51484e4cba96070677c90e3cbad17da49eef2e2f649ad4d4ff125e8e2982cfb6e952444436d7ead6f603f6173ba1270b7112b3de75126cb57b318
7
+ data.tar.gz: e6038eff74531eb7c0561437cb676336853d9a597b588d07a9561a4561ef165a3ad2f528711b3a2f5d61ebfc8df341ef93ec4d15274fb7efcc88859abc1bc6c3
data/.gitlab-ci.yml CHANGED
@@ -6,6 +6,10 @@ include:
6
6
  - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
7
7
  - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
8
8
 
9
+ stages:
10
+ - test
11
+ - dast
12
+
9
13
  default:
10
14
  image: ruby:2.7
11
15
  cache:
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab-exporter (10.4.0)
4
+ gitlab-exporter (10.5.0)
5
5
  connection_pool (= 2.2.5)
6
6
  pg (= 1.2.3)
7
7
  puma (= 5.3.2)
@@ -78,8 +78,6 @@ probes:
78
78
  opts:
79
79
  - pid_or_pattern: "sidekiq .* \\[.*?\\]"
80
80
  name: sidekiq
81
- - pid_or_pattern: "unicorn.* worker\\[.*?\\]"
82
- name: unicorn
83
81
  - pid_or_pattern: "git-upload-pack --stateless-rpc"
84
82
  name: git_upload_pack
85
83
  quantiles: true
@@ -217,7 +217,7 @@ module GitLab
217
217
  opts.on("--pid=123", "Process ID") do |val|
218
218
  @pid = val
219
219
  end
220
- opts.on("--pattern=unicorn", "Process command pattern") do |val|
220
+ opts.on("--pattern=worker", "Process command pattern") do |val|
221
221
  @pattern = val
222
222
  end
223
223
  opts.on("--name=NAME", "Process name to be used in metrics") do |val|
@@ -15,6 +15,21 @@ module GitLab
15
15
  @include_timestamp = include_timestamp
16
16
  end
17
17
 
18
+ class << self
19
+ def describe(name, description)
20
+ @metric_descriptions ||= {}
21
+ @metric_descriptions[name] = description
22
+ end
23
+
24
+ def description(name)
25
+ @metric_descriptions && @metric_descriptions[name]
26
+ end
27
+
28
+ def clear_descriptions
29
+ @metric_descriptions = {}
30
+ end
31
+ end
32
+
18
33
  def add(name, value, quantile = false, **labels)
19
34
  fail "value '#{value}' must be a number" unless value.is_a?(Numeric)
20
35
 
@@ -32,6 +47,8 @@ module GitLab
32
47
 
33
48
  buffer = ""
34
49
  @metrics.each do |name, measurements|
50
+ buffer << "# HELP #{name} #{self.class.description(name)}\n" if self.class.description(name)
51
+
35
52
  measurements.each do |measurement|
36
53
  buffer << name.to_s
37
54
  labels = (measurement[:labels] || {}).map { |label, value| "#{label}=\"#{value}\"" }.join(",")
@@ -10,6 +10,13 @@ module GitLab
10
10
  QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
11
11
  QUEUE_JOB_STATS_SHA = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
12
12
 
13
+ # The maximum depth (from the head) of each queue to probe. Probing the
14
+ # entirety of a very large queue will take longer and run the risk of
15
+ # timing out. But when we have a very large queue, we are most in need of
16
+ # reliable metrics. This trades off completeness for predictability by
17
+ # only taking a limited amount of items from the head of the queue.
18
+ PROBE_JOBS_LIMIT = 1_000
19
+
13
20
  POOL_SIZE = 3
14
21
 
15
22
  # This timeout is configured to higher interval than scrapping
@@ -17,6 +24,9 @@ module GitLab
17
24
  # needed to be re-initialized
18
25
  POOL_TIMEOUT = 90
19
26
 
27
+ PrometheusMetrics.describe("sidekiq_enqueued_jobs",
28
+ "Total number of jobs enqueued by class name. Only inspects the first #{PROBE_JOBS_LIMIT} jobs per queue.") # rubocop:disable Layout/LineLength
29
+
20
30
  def self.connection_pool
21
31
  @@connection_pool ||= Hash.new do |h, connection_hash| # rubocop:disable Style/ClassVars
22
32
  config = connection_hash.merge(pool_timeout: POOL_TIMEOUT, size: POOL_SIZE)
@@ -62,6 +72,13 @@ module GitLab
62
72
  self
63
73
  end
64
74
 
75
+ # Count worker classes present in Sidekiq queues. This uses a Lua
76
+ # script to find all jobs in all queues. That script will block
77
+ # all other Redis commands:
78
+ # https://redis.io/commands/eval#atomicity-of-scripts
79
+ #
80
+ # The script is generally fast, but may be slower with very large
81
+ # queues, which is why this is not enabled by default.
65
82
  def probe_jobs
66
83
  with_sidekiq do
67
84
  job_stats = {}
@@ -84,6 +101,38 @@ module GitLab
84
101
  self
85
102
  end
86
103
 
104
+ # This does the same as #probe_jobs, but only looks at the first
105
+ # PROBE_JOBS_LIMIT jobs in each queue. This means that we run a
106
+ # single LRANGE command for each queue, which does not block other
107
+ # commands. For queues over PROBE_JOBS_LIMIT in size, this means
108
+ # that we will not have completely accurate statistics, but the
109
+ # probe performance will also not degrade as the queue gets
110
+ # larger.
111
+ #
112
+ # DO NOT USE this and probe_jobs together, as they export the same
113
+ # metric (sidekiq_enqueued_jobs).
114
+ def probe_jobs_limit
115
+ with_sidekiq do
116
+ job_stats = Hash.new(0)
117
+
118
+ Sidekiq::Queue.all.each do |queue|
119
+ Sidekiq.redis do |conn|
120
+ conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
121
+ job_class = Sidekiq.load_json(job)["class"]
122
+
123
+ job_stats[job_class] += 1
124
+ end
125
+ end
126
+ end
127
+
128
+ job_stats.each do |class_name, count|
129
+ @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
130
+ end
131
+ end
132
+
133
+ self
134
+ end
135
+
87
136
  def probe_workers
88
137
  with_sidekiq do
89
138
  worker_stats = Hash.new(0)
@@ -1,5 +1,5 @@
1
1
  module GitLab
2
2
  module Exporter
3
- VERSION = "10.4.0".freeze
3
+ VERSION = "10.5.0".freeze
4
4
  end
5
5
  end
@@ -23,4 +23,21 @@ describe GitLab::Exporter::PrometheusMetrics do
23
23
  subject.add("mymetric", "invalid", mylabel: "x", myotherlabel: "y").to_s
24
24
  }.to raise_error(RuntimeError)
25
25
  end
26
+
27
+ it "supports described metrics" do
28
+ time = Time.now
29
+
30
+ allow(Time).to receive(:now).and_return(time)
31
+
32
+ described_class.describe("mymetric", "description")
33
+ described_class.describe("missingmetric", "otherdescription")
34
+ subject.add("mymetric", 1.3, mylabel: "x", myotherlabel: "y")
35
+
36
+ expect(subject.to_s).to eq(<<~METRICS)
37
+ # HELP mymetric description
38
+ mymetric{mylabel="x",myotherlabel="y"} 1.3 #{(time.to_f * 1000).to_i}
39
+ METRICS
40
+
41
+ described_class.clear_descriptions
42
+ end
26
43
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-exporter
3
3
  version: !ruby/object:Gem::Version
4
- version: 10.4.0
4
+ version: 10.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Carranza