gitlab-exporter 11.3.0 → 11.7.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 402b7efd2fde05a6ab82ff5728f1bbfd1dfae2d4485f54c557cb6a61aaa073c1
4
- data.tar.gz: 5e002911773de121aaa43722e56f7720e1ea75c7e22d360441b265d02fcc85c5
3
+ metadata.gz: 2c6544de4dbf86995510d054204bbe197f922819aba46715416001651719f44f
4
+ data.tar.gz: 276cb7d17ddaa79d319358db925be4f7d3d8accdbc9670e5fb4b662ad62150e4
5
5
  SHA512:
6
- metadata.gz: 8d5e1c40950e6d1581b73c14366f95777e6a06880a1a988a30945cd27d906211ac93d4312f2e890eeab9ddb1f9fccabdbf51a0c13218e4a494de3fea4663c968
7
- data.tar.gz: c58bf8aa65703c53a08fba57d54625f9bb2924ac5852398aba7b5e0d0f1916640ab9f4fe92ef2b651e334520bfe488ec7884def9585e4e7b49b42e32d62bcf8d
6
+ metadata.gz: f0d3a0a8e58cbc95ccb9c1ef43320ac52ade6e62bf44cf282ebd010a365832b8818fc045ff2a658cc2a591989b07abfb1d0554af5aff978b25046c63e8d5971a
7
+ data.tar.gz: '08fa579ef431761feabe06421f28fe450fdf3ffb7e41453a3d1dfe3ee97dabb8afef1165b090cf2b63cfb017d500ba22f1b69ca17b1012650ce4391675a2dc97'
data/Gemfile.lock CHANGED
@@ -1,14 +1,15 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab-exporter (11.3.0)
4
+ gitlab-exporter (11.7.0)
5
5
  connection_pool (= 2.2.5)
6
+ faraday (~> 1.8.0)
6
7
  pg (= 1.2.3)
7
8
  puma (= 5.3.2)
8
9
  quantile (= 0.2.1)
9
- redis (= 4.1.4)
10
+ redis (= 4.4.0)
10
11
  redis-namespace (= 1.6.0)
11
- sidekiq (= 5.2.9)
12
+ sidekiq (= 6.2.2)
12
13
  sinatra (~> 2.1.0)
13
14
 
14
15
  GEM
@@ -17,6 +18,26 @@ GEM
17
18
  ast (2.4.1)
18
19
  connection_pool (2.2.5)
19
20
  diff-lcs (1.3)
21
+ faraday (1.8.0)
22
+ faraday-em_http (~> 1.0)
23
+ faraday-em_synchrony (~> 1.0)
24
+ faraday-excon (~> 1.1)
25
+ faraday-httpclient (~> 1.0.1)
26
+ faraday-net_http (~> 1.0)
27
+ faraday-net_http_persistent (~> 1.1)
28
+ faraday-patron (~> 1.0)
29
+ faraday-rack (~> 1.0)
30
+ multipart-post (>= 1.2, < 3)
31
+ ruby2_keywords (>= 0.0.4)
32
+ faraday-em_http (1.0.0)
33
+ faraday-em_synchrony (1.0.0)
34
+ faraday-excon (1.1.0)
35
+ faraday-httpclient (1.0.1)
36
+ faraday-net_http (1.0.1)
37
+ faraday-net_http_persistent (1.2.0)
38
+ faraday-patron (1.0.0)
39
+ faraday-rack (1.0.0)
40
+ multipart-post (2.1.1)
20
41
  mustermann (1.1.1)
21
42
  ruby2_keywords (~> 0.0.1)
22
43
  nio4r (2.5.8)
@@ -31,7 +52,7 @@ GEM
31
52
  rack-protection (2.1.0)
32
53
  rack
33
54
  rainbow (3.0.0)
34
- redis (4.1.4)
55
+ redis (4.4.0)
35
56
  redis-namespace (1.6.0)
36
57
  redis (>= 3.0.4)
37
58
  regexp_parser (2.0.3)
@@ -62,11 +83,10 @@ GEM
62
83
  parser (>= 2.7.1.5)
63
84
  ruby-progressbar (1.11.0)
64
85
  ruby2_keywords (0.0.5)
65
- sidekiq (5.2.9)
66
- connection_pool (~> 2.2, >= 2.2.2)
86
+ sidekiq (6.2.2)
87
+ connection_pool (>= 2.2.2)
67
88
  rack (~> 2.0)
68
- rack-protection (>= 1.5.0)
69
- redis (>= 3.3.5, < 4.2)
89
+ redis (>= 4.2.0)
70
90
  sinatra (2.1.0)
71
91
  mustermann (~> 1.0)
72
92
  rack (~> 2.2)
data/README.md CHANGED
@@ -59,6 +59,8 @@ metrics.
59
59
  * `sidekiq_schedule_set_backlog_count`
60
60
  * `sidekiq_retry_set_processing_delay_seconds`
61
61
  * `sidekiq_retry_set_backlog_count`
62
+ 1. [Elasticsearch](lib/gitlab_exporter/elasticsearch.rb)
63
+ * [Migrations](https://docs.gitlab.com/ee/integration/elasticsearch.html#advanced-search-migrations) -- `elasticsearch_migrations_state`
62
64
 
63
65
  ### Setup with GitLab Development Kit
64
66
 
@@ -99,6 +99,22 @@ probes:
99
99
  opts:
100
100
  quantiles: false
101
101
 
102
+
103
+ # Uncomment below to add GitLab specific Elasticsearch metrics.
104
+ #
105
+ # To support multiple authorization types, opts are passed to a Faraday connection object.
106
+ # See: https://www.rubydoc.info/github/lostisland/faraday/Faraday%2FConnection:initialize
107
+ #
108
+ # elasticsearch: &elasticsearch
109
+ # methods:
110
+ # - probe_migrations
111
+ #
112
+ # opts:
113
+ # - url: http://localhost:9200
114
+ # options:
115
+ # headers:
116
+ # Authorization: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="
117
+
102
118
  metrics:
103
119
  multiple: true
104
120
  git_process:
@@ -21,12 +21,13 @@ Gem::Specification.new do |s|
21
21
  s.license = "MIT"
22
22
 
23
23
  s.add_runtime_dependency "connection_pool", "2.2.5"
24
+ s.add_runtime_dependency "faraday", "~> 1.8.0"
24
25
  s.add_runtime_dependency "pg", "1.2.3"
25
26
  s.add_runtime_dependency "puma", "5.3.2"
26
27
  s.add_runtime_dependency "quantile", "0.2.1"
27
- s.add_runtime_dependency "redis", "4.1.4"
28
+ s.add_runtime_dependency "redis", "4.4.0"
28
29
  s.add_runtime_dependency "redis-namespace", "1.6.0"
29
- s.add_runtime_dependency "sidekiq", "5.2.9"
30
+ s.add_runtime_dependency "sidekiq", "6.2.2"
30
31
  s.add_runtime_dependency "sinatra", "~> 2.1.0"
31
32
 
32
33
  s.add_development_dependency "rspec", "~> 3.7.0"
@@ -131,12 +131,7 @@ module GitLab
131
131
  visibility_level: {},
132
132
  root: { definition: "(parent_id IS NULL)" }
133
133
  }
134
- },
135
- registry_gc_manifest_review_queue: { select: :gc_manifest_review_queue },
136
- registry_gc_blob_review_queue: { select: :gc_blob_review_queue },
137
- registry_repositories: { select: :repositories },
138
- registry_manifests: { select: :manifests },
139
- registry_blobs: { select: :blobs }
134
+ }
140
135
  }.freeze
141
136
 
142
137
  def initialize(selected_queries: nil, **args)
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "faraday"
4
+ require "json"
5
+
6
+ module GitLab
7
+ module Exporter
8
+ # Exports GitLab specific Elasticsearch metrics.
9
+ #
10
+ # For generic operational metrics, see elasticsearch_exporter.
11
+ # https://github.com/prometheus-community/elasticsearch_exporter
12
+ class ElasticsearchProber
13
+ MIGRATION_STATE_MAP = {
14
+ unknown: -9,
15
+ # TODO: failed: -1
16
+ pending: 0,
17
+ running: 1,
18
+ halted: 2,
19
+ completed: 3
20
+ }.freeze
21
+
22
+ def initialize(metrics: PrometheusMetrics.new, logger: nil, **opts)
23
+ @metrics = metrics
24
+ @logger = logger
25
+ @opts = opts
26
+ end
27
+
28
+ # Probes the state of Advanced Search Migrations
29
+ # https://docs.gitlab.com/ee/integration/elasticsearch.html#advanced-search-migrations
30
+ def probe_migrations
31
+ elastic_probe do |conn|
32
+ resp = conn.get "/gitlab-*-migrations/_search"
33
+ return unless resp.status == 200
34
+
35
+ JSON.parse(resp.body).dig("hits", "hits").each do |hit|
36
+ @metrics.add(
37
+ "elasticsearch_migrations_info", 1, # 1 is a noop.
38
+ state: inferred_migration_state(hit.fetch("_source")),
39
+ name: hit.fetch("_id")
40
+ )
41
+ end
42
+ end
43
+ rescue StandardError => e
44
+ @logger&.error "ElasticsearchProper encountered an error: #{e}"
45
+ end
46
+
47
+ private
48
+
49
+ def elastic_probe
50
+ yield Faraday.new(@opts.fetch(:url), @opts.fetch(:options, {}))
51
+ end
52
+
53
+ def inferred_migration_state(migration)
54
+ return :pending if migration["started_at"] == ""
55
+
56
+ if migration["started_at"] != "" && migration["completed_at"] == "" && !migration.dig("state", "halted")
57
+ return :running
58
+ end
59
+
60
+ return :completed if migration["completed"]
61
+ return :halted if migration.dig("state", "halted")
62
+
63
+ @logger&.error("Elasticsearch probe doesn't know the state of a migration")
64
+ :unknown
65
+ end
66
+ end
67
+ end
68
+ end
@@ -1,5 +1,5 @@
1
1
  module GitLab
2
2
  module Exporter
3
- VERSION = "11.3.0".freeze
3
+ VERSION = "11.7.0".freeze
4
4
  end
5
5
  end
@@ -1,19 +1,20 @@
1
1
  module GitLab
2
2
  # GitLab Monitoring
3
3
  module Exporter
4
- autoload :CLI, "gitlab_exporter/cli"
5
- autoload :TimeTracker, "gitlab_exporter/util"
6
- autoload :Utils, "gitlab_exporter/util"
7
- autoload :PrometheusMetrics, "gitlab_exporter/prometheus"
8
- autoload :Utils, "gitlab_exporter/util"
9
- autoload :Git, "gitlab_exporter/git"
10
- autoload :GitProber, "gitlab_exporter/git"
11
- autoload :GitProcessProber, "gitlab_exporter/git"
12
- autoload :Database, "gitlab_exporter/database"
13
- autoload :ProcessProber, "gitlab_exporter/process"
14
- autoload :WebExporter, "gitlab_exporter/web_exporter"
15
- autoload :Prober, "gitlab_exporter/prober"
16
- autoload :SidekiqProber, "gitlab_exporter/sidekiq"
17
- autoload :RubyProber, "gitlab_exporter/ruby"
4
+ autoload :CLI, "gitlab_exporter/cli"
5
+ autoload :TimeTracker, "gitlab_exporter/util"
6
+ autoload :Utils, "gitlab_exporter/util"
7
+ autoload :PrometheusMetrics, "gitlab_exporter/prometheus"
8
+ autoload :Utils, "gitlab_exporter/util"
9
+ autoload :Git, "gitlab_exporter/git"
10
+ autoload :GitProber, "gitlab_exporter/git"
11
+ autoload :GitProcessProber, "gitlab_exporter/git"
12
+ autoload :Database, "gitlab_exporter/database"
13
+ autoload :ProcessProber, "gitlab_exporter/process"
14
+ autoload :WebExporter, "gitlab_exporter/web_exporter"
15
+ autoload :Prober, "gitlab_exporter/prober"
16
+ autoload :SidekiqProber, "gitlab_exporter/sidekiq"
17
+ autoload :RubyProber, "gitlab_exporter/ruby"
18
+ autoload :ElasticsearchProber, "gitlab_exporter/elasticsearch"
18
19
  end
19
20
  end
@@ -0,0 +1,132 @@
1
+ require "spec_helper"
2
+ require "gitlab_exporter/elasticsearch"
3
+
4
+ describe GitLab::Exporter::ElasticsearchProber do
5
+ subject(:prober) { GitLab::Exporter::Prober.new(**options) }
6
+ let(:options) do
7
+ {
8
+ elasticsearch: {
9
+ class_name: described_class.to_s,
10
+ methods: %i[probe_migrations],
11
+ opts: {
12
+ url: "http://elasticsearch"
13
+ }
14
+ }
15
+ }
16
+ end
17
+
18
+ let(:connection) { instance_double(Faraday::Connection) }
19
+
20
+ describe "probe_migrations" do
21
+ let(:response) { double(:response, body: results, status: 200) }
22
+ let(:metric_name) { "elasticsearch_migrations_info" }
23
+
24
+ let(:results) do
25
+ <<-RESULTS
26
+ {
27
+ "took": 5,
28
+ "timed_out": false,
29
+ "_shards": {
30
+ "total": 1,
31
+ "successful": 1,
32
+ "skipped": 0,
33
+ "failed": 0
34
+ },
35
+ "hits": {
36
+ "total": {
37
+ "value": 1,
38
+ "relation": "eq"
39
+ },
40
+ "max_score": 1,
41
+ "hits": [#{document.to_json}]
42
+ }
43
+ }
44
+ RESULTS
45
+ end
46
+
47
+ let(:document) do
48
+ { "_id" => "123", "_source" => migration }
49
+ end
50
+
51
+ let(:migration) do
52
+ {
53
+ "started_at" => "",
54
+ "completed_at" => "",
55
+ "completed" => false,
56
+ "state" => {}
57
+ }.tap do |m|
58
+ case migration_state
59
+ when "running"
60
+ m["started_at"] = "2021-09-23T11:47:48,890Z"
61
+ when "completed"
62
+ m["started_at"] = "2021-09-23T11:47:48,890Z"
63
+ m["completed_at"] = "2021-09-24T11:47:48,890Z"
64
+ m["completed"] = true
65
+ when "halted"
66
+ m["started_at"] = "2021-09-23T11:47:48,890Z"
67
+ m["state"]["halted"] = true
68
+ end
69
+ end
70
+ end
71
+
72
+ let(:output) do
73
+ prober.probe_all
74
+ StringIO.new.tap do |s|
75
+ prober.write_to(s)
76
+ end
77
+ end
78
+
79
+ before do
80
+ allow_any_instance_of(described_class).to receive(:elastic_probe).and_yield(connection)
81
+ end
82
+
83
+ describe "when an Exception is raised" do
84
+ before do
85
+ allow(connection).to receive(:get).with("/gitlab-*-migrations/_search").and_raise("boom")
86
+ end
87
+
88
+ it "does NOT raise an error" do
89
+ expect { output }.to_not raise_error
90
+ end
91
+
92
+ it "does NOT export any metrics" do
93
+ expect(output.string).not_to match(/#{metric_name}/)
94
+ end
95
+ end
96
+
97
+ describe "when elasticsearch is reachable" do
98
+ let(:expected_output) { /#{metric_name}{state="#{migration_state}",name="123"} 1 \d+/ }
99
+ before do
100
+ allow(connection).to receive(:get).with("/gitlab-*-migrations/_search").and_return response
101
+ end
102
+
103
+ context "when there is a pending migration" do
104
+ let(:migration_state) { "pending" }
105
+ it "exports state correctly" do
106
+ expect(output.string).to match(expected_output)
107
+ end
108
+ end
109
+
110
+ context "when there is a running migration" do
111
+ let(:migration_state) { "running" }
112
+ it "exports state correctly" do
113
+ expect(output.string).to match(expected_output)
114
+ end
115
+ end
116
+
117
+ context "when there is a halted migration" do
118
+ let(:migration_state) { "halted" }
119
+ it "exports state correctly" do
120
+ expect(output.string).to match(expected_output)
121
+ end
122
+ end
123
+
124
+ context "when there is a completed migration" do
125
+ let(:migration_state) { "completed" }
126
+ it "exports state correctly" do
127
+ expect(output.string).to match(expected_output)
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab-exporter
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.3.0
4
+ version: 11.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Carranza
@@ -24,6 +24,20 @@ dependencies:
24
24
  - - '='
25
25
  - !ruby/object:Gem::Version
26
26
  version: 2.2.5
27
+ - !ruby/object:Gem::Dependency
28
+ name: faraday
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: 1.8.0
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 1.8.0
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: pg
29
43
  requirement: !ruby/object:Gem::Requirement
@@ -72,14 +86,14 @@ dependencies:
72
86
  requirements:
73
87
  - - '='
74
88
  - !ruby/object:Gem::Version
75
- version: 4.1.4
89
+ version: 4.4.0
76
90
  type: :runtime
77
91
  prerelease: false
78
92
  version_requirements: !ruby/object:Gem::Requirement
79
93
  requirements:
80
94
  - - '='
81
95
  - !ruby/object:Gem::Version
82
- version: 4.1.4
96
+ version: 4.4.0
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: redis-namespace
85
99
  requirement: !ruby/object:Gem::Requirement
@@ -100,14 +114,14 @@ dependencies:
100
114
  requirements:
101
115
  - - '='
102
116
  - !ruby/object:Gem::Version
103
- version: 5.2.9
117
+ version: 6.2.2
104
118
  type: :runtime
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
122
  - - '='
109
123
  - !ruby/object:Gem::Version
110
- version: 5.2.9
124
+ version: 6.2.2
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: sinatra
113
127
  requirement: !ruby/object:Gem::Requirement
@@ -181,6 +195,7 @@ files:
181
195
  - lib/gitlab_exporter/database/remote_mirrors.rb
182
196
  - lib/gitlab_exporter/database/row_count.rb
183
197
  - lib/gitlab_exporter/database/tuple_stats.rb
198
+ - lib/gitlab_exporter/elasticsearch.rb
184
199
  - lib/gitlab_exporter/git.rb
185
200
  - lib/gitlab_exporter/memstats.rb
186
201
  - lib/gitlab_exporter/memstats/mapping.rb
@@ -196,6 +211,7 @@ files:
196
211
  - spec/database/bloat_spec.rb
197
212
  - spec/database/ci_builds_spec.rb
198
213
  - spec/database/row_count_spec.rb
214
+ - spec/elasticsearch_spec.rb
199
215
  - spec/fixtures/smaps/sample.txt
200
216
  - spec/git_process_proper_spec.rb
201
217
  - spec/git_spec.rb
@@ -223,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
223
239
  - !ruby/object:Gem::Version
224
240
  version: '0'
225
241
  requirements: []
226
- rubygems_version: 3.1.4
242
+ rubygems_version: 3.0.3
227
243
  signing_key:
228
244
  specification_version: 4
229
245
  summary: GitLab metrics exporter
@@ -232,6 +248,7 @@ test_files:
232
248
  - spec/database/bloat_spec.rb
233
249
  - spec/database/ci_builds_spec.rb
234
250
  - spec/database/row_count_spec.rb
251
+ - spec/elasticsearch_spec.rb
235
252
  - spec/fixtures/smaps/sample.txt
236
253
  - spec/git_process_proper_spec.rb
237
254
  - spec/git_spec.rb