gitlab-monitor 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +2 -0
  3. data/.gitlab-ci.yml +18 -0
  4. data/.rubocop.yml +34 -0
  5. data/CONTRIBUTING.md +651 -0
  6. data/Gemfile +8 -0
  7. data/Gemfile.lock +75 -0
  8. data/LICENSE +25 -0
  9. data/README.md +110 -0
  10. data/bin/gitlab-mon +17 -0
  11. data/config/gitlab-monitor.yml.example +112 -0
  12. data/gitlab-monitor.gemspec +33 -0
  13. data/lib/gitlab_monitor.rb +18 -0
  14. data/lib/gitlab_monitor/cli.rb +341 -0
  15. data/lib/gitlab_monitor/database.rb +13 -0
  16. data/lib/gitlab_monitor/database/base.rb +44 -0
  17. data/lib/gitlab_monitor/database/bloat.rb +74 -0
  18. data/lib/gitlab_monitor/database/bloat_btree.sql +84 -0
  19. data/lib/gitlab_monitor/database/bloat_table.sql +63 -0
  20. data/lib/gitlab_monitor/database/ci_builds.rb +527 -0
  21. data/lib/gitlab_monitor/database/remote_mirrors.rb +74 -0
  22. data/lib/gitlab_monitor/database/row_count.rb +164 -0
  23. data/lib/gitlab_monitor/database/tuple_stats.rb +53 -0
  24. data/lib/gitlab_monitor/git.rb +144 -0
  25. data/lib/gitlab_monitor/memstats.rb +98 -0
  26. data/lib/gitlab_monitor/memstats/mapping.rb +91 -0
  27. data/lib/gitlab_monitor/prober.rb +40 -0
  28. data/lib/gitlab_monitor/process.rb +122 -0
  29. data/lib/gitlab_monitor/prometheus.rb +64 -0
  30. data/lib/gitlab_monitor/sidekiq.rb +149 -0
  31. data/lib/gitlab_monitor/sidekiq_queue_job_stats.lua +42 -0
  32. data/lib/gitlab_monitor/util.rb +83 -0
  33. data/lib/gitlab_monitor/version.rb +5 -0
  34. data/lib/gitlab_monitor/web_exporter.rb +77 -0
  35. data/spec/cli_spec.rb +31 -0
  36. data/spec/database/bloat_spec.rb +99 -0
  37. data/spec/database/ci_builds_spec.rb +421 -0
  38. data/spec/database/row_count_spec.rb +37 -0
  39. data/spec/fixtures/smaps/sample.txt +10108 -0
  40. data/spec/git_process_proper_spec.rb +27 -0
  41. data/spec/git_spec.rb +52 -0
  42. data/spec/memstats_spec.rb +28 -0
  43. data/spec/prometheus_metrics_spec.rb +17 -0
  44. data/spec/spec_helper.rb +63 -0
  45. data/spec/util_spec.rb +15 -0
  46. metadata +225 -0
@@ -0,0 +1,42 @@
1
+ --
2
+ -- Adapted from https://github.com/mperham/sidekiq/blob/2f9258e4fe77991c526f7a65c92bcf792eef8338/lib/sidekiq/api.rb#L231
3
+ --
4
+ local queue_name = KEYS[1]
5
+ local initial_size = redis.call('llen', queue_name)
6
+ local deleted_size = 0
7
+ local page = 0
8
+ local page_size = 2000
9
+ local temp_job_stats = {}
10
+ local final_job_stats = {}
11
+
12
+ while true do
13
+ local range_start = page * page_size - deleted_size
14
+ local range_end = range_start + page_size - 1
15
+ local entries = redis.call('lrange', queue_name, range_start, range_end)
16
+
17
+ if #entries == 0 then
18
+ break
19
+ end
20
+
21
+ page = page + 1
22
+
23
+ for index, entry in next, entries do
24
+ local class = cjson.decode(entry)['class']
25
+ if class ~= nil then
26
+ if temp_job_stats[class] ~= nil then
27
+ temp_job_stats[class] = temp_job_stats[class] + 1
28
+ else
29
+ temp_job_stats[class] = 1
30
+ end
31
+ end
32
+ end
33
+
34
+ deleted_size = initial_size - redis.call('llen', queue_name)
35
+ end
36
+
37
+ for class, count in next, temp_job_stats do
38
+ local stat_entry = {class, count}
39
+ table.insert(final_job_stats, stat_entry)
40
+ end
41
+
42
+ return final_job_stats
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ module GitLab
4
+ module Monitor
5
+ # Simple time wrapper that provides a to_i and wraps the execution result
6
+ TrackedResult = Struct.new(:result, :time) do
7
+ def to_i
8
+ time
9
+ end
10
+ end
11
+
12
+ # Time tracking object
13
+ #
14
+ # Provides a simple time tracking, and returns back the result plus the tracked time
15
+ # wraped in a TrackedResult struct
16
+ class TimeTracker
17
+ def track
18
+ @start = Time.now.to_f
19
+ result = yield
20
+ TrackedResult.new(result, Time.now.to_f - @start)
21
+ end
22
+ end
23
+
24
+ # Helper methods, some stuff was copied from ActiveSupport
25
+ module Utils
26
+ def camel_case_string(str)
27
+ str.gsub(/(?:_|^)([a-z\d]*)/i) { $1.capitalize } # rubocop:disable PerlBackrefs
28
+ end
29
+ module_function :camel_case_string
30
+
31
+ def deep_symbolize_hash_keys(hash)
32
+ deep_transform_keys_in_object(hash, &:to_sym)
33
+ end
34
+ module_function :deep_symbolize_hash_keys
35
+
36
+ def deep_transform_keys_in_object(object, &block)
37
+ case object
38
+ when Hash
39
+ object.keys.each do |key|
40
+ value = object.delete(key)
41
+ object[yield(key)] = deep_transform_keys_in_object(value, &block)
42
+ end
43
+ object
44
+ when Array
45
+ object.map! { |e| deep_transform_keys_in_object(e, &block) }
46
+ else
47
+ object
48
+ end
49
+ end
50
+ module_function :deep_transform_keys_in_object
51
+
52
+ def pgrep(pattern)
53
+ # pgrep will include the PID of the shell, so strip that out
54
+ exec_pgrep(pattern).split("\n").each_with_object([]) do |line, arr|
55
+ pid, name = line.split(" ")
56
+ arr << pid if name != "sh"
57
+ end
58
+ end
59
+ module_function :pgrep
60
+
61
+ def exec_pgrep(pattern)
62
+ `pgrep -fl "#{pattern}"`
63
+ end
64
+ module_function :exec_pgrep
65
+
66
+ def system_uptime
67
+ File.read("/proc/uptime").split(" ")[0].to_f
68
+ end
69
+ module_function :system_uptime
70
+
71
+ def wrap_in_array(object)
72
+ if object.nil?
73
+ []
74
+ elsif object.respond_to?(:to_ary)
75
+ object.to_ary || [object]
76
+ else
77
+ [object]
78
+ end
79
+ end
80
+ module_function :wrap_in_array
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,5 @@
1
+ module GitLab
2
+ module Monitor
3
+ VERSION = "4.0.0".freeze
4
+ end
5
+ end
@@ -0,0 +1,77 @@
1
+ require "sinatra/base"
2
+ require "English"
3
+
4
+ module GitLab
5
+ module Monitor
6
+ # Metrics web exporter
7
+ class WebExporter < Sinatra::Base
8
+ # A middleware to kill the process if we exceeded a certain threshold
9
+ class MemoryKillerMiddleware
10
+ def initialize(app, memory_threshold)
11
+ @app = app
12
+ @memory_threshold = memory_threshold.to_i * 1024
13
+ end
14
+
15
+ def call(env)
16
+ if memory_usage > @memory_threshold
17
+ puts "Memory usage of #{memory_usage} exceeded threshold of #{@memory_threshold}, signalling KILL"
18
+ Process.kill("KILL", $PID)
19
+ end
20
+
21
+ @app.call(env)
22
+ end
23
+
24
+ private
25
+
26
+ def memory_usage
27
+ io = IO.popen(%W(ps -o rss= -p #{$PID}))
28
+
29
+ mem = io.read
30
+ io.close
31
+
32
+ return 0 unless $CHILD_STATUS.to_i.zero?
33
+
34
+ mem.to_i
35
+ end
36
+ end
37
+
38
+ class << self
39
+ def setup(config)
40
+ setup_server(config[:server])
41
+ setup_probes(config[:probes])
42
+
43
+ memory_threshold = (config[:server] && config[:server][:memory_threshold]) || 1024
44
+ use MemoryKillerMiddleware, memory_threshold
45
+ end
46
+
47
+ def setup_server(config)
48
+ config ||= {}
49
+
50
+ set(:bind, config.fetch(:listen_address, "0.0.0.0"))
51
+ set(:port, config.fetch(:listen_port, 9168))
52
+ end
53
+
54
+ def setup_probes(config)
55
+ (config || {}).each do |probe_name, params|
56
+ opts =
57
+ if params.delete(:multiple)
58
+ params
59
+ else
60
+ { probe_name => params }
61
+ end
62
+
63
+ get "/#{probe_name}" do
64
+ content_type "text/plain; version=0.0.4"
65
+ prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false))
66
+
67
+ prober.probe_all
68
+ prober.write_to(response)
69
+
70
+ response
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,31 @@
1
+ require "spec_helper"
2
+ require "gitlab_monitor/cli"
3
+
4
+ context "With valid pair of repositories" do
5
+ let(:repos) { GitRepoBuilder.new }
6
+
7
+ after do
8
+ repos.cleanup
9
+ end
10
+
11
+ describe GitLab::Monitor::CLI do
12
+ it "returns the rigth parser" do
13
+ expect(GitLab::Monitor::CLI.for("git")).to be(GitLab::Monitor::CLI::GIT)
14
+ end
15
+
16
+ it "returns a null parser if it is not found" do
17
+ expect(GitLab::Monitor::CLI.for("invalid")).to be(GitLab::Monitor::CLI::NullRunner)
18
+ end
19
+ end
20
+
21
+ describe GitLab::Monitor::CLI::GIT do
22
+ let(:output) { StringIO.new }
23
+ it "works end to end" do
24
+ args = CLIArgs.new([repos.cloned_repo, output])
25
+ ssh = GitLab::Monitor::CLI::GIT.new(args)
26
+ ssh.run
27
+ output.rewind
28
+ expect(output.read).to match(/git_push_time_milliseconds \d+ \d+/)
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,99 @@
1
+ require "spec_helper"
2
+ require "gitlab_monitor/database/bloat"
3
+
4
+ describe GitLab::Monitor::Database::BloatCollector do
5
+ let(:connection_pool) { double("connection pool") }
6
+ let(:connection) { double("connection") }
7
+
8
+ subject { described_class.new(connection_string: "").run(type) }
9
+ let(:type) { :btree }
10
+ let(:query) { "select something" }
11
+
12
+ before do
13
+ allow_any_instance_of(described_class).to receive(:connection_pool).and_return(connection_pool)
14
+ allow(connection_pool).to receive(:with).and_yield(connection)
15
+ end
16
+
17
+ it "converts query results into a hash" do
18
+ row1 = { "object_name" => "o", "more_stuff" => 1 }
19
+ row2 = { "object_name" => "a", "more_stuff" => 2 }
20
+
21
+ expect(File).to receive(:read).and_return(query)
22
+ expect(connection).to receive(:exec).with(query).and_return([row1, row2])
23
+
24
+ expect(subject).to eq("o" => row1, "a" => row2)
25
+ end
26
+ end
27
+
28
+ describe GitLab::Monitor::Database::BloatProber do
29
+ let(:opts) { { bloat_types: %i(btree table) } }
30
+ let(:metrics) { double("PrometheusMetrics", add: nil) }
31
+ let(:collector) { double("BloatCollector", run: data) }
32
+
33
+ let(:data) do
34
+ {
35
+ "object" => {
36
+ "object_name" => "object",
37
+ "bloat_ratio" => 1,
38
+ "bloat_size" => 2,
39
+ "extra_size" => 3,
40
+ "real_size" => 4
41
+ }
42
+ }
43
+ end
44
+
45
+ describe "#probe_db" do
46
+ subject { described_class.new(opts, metrics: metrics, collector: collector).probe_db }
47
+
48
+ it "invokes the collector for each bloat type" do
49
+ expect(collector).to receive(:run).with(:btree)
50
+ expect(collector).to receive(:run).with(:table)
51
+
52
+ subject
53
+ end
54
+
55
+ it "adds bloat_ratio metric" do
56
+ opts[:bloat_types].each do |type|
57
+ expect(metrics).to receive(:add).with("gitlab_database_bloat_#{type}_bloat_ratio", 1, query_name: "object")
58
+ end
59
+
60
+ subject
61
+ end
62
+
63
+ it "adds bloat_size metric" do
64
+ opts[:bloat_types].each do |type|
65
+ expect(metrics).to receive(:add).with("gitlab_database_bloat_#{type}_bloat_size", 2, query_name: "object")
66
+ end
67
+
68
+ subject
69
+ end
70
+
71
+ it "adds extra_size metric" do
72
+ opts[:bloat_types].each do |type|
73
+ expect(metrics).to receive(:add).with("gitlab_database_bloat_#{type}_extra_size", 3, query_name: "object")
74
+ end
75
+
76
+ subject
77
+ end
78
+
79
+ it "adds real_size metric" do
80
+ opts[:bloat_types].each do |type|
81
+ expect(metrics).to receive(:add).with("gitlab_database_bloat_#{type}_real_size", 4, query_name: "object")
82
+ end
83
+
84
+ subject
85
+ end
86
+ end
87
+
88
+ describe "#write_to" do
89
+ let(:target) { double }
90
+ let(:metrics) { double("PrometheusMetrics", to_s: double) }
91
+ subject { described_class.new(opts, metrics: metrics).write_to(target) }
92
+
93
+ it "writes to given target" do
94
+ expect(target).to receive(:write).with(metrics.to_s)
95
+
96
+ subject
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,421 @@
1
+ require "spec_helper"
2
+ require "gitlab_monitor/database/ci_builds"
3
+
4
+ # rubocop:disable Metrics/LineLength
5
+ describe GitLab::Monitor::Database do
6
+ let(:set_random_page_cost_query) { "SET random_page_cost" }
7
+ let(:builds_query_ee) { "SELECT BUILDS EE" }
8
+ let(:builds_query_ce) { "SELECT BUILDS CE" }
9
+ let(:stale_builds_query) { "SELECT NOT UPDATED RUNNING" }
10
+ let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
11
+ let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
12
+ let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
13
+ let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
14
+ let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
15
+ let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
16
+ let(:connection_pool) { double("connection pool") }
17
+ let(:connection) { double("connection") }
18
+ let(:allowed_repeated_commands_count) { 5 }
19
+ let(:created_builds_counting_disabled) { true }
20
+ let(:time_now) { Time.new(2019, 4, 9, 6, 30, 0) }
21
+ let(:unarchived_traces_query_time) { "2019-04-09 05:30:00" }
22
+ let(:unarchived_traces_offset_minutes) { 60 }
23
+
24
+ def stub_ee
25
+ allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
26
+ end
27
+
28
+ def stub_ce
29
+ allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "f" }])
30
+ end
31
+
32
+ def builds_query_row_ee(shared_runners_enabled, status, namespace_id, has_minutes, count)
33
+ row = builds_query_row_ce(shared_runners_enabled, status, namespace_id, count)
34
+ row["has_minutes"] = has_minutes
35
+ row
36
+ end
37
+
38
+ def builds_query_row_ce(shared_runners_enabled, status, namespace_id, count)
39
+ { "shared_runners_enabled" => shared_runners_enabled,
40
+ "status" => status,
41
+ "namespace_id" => namespace_id,
42
+ "count" => count }
43
+ end
44
+
45
+ # rubocop:disable Metrics/ParameterLists
46
+ def per_runner_query_row_ee(runner_id, is_shared, namespace_id, mirror, mirror_trigger_builds, pipeline_schedule_id, trigger_request_id, has_minutes, count)
47
+ row = per_runner_query_row_ce(runner_id, is_shared, namespace_id, pipeline_schedule_id, trigger_request_id, count)
48
+ row["mirror"] = mirror
49
+ row["mirror_trigger_builds"] = mirror_trigger_builds
50
+ row["has_minutes"] = has_minutes
51
+ row
52
+ end
53
+ # rubocop:enable Metrics/ParameterLists
54
+
55
+ # rubocop:disable Metrics/ParameterLists
56
+ def per_runner_query_row_ce(runner_id, is_shared, namespace_id, pipeline_schedule_id, trigger_request_id, count)
57
+ { "runner_id" => runner_id,
58
+ "is_shared" => is_shared,
59
+ "namespace_id" => namespace_id,
60
+ "pipeline_schedule_id" => pipeline_schedule_id,
61
+ "trigger_request_id" => trigger_request_id,
62
+ "count" => count }
63
+ end
64
+ # rubocop:enable Metrics/ParameterLists
65
+
66
+ # rubocop:disable Metrics/ParameterLists
67
+ def repeated_commands_query_row_ee(namespace_id, shared_runners_enabled, project_id, status, has_minutes, count)
68
+ row = repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
69
+ row["has_minutes"] = has_minutes
70
+ row
71
+ end
72
+ # rubocop:enable Metrics/ParameterLists
73
+
74
+ def repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
75
+ { "namespace_id" => namespace_id,
76
+ "shared_runners_enabled" => shared_runners_enabled,
77
+ "project_id" => project_id,
78
+ "status" => status,
79
+ "count" => count }
80
+ end
81
+
82
+ before do
83
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::SET_RANDOM_PAGE_COST", set_random_page_cost_query)
84
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::BUILDS_QUERY_EE", builds_query_ee)
85
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::BUILDS_QUERY_CE", builds_query_ce)
86
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::STALE_BUILDS_QUERY", stale_builds_query)
87
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
88
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
89
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
90
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
91
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
92
+ stub_const("GitLab::Monitor::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)
93
+
94
+ allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool)
95
+ allow(connection_pool).to receive(:with).and_yield(connection)
96
+
97
+ allow(connection).to receive(:transaction).and_yield(connection)
98
+ allow(connection).to receive(:exec).with(set_random_page_cost_query)
99
+
100
+ allow(Time).to receive(:now).and_return(time_now)
101
+
102
+ allow(connection).to receive(:exec).with(builds_query_ee)
103
+ .and_return([builds_query_row_ee("f", "created", "1", "f", 10),
104
+ builds_query_row_ee("t", "pending", "1", "t", 30),
105
+ builds_query_row_ee("f", "created", "2", "f", 20),
106
+ builds_query_row_ee("t", "pending", "2", "t", 50),
107
+ builds_query_row_ee("t", "pending", "3", "f", 1),
108
+ builds_query_row_ee("t", "pending", "4", "t", 2),
109
+ builds_query_row_ee("f", "pending", "5", "f", 2)])
110
+ allow(connection).to receive(:exec).with(builds_query_ce)
111
+ .and_return([builds_query_row_ce("f", "created", "1", 10),
112
+ builds_query_row_ce("t", "pending", "1", 30),
113
+ builds_query_row_ce("f", "created", "2", 20),
114
+ builds_query_row_ce("t", "pending", "2", 50),
115
+ builds_query_row_ce("t", "pending", "3", 1),
116
+ builds_query_row_ce("t", "pending", "4", 2),
117
+ builds_query_row_ce("f", "pending", "5", 2)])
118
+
119
+ allow(connection).to receive(:exec).with(stale_builds_query).and_return([{ "count" => 2 }])
120
+
121
+ allow(connection).to receive(:exec).with(per_runner_query_ee)
122
+ .and_return([per_runner_query_row_ee(1, "t", 1, "f", "f", 1, nil, "t", 15),
123
+ per_runner_query_row_ee(2, "f", 2, "t", "t", nil, 3, "f", 5),
124
+ per_runner_query_row_ee(2, "f", 3, "t", "t", nil, 3, "t", 5),
125
+ per_runner_query_row_ee(3, "f", 4, "t", "t", nil, 3, "f", 5)])
126
+
127
+ allow(connection).to receive(:exec).with(per_runner_query_ce)
128
+ .and_return([per_runner_query_row_ce(1, "t", 1, 1, nil, 15),
129
+ per_runner_query_row_ce(2, "f", 2, nil, 3, 5),
130
+ per_runner_query_row_ce(2, "f", 3, nil, 3, 5),
131
+ per_runner_query_row_ce(3, "f", 4, nil, 3, 5)])
132
+
133
+ # rubocop:disable Style/FormatString
134
+ repeated_commands_query_ee_with_limit = repeated_commands_query_ee % [allowed_repeated_commands_count]
135
+ repeated_commands_query_ce_with_limit = repeated_commands_query_ce % [allowed_repeated_commands_count]
136
+ # rubocop:enable Style/FormatString
137
+
138
+ allow(connection).to receive(:exec)
139
+ .with(repeated_commands_query_ee_with_limit)
140
+ .and_return([repeated_commands_query_row_ee(1, "t", 1, "pending", "t", 10),
141
+ repeated_commands_query_row_ee(2, "f", 2, "running", "f", 20),
142
+ repeated_commands_query_row_ee(1, "f", 3, "pending", "t", 30),
143
+ repeated_commands_query_row_ee(2, "t", 4, "running", "f", 40)])
144
+ allow(connection).to receive(:exec)
145
+ .with(repeated_commands_query_ce_with_limit)
146
+ .and_return([repeated_commands_query_row_ce(1, "t", 1, "pending", 10),
147
+ repeated_commands_query_row_ce(2, "f", 2, "running", 20),
148
+ repeated_commands_query_row_ce(1, "f", 3, "pending", 30),
149
+ repeated_commands_query_row_ce(2, "t", 4, "running", 40)])
150
+
151
+ unarchived_traces_query_with_time = unarchived_traces_query % [unarchived_traces_query_time] # rubocop:disable Style/FormatString
152
+
153
+ allow(connection).to receive(:exec).with(unarchived_traces_query_with_time).and_return([{ "count" => 10 }])
154
+ end
155
+
156
+ describe GitLab::Monitor::Database::CiBuildsCollector do
157
+ let(:collector) do
158
+ described_class.new(connection_string: "host=localhost",
159
+ allowed_repeated_commands_count: allowed_repeated_commands_count,
160
+ created_builds_counting_disabled: created_builds_counting_disabled,
161
+ unarchived_traces_offset_minutes: unarchived_traces_offset_minutes)
162
+ end
163
+ let(:expected_stale_builds) { 2 }
164
+ let(:expected_unarchived_traces) { 10 }
165
+
166
+ shared_examples "data collector" do
167
+ subject { collector.run }
168
+
169
+ it "returns raw per_runner data" do
170
+ expect(subject[:per_runner]).to include(*expected_per_runner)
171
+ end
172
+
173
+ it "returns raw pending_builds data" do
174
+ expect(subject[:pending_builds]).to include(*expected_pending_builds)
175
+ end
176
+
177
+ context "when created_builds_counting_disabled is set to false" do
178
+ let(:created_builds_counting_disabled) { false }
179
+
180
+ it "returns raw created_builds data" do
181
+ expect(subject).to have_key(:created_builds)
182
+ expect(subject[:created_builds]).to include(*expected_created_builds)
183
+ end
184
+ end
185
+
186
+ context "when created_builds_counting_disabled is set to true" do
187
+ let(:created_builds_counting_disabled) { true }
188
+
189
+ it "doesn't return raw created_builds data" do
190
+ expect(subject).not_to have_key(:created_builds)
191
+ end
192
+ end
193
+
194
+ it "returns raw stale_builds data" do
195
+ expect(subject[:stale_builds]).to eq(expected_stale_builds)
196
+ end
197
+
198
+ it "returns raw repeated_commands data" do
199
+ expect(subject[:repeated_commands]).to include(*expected_repeated_commands)
200
+ end
201
+
202
+ it "returns raw unarchived_traces data" do
203
+ expect(subject[:unarchived_traces]).to eq(expected_unarchived_traces)
204
+ end
205
+ end
206
+
207
+ context "when executed on EE" do
208
+ let(:expected_pending_builds) do
209
+ [{ namespace: "1", shared_runners: "yes", has_minutes: "yes", value: 30 },
210
+ { namespace: "2", shared_runners: "yes", has_minutes: "yes", value: 50 },
211
+ { namespace: "3", shared_runners: "yes", has_minutes: "no", value: 1 },
212
+ { namespace: "4", shared_runners: "yes", has_minutes: "yes", value: 2 },
213
+ { namespace: "5", shared_runners: "no", has_minutes: "no", value: 2 }]
214
+ end
215
+ let(:expected_created_builds) do
216
+ [{ namespace: "1", shared_runners: "no", has_minutes: "no", value: 10 },
217
+ { namespace: "2", shared_runners: "no", has_minutes: "no", value: 20 }]
218
+ end
219
+ let(:expected_per_runner) do
220
+ [{ runner: "1", shared_runner: "yes", namespace: "1", mirror: "no", mirror_trigger_builds: "no", scheduled: "yes", triggered: "no", has_minutes: "yes", value: 15 },
221
+ { runner: "2", shared_runner: "no", namespace: "2", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "no", value: 5 },
222
+ { runner: "2", shared_runner: "no", namespace: "3", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "yes", value: 5 },
223
+ { runner: "3", shared_runner: "no", namespace: "4", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "no", value: 5 }]
224
+ end
225
+ let(:expected_repeated_commands) do
226
+ [{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", has_minutes: "yes", value: 10 },
227
+ { namespace: "2", project: "2", shared_runners: "no", status: "running", has_minutes: "no", value: 20 },
228
+ { namespace: "1", project: "3", shared_runners: "no", status: "pending", has_minutes: "yes", value: 30 },
229
+ { namespace: "2", project: "4", shared_runners: "yes", status: "running", has_minutes: "no", value: 40 }]
230
+ end
231
+
232
+ before do
233
+ stub_ee
234
+ end
235
+
236
+ it_behaves_like "data collector"
237
+ end
238
+
239
+ context "when executed on CE" do
240
+ let(:expected_pending_builds) do
241
+ [{ namespace: "1", shared_runners: "yes", value: 30 },
242
+ { namespace: "2", shared_runners: "yes", value: 50 },
243
+ { namespace: "3", shared_runners: "yes", value: 1 },
244
+ { namespace: "4", shared_runners: "yes", value: 2 },
245
+ { namespace: "5", shared_runners: "no", value: 2 }]
246
+ end
247
+ let(:expected_created_builds) do
248
+ [{ namespace: "1", shared_runners: "no", value: 10 },
249
+ { namespace: "2", shared_runners: "no", value: 20 }]
250
+ end
251
+ let(:expected_per_runner) do
252
+ [{ runner: "1", shared_runner: "yes", namespace: "1", scheduled: "yes", triggered: "no", value: 15 },
253
+ { runner: "2", shared_runner: "no", namespace: "2", scheduled: "no", triggered: "yes", value: 5 },
254
+ { runner: "2", shared_runner: "no", namespace: "3", scheduled: "no", triggered: "yes", value: 5 },
255
+ { runner: "3", shared_runner: "no", namespace: "4", scheduled: "no", triggered: "yes", value: 5 }]
256
+ end
257
+ let(:expected_repeated_commands) do
258
+ [{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", value: 10 },
259
+ { namespace: "2", project: "2", shared_runners: "no", status: "running", value: 20 },
260
+ { namespace: "1", project: "3", shared_runners: "no", status: "pending", value: 30 },
261
+ { namespace: "2", project: "4", shared_runners: "yes", status: "running", value: 40 }]
262
+ end
263
+
264
+ before do
265
+ stub_ce
266
+ end
267
+
268
+ it_behaves_like "data collector"
269
+ end
270
+ end
271
+
272
+ describe GitLab::Monitor::Database::CiBuildsProber do
273
+ let(:writer) { StringIO.new }
274
+ let(:prober) do
275
+ opts = { connection_string: "host=localhost",
276
+ allowed_repeated_commands_count: allowed_repeated_commands_count,
277
+ created_builds_counting_disabled: created_builds_counting_disabled,
278
+ unarchived_traces_offset_minutes: unarchived_traces_offset_minutes }
279
+ described_class.new(opts,
280
+ metrics: GitLab::Monitor::PrometheusMetrics.new(include_timestamp: false))
281
+ end
282
+
283
+ before do
284
+ allow_any_instance_of(GitLab::Monitor::Database::CiBuildsCollector).to receive(:connected?).and_return(true)
285
+ end
286
+
287
+ shared_examples "metrics server" do
288
+ subject do
289
+ prober.probe_db
290
+ prober.write_to(writer)
291
+ writer.string
292
+ end
293
+
294
+ context "when PG exceptions aren't raised" do
295
+ context "when created_builds_counting_disabled is set to false" do
296
+ let(:created_builds_counting_disabled) { false }
297
+
298
+ it "responds with created builds Prometheus metrics" do
299
+ ci_created_builds_expected_lines.each do |expected_line|
300
+ expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
301
+ end
302
+ end
303
+ end
304
+
305
+ context "when created_builds_counting_disabled is set to true" do
306
+ let(:created_builds_counting_disabled) { true }
307
+
308
+ it "doesn't respond with created builds Prometheus metrics" do
309
+ ci_created_builds_expected_lines.each do |expected_line|
310
+ expect(subject).not_to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
311
+ end
312
+ end
313
+ end
314
+
315
+ it "responds with pending builds Prometheus metrics" do
316
+ ci_pending_builds_expected_lines.each do |expected_line|
317
+ expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
318
+ end
319
+ end
320
+
321
+ it "responds with running builds Prometheus metrics" do
322
+ ci_running_builds_expected_lines.each do |expected_line|
323
+ expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
324
+ end
325
+ end
326
+
327
+ it "responds with repeated commands Prometheus metrics" do
328
+ ci_repeated_commands_builds_lines.each do |expected_line|
329
+ expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
330
+ end
331
+ end
332
+
333
+ it "responds with stale builds Prometheus metrics" do
334
+ expect(subject).to match(/^ci_stale_builds 2$/m)
335
+ end
336
+
337
+ it "responds with unarchived traces Prometheus metrics" do
338
+ expect(subject).to match(/^ci_unarchived_traces 10$/m)
339
+ end
340
+ end
341
+
342
+ context "when PG exceptions are raised" do
343
+ before do
344
+ allow(connection).to receive(:exec).and_raise(PG::UndefinedColumn)
345
+ end
346
+
347
+ it "responds with Prometheus metrics" do
348
+ prober.probe_db
349
+ prober.write_to(writer)
350
+ output = writer.string
351
+
352
+ expect(output).to match(/^ci_stale_builds 0$/m)
353
+ end
354
+ end
355
+ end
356
+
357
+ context "when executed on EE" do
358
+ let(:ci_created_builds_expected_lines) do
359
+ ['ci_created_builds\{has_minutes="no",namespace="1",shared_runners="no"\} 10',
360
+ 'ci_created_builds\{has_minutes="no",namespace="2",shared_runners="no"\} 20']
361
+ end
362
+ let(:ci_pending_builds_expected_lines) do
363
+ ['ci_pending_builds\{has_minutes="yes",namespace="1",shared_runners="yes"\} 30',
364
+ 'ci_pending_builds\{has_minutes="yes",namespace="2",shared_runners="yes"\} 50',
365
+ 'ci_pending_builds\{has_minutes="no",namespace="",shared_runners="yes"\} 1',
366
+ 'ci_pending_builds\{has_minutes="yes",namespace="",shared_runners="yes"\} 2',
367
+ 'ci_pending_builds\{has_minutes="no",namespace="",shared_runners="no"\} 2']
368
+ end
369
+ let(:ci_running_builds_expected_lines) do
370
+ ['ci_running_builds\{has_minutes="yes",mirror="no",mirror_trigger_builds="no",namespace="1",runner="1",scheduled="yes",shared_runner="yes",triggered="no"\} 15',
371
+ 'ci_running_builds\{has_minutes="no",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="2",scheduled="no",shared_runner="no",triggered="yes"\} 5',
372
+ 'ci_running_builds\{has_minutes="yes",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="2",scheduled="no",shared_runner="no",triggered="yes"\} 5',
373
+ 'ci_running_builds\{has_minutes="no",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="3",scheduled="no",shared_runner="no",triggered="yes"\} 5']
374
+ end
375
+ let(:ci_repeated_commands_builds_lines) do
376
+ ['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending",has_minutes="yes"\} 10',
377
+ 'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running",has_minutes="no"\} 20',
378
+ 'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending",has_minutes="yes"\} 30',
379
+ 'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running",has_minutes="no"\} 40']
380
+ end
381
+ let(:namespace_out_of_limit) { 2 }
382
+
383
+ before do
384
+ stub_ee
385
+ end
386
+
387
+ it_behaves_like "metrics server"
388
+ end
389
+
390
+ context "when executed on CE" do
391
+ let(:ci_created_builds_expected_lines) do
392
+ ['ci_created_builds\{namespace="1",shared_runners="no"\} 10',
393
+ 'ci_created_builds\{namespace="2",shared_runners="no"\} 20']
394
+ end
395
+ let(:ci_pending_builds_expected_lines) do
396
+ ['ci_pending_builds\{namespace="1",shared_runners="yes"\} 30',
397
+ 'ci_pending_builds\{namespace="2",shared_runners="yes"\} 50',
398
+ 'ci_pending_builds\{namespace="",shared_runners="yes"\} 3',
399
+ 'ci_pending_builds\{namespace="",shared_runners="no"\} 2']
400
+ end
401
+ let(:ci_running_builds_expected_lines) do
402
+ ['ci_running_builds\{namespace="1",runner="1",scheduled="yes",shared_runner="yes",triggered="no"\} 15',
403
+ 'ci_running_builds\{namespace="",runner="2",scheduled="no",shared_runner="no",triggered="yes"\} 10',
404
+ 'ci_running_builds\{namespace="",runner="3",scheduled="no",shared_runner="no",triggered="yes"\} 5']
405
+ end
406
+ let(:ci_repeated_commands_builds_lines) do
407
+ ['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending"\} 10',
408
+ 'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running"\} 20',
409
+ 'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending"\} 30',
410
+ 'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running"\} 40']
411
+ end
412
+ let(:namespace_out_of_limit) { 0 }
413
+
414
+ before do
415
+ stub_ce
416
+ end
417
+
418
+ it_behaves_like "metrics server"
419
+ end
420
+ end
421
+ end