gitlab-exporter 7.2.0 → 10.1.0

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "7.2.0".freeze
+    VERSION = "10.1.0".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -24,7 +24,7 @@ module GitLab
         private
 
         def memory_usage
-          io = IO.popen(%W(ps -o rss= -p #{$PID}))
+          io = IO.popen(%W[ps -o rss= -p #{$PID}])
 
           mem = io.read
           io.close
@@ -35,7 +35,24 @@ module GitLab
         end
       end
 
+      # Performs a major GC after each request. We found that this helps to free up
+      # several MB of memory in conjunction with sricter malloc config.
+      # See https://gitlab.com/gitlab-org/gitlab/-/issues/297241
+      class RunGC
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          @app.call(env).tap do
+            GC.start
+          end
+        end
+      end
+
       class << self
+        DEFAULT_WEB_SERVER = "webrick".freeze
+
         def setup(config)
           setup_server(config[:server])
           setup_probes(config[:probes])
@@ -43,6 +60,10 @@ module GitLab
           memory_threshold = (config[:server] && config[:server][:memory_threshold]) || 1024
           use MemoryKillerMiddleware, memory_threshold
           use Rack::Logger
+          use RunGC
+
+          # Defrag heap after everything is loaded into memory.
+          GC.compact
         end
 
         def logger
@@ -52,6 +73,7 @@ module GitLab
         def setup_server(config)
           config ||= {}
 
+          set(:server, config.fetch(:name, DEFAULT_WEB_SERVER))
           set(:bind, config.fetch(:listen_address, "0.0.0.0"))
           set(:port, config.fetch(:listen_port, 9168))
         end

data/spec/database/bloat_spec.rb

@@ -26,7 +26,7 @@ describe GitLab::Exporter::Database::BloatCollector do
 end
 
 describe GitLab::Exporter::Database::BloatProber do
-  let(:opts) { { bloat_types: %i(btree table) } }
+  let(:opts) { { bloat_types: %i[btree table] } }
   let(:metrics) { double("PrometheusMetrics", add: nil) }
   let(:collector) { double("BloatCollector", run: data) }
 

data/spec/database/ci_builds_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 require "gitlab_exporter/database/ci_builds"
 
-# rubocop:disable Metrics/LineLength
+# rubocop:disable Layout/LineLength
 describe GitLab::Exporter::Database do
   let(:set_random_page_cost_query) { "SET random_page_cost" }
   let(:builds_query_ee) { "SELECT BUILDS EE" }
@@ -10,12 +10,9 @@ describe GitLab::Exporter::Database do
   let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
   let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
   let(:ee_check_query) { "SELECT COUNT(*) FROM licenses" }
-  let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
-  let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
   let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
   let(:connection_pool) { double("connection pool") }
   let(:connection) { double("connection") }
-  let(:allowed_repeated_commands_count) { 5 }
   let(:created_builds_counting_disabled) { true }
   let(:time_now) { Time.new(2019, 4, 9, 6, 30, 0) }
   let(:unarchived_traces_query_time) { "2019-04-09 05:30:00" }
@@ -63,22 +60,6 @@ describe GitLab::Exporter::Database do
   end
   # rubocop:enable Metrics/ParameterLists
 
-  # rubocop:disable Metrics/ParameterLists
-  def repeated_commands_query_row_ee(namespace_id, shared_runners_enabled, project_id, status, has_minutes, count)
-    row = repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
-    row["has_minutes"] = has_minutes
-    row
-  end
-  # rubocop:enable Metrics/ParameterLists
-
-  def repeated_commands_query_row_ce(namespace_id, shared_runners_enabled, project_id, status, count)
-    { "namespace_id" => namespace_id,
-      "shared_runners_enabled" => shared_runners_enabled,
-      "project_id" => project_id,
-      "status" => status,
-      "count" => count }
-  end
-
   before do
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::SET_RANDOM_PAGE_COST", set_random_page_cost_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::BUILDS_QUERY_EE", builds_query_ee)
@@ -87,8 +68,6 @@ describe GitLab::Exporter::Database do
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::EE_CHECK_QUERY", ee_check_query)
-    stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
-    stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)
 
     allow_any_instance_of(GitLab::Exporter::Database::CiBuildsCollector).to receive(:connection_pool).and_return(connection_pool)
@@ -100,53 +79,35 @@ describe GitLab::Exporter::Database do
     allow(Time).to receive(:now).and_return(time_now)
 
     allow(connection).to receive(:exec).with(builds_query_ee)
-      .and_return([builds_query_row_ee("f", "created", "1", "f", 10),
-                   builds_query_row_ee("t", "pending", "1", "t", 30),
-                   builds_query_row_ee("f", "created", "2", "f", 20),
-                   builds_query_row_ee("t", "pending", "2", "t", 50),
-                   builds_query_row_ee("t", "pending", "3", "f", 1),
-                   builds_query_row_ee("t", "pending", "4", "t", 2),
-                   builds_query_row_ee("f", "pending", "5", "f", 2)])
+                                       .and_return([builds_query_row_ee("f", "created", "1", "f", 10),
+                                                    builds_query_row_ee("t", "pending", "1", "t", 30),
+                                                    builds_query_row_ee("f", "created", "2", "f", 20),
+                                                    builds_query_row_ee("t", "pending", "2", "t", 50),
+                                                    builds_query_row_ee("t", "pending", "3", "f", 1),
+                                                    builds_query_row_ee("t", "pending", "4", "t", 2),
+                                                    builds_query_row_ee("f", "pending", "5", "f", 2)])
     allow(connection).to receive(:exec).with(builds_query_ce)
-      .and_return([builds_query_row_ce("f", "created", "1", 10),
-                   builds_query_row_ce("t", "pending", "1", 30),
-                   builds_query_row_ce("f", "created", "2", 20),
-                   builds_query_row_ce("t", "pending", "2", 50),
-                   builds_query_row_ce("t", "pending", "3", 1),
-                   builds_query_row_ce("t", "pending", "4", 2),
-                   builds_query_row_ce("f", "pending", "5", 2)])
+                                       .and_return([builds_query_row_ce("f", "created", "1", 10),
+                                                    builds_query_row_ce("t", "pending", "1", 30),
+                                                    builds_query_row_ce("f", "created", "2", 20),
+                                                    builds_query_row_ce("t", "pending", "2", 50),
+                                                    builds_query_row_ce("t", "pending", "3", 1),
+                                                    builds_query_row_ce("t", "pending", "4", 2),
+                                                    builds_query_row_ce("f", "pending", "5", 2)])
 
     allow(connection).to receive(:exec).with(stale_builds_query).and_return([{ "count" => 2 }])
 
     allow(connection).to receive(:exec).with(per_runner_query_ee)
-      .and_return([per_runner_query_row_ee(1, "instance_type", 1, "f", "f", 1, nil, "t", 15),
-                   per_runner_query_row_ee(2, "project_type", 2, "t", "t", nil, 3, "f", 5),
-                   per_runner_query_row_ee(2, "project_type", 3, "t", "t", nil, 3, "t", 5),
-                   per_runner_query_row_ee(3, "project_type", 4, "t", "t", nil, 3, "f", 5)])
+                                       .and_return([per_runner_query_row_ee(1, "instance_type", 1, "f", "f", 1, nil, "t", 15),
+                                                    per_runner_query_row_ee(2, "project_type", 2, "t", "t", nil, 3, "f", 5),
+                                                    per_runner_query_row_ee(2, "project_type", 3, "t", "t", nil, 3, "t", 5),
+                                                    per_runner_query_row_ee(3, "project_type", 4, "t", "t", nil, 3, "f", 5)])
 
     allow(connection).to receive(:exec).with(per_runner_query_ce)
-      .and_return([per_runner_query_row_ce(1, "instance_type", 1, 1, nil, 15),
-                   per_runner_query_row_ce(2, "project_type", 2, nil, 3, 5),
-                   per_runner_query_row_ce(2, "project_type", 3, nil, 3, 5),
-                   per_runner_query_row_ce(3, "project_type", 4, nil, 3, 5)])
-
-    # rubocop:disable Style/FormatString
-    repeated_commands_query_ee_with_limit = repeated_commands_query_ee % [allowed_repeated_commands_count]
-    repeated_commands_query_ce_with_limit = repeated_commands_query_ce % [allowed_repeated_commands_count]
-    # rubocop:enable Style/FormatString
-
-    allow(connection).to receive(:exec)
-      .with(repeated_commands_query_ee_with_limit)
-      .and_return([repeated_commands_query_row_ee(1, "t", 1, "pending", "t", 10),
-                   repeated_commands_query_row_ee(2, "f", 2, "running", "f", 20),
-                   repeated_commands_query_row_ee(1, "f", 3, "pending", "t", 30),
-                   repeated_commands_query_row_ee(2, "t", 4, "running", "f", 40)])
-    allow(connection).to receive(:exec)
-      .with(repeated_commands_query_ce_with_limit)
-      .and_return([repeated_commands_query_row_ce(1, "t", 1, "pending", 10),
-                   repeated_commands_query_row_ce(2, "f", 2, "running", 20),
-                   repeated_commands_query_row_ce(1, "f", 3, "pending", 30),
-                   repeated_commands_query_row_ce(2, "t", 4, "running", 40)])
+                                       .and_return([per_runner_query_row_ce(1, "instance_type", 1, 1, nil, 15),
+                                                    per_runner_query_row_ce(2, "project_type", 2, nil, 3, 5),
+                                                    per_runner_query_row_ce(2, "project_type", 3, nil, 3, 5),
+                                                    per_runner_query_row_ce(3, "project_type", 4, nil, 3, 5)])
 
     unarchived_traces_query_with_time = unarchived_traces_query % [unarchived_traces_query_time] # rubocop:disable Style/FormatString
 
@@ -156,7 +117,6 @@ describe GitLab::Exporter::Database do
   describe GitLab::Exporter::Database::CiBuildsCollector do
     let(:collector) do
       described_class.new(connection_string: "host=localhost",
-                          allowed_repeated_commands_count: allowed_repeated_commands_count,
                           created_builds_counting_disabled: created_builds_counting_disabled,
                           unarchived_traces_offset_minutes: unarchived_traces_offset_minutes)
     end
@@ -195,10 +155,6 @@ describe GitLab::Exporter::Database do
         expect(subject[:stale_builds]).to eq(expected_stale_builds)
       end
 
-      it "returns raw repeated_commands data" do
-        expect(subject[:repeated_commands]).to include(*expected_repeated_commands)
-      end
-
       it "returns raw unarchived_traces data" do
         expect(subject[:unarchived_traces]).to eq(expected_unarchived_traces)
       end
@@ -222,12 +178,6 @@ describe GitLab::Exporter::Database do
          { runner: "2", runner_type: "project_type", namespace: "3", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "yes", value: 5.0 },
          { runner: "3", runner_type: "project_type", namespace: "4", mirror: "yes", mirror_trigger_builds: "yes", scheduled: "no", triggered: "yes", has_minutes: "no", value: 5.0 }]
       end
-      let(:expected_repeated_commands) do
-        [{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", has_minutes: "yes", value: 10.0 },
-         { namespace: "2", project: "2", shared_runners: "no", status: "running", has_minutes: "no", value: 20.0 },
-         { namespace: "1", project: "3", shared_runners: "no", status: "pending", has_minutes: "yes", value: 30.0 },
-         { namespace: "2", project: "4", shared_runners: "yes", status: "running", has_minutes: "no", value: 40.0 }]
-      end
 
       before do
         stub_ee
@@ -254,12 +204,6 @@ describe GitLab::Exporter::Database do
          { runner: "2", runner_type: "project_type", namespace: "3", scheduled: "no", triggered: "yes", value: 5 },
          { runner: "3", runner_type: "project_type", namespace: "4", scheduled: "no", triggered: "yes", value: 5 }]
       end
-      let(:expected_repeated_commands) do
-        [{ namespace: "1", project: "1", shared_runners: "yes", status: "pending", value: 10 },
-         { namespace: "2", project: "2", shared_runners: "no", status: "running", value: 20 },
-         { namespace: "1", project: "3", shared_runners: "no", status: "pending", value: 30 },
-         { namespace: "2", project: "4", shared_runners: "yes", status: "running", value: 40 }]
-      end
 
       before do
         stub_ce
@@ -273,7 +217,6 @@ describe GitLab::Exporter::Database do
     let(:writer) { StringIO.new }
     let(:prober) do
       opts = { connection_string: "host=localhost",
-               allowed_repeated_commands_count: allowed_repeated_commands_count,
                created_builds_counting_disabled: created_builds_counting_disabled,
                unarchived_traces_offset_minutes: unarchived_traces_offset_minutes }
       described_class.new(opts,
@@ -324,12 +267,6 @@ describe GitLab::Exporter::Database do
           end
         end
 
-        it "responds with repeated commands Prometheus metrics" do
-          ci_repeated_commands_builds_lines.each do |expected_line|
-            expect(subject).to match(Regexp.new("^#{expected_line}$", Regexp::MULTILINE))
-          end
-        end
-
         it "responds with stale builds Prometheus metrics" do
           expect(subject).to match(/^ci_stale_builds 2.0$/m)
         end
@@ -372,12 +309,6 @@ describe GitLab::Exporter::Database do
          'ci_running_builds\{has_minutes="yes",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="2",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0',
          'ci_running_builds\{has_minutes="no",mirror="yes",mirror_trigger_builds="yes",namespace="",runner="3",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0']
       end
-      let(:ci_repeated_commands_builds_lines) do
-        ['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending",has_minutes="yes"\} 10.0',
-         'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running",has_minutes="no"\} 20.0',
-         'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending",has_minutes="yes"\} 30.0',
-         'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running",has_minutes="no"\} 40.0']
-      end
       let(:namespace_out_of_limit) { 2 }
 
       before do
@@ -403,12 +334,6 @@ describe GitLab::Exporter::Database do
          'ci_running_builds\{namespace="",runner="2",runner_type="project_type",scheduled="no",triggered="yes"\} 10.0',
          'ci_running_builds\{namespace="",runner="3",runner_type="project_type",scheduled="no",triggered="yes"\} 5.0']
       end
-      let(:ci_repeated_commands_builds_lines) do
-        ['ci_repeated_commands_builds\{namespace="1",project="1",shared_runners="yes",status="pending"\} 10.0',
-         'ci_repeated_commands_builds\{namespace="2",project="2",shared_runners="no",status="running"\} 20.0',
-         'ci_repeated_commands_builds\{namespace="1",project="3",shared_runners="no",status="pending"\} 30.0',
-         'ci_repeated_commands_builds\{namespace="2",project="4",shared_runners="yes",status="running"\} 40.0']
-      end
       let(:namespace_out_of_limit) { 0 }
 
       before do

data/spec/git_process_proper_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 require "gitlab_exporter"
 
-# rubocop:disable Metrics/LineLength
+# rubocop:disable Layout/LineLength
 describe GitLab::Exporter::GitProcessProber do
   describe ".extract_subcommand" do
     it "extract git subcommand" do

data/spec/memstats_spec.rb

@@ -14,11 +14,11 @@ describe GitLab::Exporter::MemStats do
   it "parses the data properly" do
     expect(subject.valid?).to be_truthy
 
-    nonzero_fields = %w(size rss shared_clean shared_dirty private_dirty pss)
-    zero_fields = %w(private_clean swap)
+    nonzero_fields = %w[size rss shared_clean shared_dirty private_dirty pss]
+    zero_fields = %w[private_clean swap]
 
     nonzero_fields.each do |field|
-      expect(subject.totals[field]).to be > 0 # rubocop:disable Style/NumericPredicate
+      expect(subject.totals[field]).to be > 0
     end
 
     zero_fields.each do |field|

data/spec/ruby_spec.rb

@@ -0,0 +1,25 @@
+require "spec_helper"
+require "gitlab_exporter/ruby"
+require "gitlab_exporter/prober"
+
+describe GitLab::Exporter::RubyProber do
+  let(:prober) { GitLab::Exporter::Prober.new(options) }
+
+  let(:options) do
+    {
+      ruby: {
+        class_name: described_class.to_s,
+        methods: %w[probe_gc],
+        opts: { quantiles: false }
+      }
+    }
+  end
+
+  it "probes and returns GC stats" do
+    prober.probe_all
+
+    output = StringIO.new
+    prober.write_to(output)
+    expect(output.string).to match(/ruby_gc_stat_count \d+ \d+/)
+  end
+end

data/spec/spec_helper.rb

@@ -51,11 +51,9 @@ class CLIArgs
     yield self
   end
 
-  def on(*args)
-  end
+  def on(*args); end
 
-  def banner=(banner)
-  end
+  def banner=(banner); end
 
   def parse!
     @arguments

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 7.2.0
+  version: 10.1.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -11,61 +11,61 @@ cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: connection_pool
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 2.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 2.2.1
 - !ruby/object:Gem::Dependency
-  name: sinatra
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.4
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.4
+        version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: quantile
+  name: puma
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 5.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 5.1.1
 - !ruby/object:Gem::Dependency
-  name: sidekiq
+  name: quantile
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.1
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.2.1
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -95,19 +95,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.6.0
 - !ruby/object:Gem::Dependency
-  name: connection_pool
+  name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: 5.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: 5.2.1
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.4
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -146,6 +160,8 @@ files:
 - ".gitignore"
 - ".gitlab-ci.yml"
 - ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".ruby-version"
 - CONTRIBUTING.md
 - Gemfile
 - Gemfile.lock
@@ -171,6 +187,7 @@ files:
 - lib/gitlab_exporter/prober.rb
 - lib/gitlab_exporter/process.rb
 - lib/gitlab_exporter/prometheus.rb
+- lib/gitlab_exporter/ruby.rb
 - lib/gitlab_exporter/sidekiq.rb
 - lib/gitlab_exporter/sidekiq_queue_job_stats.lua
 - lib/gitlab_exporter/util.rb
@@ -185,6 +202,7 @@ files:
 - spec/git_spec.rb
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
+- spec/ruby_spec.rb
 - spec/spec_helper.rb
 - spec/util_spec.rb
 homepage: http://gitlab.com
@@ -206,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: GitLab metrics exporter
@@ -220,5 +238,6 @@ test_files:
 - spec/git_spec.rb
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
+- spec/ruby_spec.rb
 - spec/spec_helper.rb
 - spec/util_spec.rb