gitlab-exporter 8.0.0 → 10.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37cc46db50f1361ccd9451418ad48edefaaa696d58af41c8a09f96a61aea93f9
-  data.tar.gz: 853e76cd0bee22d18e941c67588cbc06b3761098a604f382d87bcc8acbdb4555
+  metadata.gz: b54a7fd5cab81e9ffb2fe44bdff47d707c2a1cb2d31aa606246fd66ed3b15adf
+  data.tar.gz: f0d096e1a97a28f81ff360bad1925aa9abb764724387ec92f843a5184fead7b0
 SHA512:
-  metadata.gz: 11bec1adc02cbdac35bfe5bf699e12ae3008cf7ce54681ab73f0f176030a6d087d11ab08c5bb8ace5fee60efb48e66061e53fe904537274dc7ec76fbb90b2b76
-  data.tar.gz: eea37ccb83c041df049402d9a9aa471bc46f1892251bff3aa1149f1124db2847be7a874f3b0bba32df2bf3c6e8dd2e55e7dcddf66c782ef20e010fba18855407
+  metadata.gz: ab3fe334be8aeab22baf7ddddd2d4d6e650d6a8d30e290c98ef440205bf522c80338f311796bd1a2003a780a55ae8d5e9bd1fd71a51c3c61ed394a297fae700a
+  data.tar.gz: 874907c793cb4bf1947cc40a4fbf0bbfb70eb95a5769971688f64285346b01bf87ec45bd367cb44aec0c58fb3d1bcd369500d46e0f498a8a1fc0c73757d9c08f

data/.gitlab-ci.yml

@@ -7,7 +7,7 @@ include:
   - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
 
 default:
-  image: "ruby:2.3"
+  image: ruby:2.7
   cache:
     paths:
       - vendor

data/.rubocop.yml

@@ -1,5 +1,7 @@
+inherit_from: .rubocop_todo.yml
+
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.7
   DisplayCopNames: true
 
 # Just use double quotes please
@@ -20,15 +22,15 @@ Style/RaiseArgs:
 Style/FrozenStringLiteralComment:
   Enabled: false
 
+# Commonly used screens these days easily fit more than 80 characters.
+Layout/LineLength:
+  Max: 120
+
 Metrics/MethodLength:
   Max: 15
 
 Metrics/AbcSize:
   Enabled: false
 
-# Commonly used screens these days easily fit more than 80 characters.
-Metrics/LineLength:
-  Max: 120
-
 Metrics/ClassLength:
   Max: 150

data/.rubocop_todo.yml

@@ -0,0 +1,64 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2021-01-11 12:24:03 UTC using RuboCop version 0.93.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+# Configuration parameters: Include.
+# Include: **/*.gemspec
+Gemspec/RequiredRubyVersion:
+  Exclude:
+    - 'gitlab-exporter.gemspec'
+
+# Offense count: 3
+Lint/AmbiguousBlockAssociation:
+  Exclude:
+    - 'spec/git_spec.rb'
+    - 'spec/util_spec.rb'
+
+# Offense count: 2
+# Configuration parameters: MaximumRangeSize.
+Lint/MissingCopEnableDirective:
+  Exclude:
+    - 'spec/database/ci_builds_spec.rb'
+    - 'spec/git_process_proper_spec.rb'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Lint/NonDeterministicRequireOrder:
+  Exclude:
+    - 'spec/spec_helper.rb'
+
+# Offense count: 16
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods.
+# ExcludedMethods: refine
+Metrics/BlockLength:
+  Max: 349
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/CyclomaticComplexity:
+  Max: 8
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/PerceivedComplexity:
+  Max: 9
+
+# Offense count: 1
+# Configuration parameters: EnforcedStyle, AllowModifiersOnSymbols.
+# SupportedStyles: inline, group
+Style/AccessModifierDeclarations:
+  Exclude:
+    - 'lib/gitlab_exporter/util.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+# Configuration parameters: ConvertCodeThatCanStartToReturnNil, AllowedMethods.
+# AllowedMethods: present?, blank?, presence, try, try!
+Style/SafeNavigation:
+  Exclude:
+    - 'lib/gitlab_exporter/database/base.rb'

data/.ruby-version

@@ -0,0 +1 @@
+2.7.2

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (8.0.0)
+    gitlab-exporter (10.2.0)
       connection_pool (~> 2.2.1)
       pg (~> 1.1)
+      puma (~> 5.1.1)
       quantile (~> 0.2.0)
       redis (~> 4.1.2)
       redis-namespace (~> 1.6.0)
@@ -13,23 +14,28 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.0)
-    connection_pool (2.2.3)
+    ast (2.4.1)
+    connection_pool (2.2.5)
     diff-lcs (1.3)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    parser (2.5.1.0)
-      ast (~> 2.4.0)
+    nio4r (2.5.7)
+    parallel (1.20.1)
+    parser (3.0.0.0)
+      ast (~> 2.4.1)
     pg (1.2.3)
-    powerpack (0.1.1)
+    puma (5.1.1)
+      nio4r (~> 2.0)
     quantile (0.2.1)
     rack (2.2.3)
     rack-protection (2.0.8.1)
       rack
-    rainbow (2.1.0)
+    rainbow (3.0.0)
     redis (4.1.4)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
+    regexp_parser (2.0.3)
+    rexml (3.2.4)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
       rspec-expectations (~> 3.7.0)
@@ -43,14 +49,19 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    rubocop (0.42.0)
-      parser (>= 2.3.1.1, < 3.0)
-      powerpack (~> 0.1)
-      rainbow (>= 1.99.1, < 3.0)
+    rubocop (0.93.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.1.5)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
+      rexml
+      rubocop-ast (>= 0.6.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.8.1)
-    ruby2_keywords (0.0.2)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (1.4.0)
+      parser (>= 2.7.1.5)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.4)
     sidekiq (5.2.9)
       connection_pool (~> 2.2, >= 2.2.2)
       rack (~> 2.0)
@@ -62,7 +73,7 @@ GEM
       rack-protection (= 2.0.8.1)
       tilt (~> 2.0)
     tilt (2.0.10)
-    unicode-display_width (1.6.0)
+    unicode-display_width (1.7.0)
 
 PLATFORMS
   ruby
@@ -74,4 +85,4 @@ DEPENDENCIES
   rubocop (~> 0.42)
 
 BUNDLED WITH
-   1.17.3
+   2.1.4

data/README.md

@@ -33,23 +33,6 @@ metrics.
     * [git pull/push timings](lib/gitlab_exporter/git.rb) --
       `git_pull_time_milliseconds`, `git_push_time_milliseconds`
     * git processes stats (see Process below)
-1. [Process](lib/gitlab_exporter/process.rb)
-    * CPU time -- `process_cpu_seconds_total`
-    * Start time -- `process_start_time_seconds`
-    * Count -- `process_count`
-    * Memory usage
-      * Data from /proc/<pid>/cmdline:
-        * `process_resident_memory_bytes`
-        * `process_virtual_memory_bytes`
-      * Data from /proc/<pid>/smaps -- `probe_smaps` (off by default):
-        * `process_smaps_size_bytes`
-        * `process_smaps_rss_bytes`
-        * `process_smaps_shared_clean_bytes`
-        * `process_smaps_shared_dirty_bytes`
-        * `process_smaps_private_clean_bytes`
-        * `process_smaps_private_dirty_bytes`
-        * `process_smaps_swap_bytes`
-        * `process_smaps_pss_bytes`
 1. [Sidekiq](lib/gitlab_exporter/sidekiq.rb)
 	* Stats
 	  * `sidekiq_jobs_processed_total`

data/bin/gitlab-exporter

@@ -1,7 +1,10 @@
 #!/usr/bin/env ruby
 
-$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
 
+require "rubygems"
+require "bundler"
+require "bundler/setup"
 require "optparse"
 require "gitlab_exporter"
 

data/config/gitlab-exporter.yml.example

@@ -46,7 +46,6 @@ probes:
       <<: *db_common
       opts:
         <<: *db_common_opts
-        allowed_repeated_commands_count: 2
         created_builds_counting_disabled: true
         unarchived_traces_offset_minutes: 1440
     tuple_stats:
@@ -95,12 +94,21 @@ probes:
       redis_url: "redis://localhost:6379"
       redis_enable_client: true
 
+  ruby: &ruby
+    class_name: RubyProber
+    methods:
+    - probe_gc
+    opts:
+      quantiles: false
+
   metrics:
     multiple: true
     git_process:
       <<: *git_process
     process:
       <<: *process
+    ruby:
+      <<: *ruby
     sidekiq:
       <<: *sidekiq
     ci_builds:

data/gitlab-exporter.gemspec

@@ -1,4 +1,4 @@
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "gitlab_exporter/version"
 
@@ -20,13 +20,14 @@ Gem::Specification.new do |s|
   s.homepage      = "http://gitlab.com"
   s.license       = "MIT"
 
+  s.add_runtime_dependency "connection_pool", "~> 2.2.1"
   s.add_runtime_dependency "pg", "~> 1.1"
-  s.add_runtime_dependency "sinatra", "~> 2.0.4"
+  s.add_runtime_dependency "puma", "~> 5.1.1"
   s.add_runtime_dependency "quantile", "~> 0.2.0"
-  s.add_runtime_dependency "sidekiq", "~> 5.2.1"
   s.add_runtime_dependency "redis", "~> 4.1.2"
   s.add_runtime_dependency "redis-namespace", "~> 1.6.0"
-  s.add_runtime_dependency "connection_pool", "~> 2.2.1"
+  s.add_runtime_dependency "sidekiq", "~> 5.2.1"
+  s.add_runtime_dependency "sinatra", "~> 2.0.4"
 
   s.add_development_dependency "rspec", "~> 3.7.0"
   s.add_development_dependency "rspec-expectations", "~> 3.7.0"

data/lib/gitlab_exporter.rb

@@ -14,5 +14,6 @@ module GitLab
     autoload :WebExporter,       "gitlab_exporter/web_exporter"
     autoload :Prober,            "gitlab_exporter/prober"
     autoload :SidekiqProber,     "gitlab_exporter/sidekiq"
+    autoload :RubyProber,        "gitlab_exporter/ruby"
   end
 end

data/lib/gitlab_exporter/cli.rb

@@ -19,8 +19,7 @@ module GitLab
       # Empty runner that will raise an InvalidCLICommand when executed to provide the usage
       # in the exception message
       class NullRunner
-        def initialize(args)
-        end
+        def initialize(args); end
 
         def run
           fail InvalidCLICommand.new("Usage: #{EXECUTABLE_NAME} <command> [options] [arguments...]\n\n"\

data/lib/gitlab_exporter/database/base.rb

@@ -8,14 +8,39 @@ module GitLab
       #
       # It takes a connection string (e.g. "dbname=test port=5432")
       class Base
+        POOL_SIZE = 3
+
+        # This timeout is configured to higher interval than scrapping
+        # of Prometheus to ensure that connection is kept instead of
+        # needed to be re-initialized
+        POOL_TIMEOUT = 90
+
         def self.connection_pool
-          @connection_pool ||= Hash.new do |h, connection_string|
-            h[connection_string] = ConnectionPool.new(size: 3, timeout: 5) do
-              PG.connect(connection_string)
+          @@connection_pool ||= Hash.new do |h, connection_string| # rubocop:disable Style/ClassVars
+            h[connection_string] = ConnectionPool.new(size: POOL_SIZE, timeout: POOL_TIMEOUT) do
+              PG.connect(connection_string).tap do |conn|
+                configure_type_map_for_results(conn)
+              end
             end
           end
         end
 
+        def self.configure_type_map_for_results(conn)
+          tm = PG::BasicTypeMapForResults.new(conn)
+
+          # Remove warning message:
+          # Warning: no type cast defined for type "name" with oid 19.
+          # Please cast this type explicitly to TEXT to be safe for future changes.
+          # Warning: no type cast defined for type "regproc" with oid 24.
+          # Please cast this type explicitly to TEXT to be safe for future changes.
+          [{ "type": "text", "oid": 19 }, { "type": "int4", "oid": 24 }].each do |value|
+            old_coder = tm.coders.find { |c| c.name == value[:type] }
+            tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
+          end
+
+          conn.type_map_for_results = tm
+        end
+
         def initialize(args, logger: nil)
           @connection_string = args[:connection_string]
           @logger = logger
@@ -31,13 +56,11 @@ module GitLab
 
         def with_connection_pool
           connection_pool.with do |conn|
-            begin
-              yield conn
-            rescue PG::UnableToSend => e
-              @logger.warn "Error sending to the database: #{e}" if @logger
-              conn.reset
-              raise e
-            end
+            yield conn
+          rescue PG::UnableToSend => e
+            @logger.warn "Error sending to the database: #{e}" if @logger
+            conn.reset
+            raise e
           end
         rescue PG::Error => e
           @logger.error "Error connecting to the database: #{e}" if @logger

data/lib/gitlab_exporter/database/bloat.rb

@@ -35,7 +35,7 @@ module GitLab
 
       # Prober class to gather bloat metrics
       class BloatProber
-        METRIC_KEYS = %w(bloat_ratio bloat_size extra_size real_size).freeze
+        METRIC_KEYS = %w[bloat_ratio bloat_size extra_size real_size].freeze
 
         attr_reader :metrics, :collector, :bloat_types
 
@@ -46,7 +46,7 @@ module GitLab
           @metrics = metrics
           @collector = collector
           @collector.logger = logger
-          @bloat_types = opts[:bloat_types] || %i(btree table)
+          @bloat_types = opts[:bloat_types] || %i[btree table]
         end
 
         def probe_db

data/lib/gitlab_exporter/database/ci_builds.rb

@@ -147,100 +147,6 @@ module GitLab
             SELECT COUNT(*) FROM licenses
           SQL
 
-        REPEATED_COMMANDS_QUERY_EE =
-          <<~SQL.freeze
-            SELECT
-              subquery.namespace_id,
-              subquery.shared_runners_enabled,
-              subquery.project_id,
-              subquery.status,
-              subquery.has_minutes,
-              MAX(subquery.count) as count
-            FROM (
-              SELECT
-                projects.namespace_id,
-                projects.shared_runners_enabled,
-                ci_builds.project_id,
-                ci_builds.commit_id,
-                ci_builds.status,
-                (COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) = 0 OR
-                   COALESCE(namespace_statistics.shared_runners_seconds, 0) < COALESCE(namespaces.shared_runners_minutes_limit, application_settings.shared_runners_minutes, 0) * 60) as has_minutes,
-                COUNT(*) AS count
-              FROM ci_builds
-              JOIN projects
-                ON projects.id = ci_builds.project_id
-              JOIN namespaces
-                ON namespaces.id = projects.namespace_id
-              LEFT JOIN namespace_statistics
-                ON namespace_statistics.namespace_id = namespaces.id
-              JOIN application_settings ON (TRUE)
-              WHERE ci_builds.type = 'Ci::Build'
-                AND ci_builds.status IN ('running', 'pending')
-                -- The created_at filter has been introduced for performance reasons only
-                AND ci_builds.created_at > NOW() - INTERVAL '7 days'
-              GROUP BY
-                projects.namespace_id,
-                projects.shared_runners_enabled,
-                ci_builds.project_id,
-                ci_builds.commit_id,
-                ci_builds.status,
-                ci_builds.commands,
-                namespaces.shared_runners_minutes_limit,
-                namespace_statistics.shared_runners_seconds,
-                application_settings.shared_runners_minutes
-              HAVING COUNT(*) > %d
-            ) AS subquery
-            GROUP BY
-              subquery.namespace_id,
-              subquery.shared_runners_enabled,
-              subquery.project_id,
-              subquery.commit_id,
-              subquery.status,
-              subquery.has_minutes
-          SQL
-
-        REPEATED_COMMANDS_QUERY_CE =
-          <<~SQL.freeze
-            SELECT
-              subquery.namespace_id,
-              subquery.shared_runners_enabled,
-              subquery.project_id,
-              subquery.status,
-              MAX(subquery.count) as count
-            FROM (
-              SELECT
-                projects.namespace_id,
-                projects.shared_runners_enabled,
-                ci_builds.project_id,
-                ci_builds.commit_id,
-                ci_builds.status,
-                COUNT(*) AS count
-              FROM ci_builds
-              JOIN projects
-                ON projects.id = ci_builds.project_id
-              JOIN namespaces
-                ON namespaces.id = projects.namespace_id
-              WHERE ci_builds.type = 'Ci::Build'
-                AND ci_builds.status IN ('running', 'pending')
-              -- The created_at filter has been introduced for performance reasons only
-              AND ci_builds.created_at > NOW() - INTERVAL '7 days'
-              GROUP BY
-                projects.namespace_id,
-                projects.shared_runners_enabled,
-                ci_builds.project_id,
-                ci_builds.commit_id,
-                ci_builds.status,
-                ci_builds.commands
-              HAVING COUNT(*) > %d
-            ) AS subquery
-            GROUP BY
-              subquery.namespace_id,
-              subquery.shared_runners_enabled,
-              subquery.project_id,
-              subquery.commit_id,
-              subquery.status
-          SQL
-
         UNARCHIVED_TRACES_QUERY =
           <<~SQL.freeze
             SELECT
@@ -265,7 +171,6 @@ module GitLab
         def initialize(opts, logger: nil)
           super(opts, logger: logger)
 
-          @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
           @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
           @unarchived_traces_offset_minutes = opts[:unarchived_traces_offset_minutes]
         end
@@ -276,7 +181,6 @@ module GitLab
           results[:pending_builds] = builds(STATUS_PENDING)
           results[:stale_builds] = stale_builds
           results[:per_runner] = per_runner_builds
-          results[:repeated_commands] = repeated_commands
           results[:unarchived_traces] = unarchived_traces
           results
         end
@@ -335,34 +239,6 @@ module GitLab
           include_ee_fields(values, row)
         end
 
-        def repeated_commands
-          results = []
-
-          query = ee? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
-          query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
-          exec_query_with_custom_random_page_cost(query).each do |row|
-            results << transform_repeated_commands_row_to_values(row)
-          end
-
-          results
-        rescue PG::UndefinedTable, PG::UndefinedColumn
-          []
-        end
-
-        def allowed_repeated_commands_count
-          @allowed_repeated_commands_count ||= 2
-        end
-
-        def transform_repeated_commands_row_to_values(row)
-          values = { namespace: row["namespace_id"].to_s,
-                     project: row["project_id"].to_s,
-                     shared_runners: row["shared_runners_enabled"] == "t" ? "yes" : "no",
-                     status: row["status"].to_s,
-                     value: row["count"].to_i }
-
-          include_has_minutes_field(values, row)
-        end
-
         def unarchived_traces
           time = Time.now - (unarchived_traces_offset_minutes * 60)
           query = UNARCHIVED_TRACES_QUERY % [time.strftime("%F %T")] # rubocop:disable Style/FormatString
@@ -391,6 +267,7 @@ module GitLab
 
         def include_bool_if_row_defined(row, field)
           return {} unless row[field.to_s]
+
           { field => row[field.to_s] == "t" ? "yes" : "no" }
         end
 
@@ -421,7 +298,6 @@ module GitLab
           @metrics = metrics
 
           collector_opts = { connection_string: opts[:connection_string],
-                             allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
                              created_builds_counting_disabled: opts[:created_builds_counting_disabled],
                              unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
           @collector = CiBuildsCollector.new(collector_opts, logger: logger)
@@ -434,7 +310,6 @@ module GitLab
           ci_builds_metrics(@results[:pending_builds], "ci_pending_builds")
           ci_stale_builds_metrics
           metrics_per_runner
-          repeated_commands_metrics
           unarchived_traces_metrics
 
           self
@@ -470,7 +345,7 @@ module GitLab
 
         def add_ci_created_pending_builds(metric_name, value, labels)
           add_metric_with_namespace_label(metric_name,
-                                          [:namespace, :shared_runners, :has_minutes],
+                                          %i[namespace shared_runners has_minutes],
                                           value,
                                           labels)
         end
@@ -505,8 +380,8 @@ module GitLab
         def add_ci_running_builds(value, labels)
           add_metric_with_namespace_label(
             "ci_running_builds",
-            [:runner, :namespace, :runner_type, :scheduled,
-             :triggered, :mirror, :mirror_trigger_builds, :has_minutes],
+            %i[runner namespace runner_type scheduled
+               triggered mirror mirror_trigger_builds has_minutes],
             value,
             labels
           )
@@ -519,14 +394,6 @@ module GitLab
           @metrics.add(metric_name, value.to_f, selected_labels)
         end
 
-        def repeated_commands_metrics
-          @results[:repeated_commands].each do |metric|
-            value = metric.delete(:value)
-
-            @metrics.add("ci_repeated_commands_builds", value.to_f, metric)
-          end
-        end
-
         def unarchived_traces_metrics
           @metrics.add("ci_unarchived_traces", @results[:unarchived_traces].to_f)
         end