gitlab-exporter 7.0.2 → 7.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd49abef867540a7fcfb667672ddef699d7bf883ddf4a3fc3856b1097959386b
-  data.tar.gz: e959328537cd1b32003a9d1d7f006dc4812bf52bf82e71dab9a972f4ee86e4d5
+  metadata.gz: 2590101193dd650767c1f39170beb0b54661c188b7c1222fa7d01d59ee1f442a
+  data.tar.gz: 9237f257766fefe33cf0e398a86d44ecae609a393c046c7e5203b07a74adaca4
 SHA512:
-  metadata.gz: 6c6a2a358ae1f30483f35cad1e26e5d074772d30522cb9bfea85436481f59f160170c90a309012f8def286808ebc334d3b0f1c451045a99fc7a7048505b7efc9
-  data.tar.gz: 2df79f4b275b371846160f3d552a14254a21dcad1ba2e3650c92383384d17671b325d4e5daf7235b89f83c16eb7a825ca60b38be8a5f145df1c0ca8be73bdeaa
+  metadata.gz: 4109296df37f483e64b5c3f9c9da8ce6359a8b36cbb8b2bc7a40c3147ee0401fafee78829b6e67e43167b1a859d771669e6f94e85f7ebf8529a7bbea45be6431
+  data.tar.gz: e4d05c4fad006d5a27f693887320b6f2a633f96b1109e97f7e89f5b990d695528b72ef2e7b876901d520df3f4be6c6b24850208228923c227443e06b70b81988

data/.gitlab-ci.yml

@@ -26,3 +26,11 @@ rspec:
 rubocop:
   script:
     - bundle exec rubocop
+
+include:
+  - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+  - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+  - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (7.0.2)
+    gitlab-exporter (7.1.0)
       connection_pool (~> 2.2.1)
       pg (~> 1.1)
       quantile (~> 0.2.0)
@@ -14,19 +14,20 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
-    mustermann (1.0.3)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     parser (2.5.1.0)
       ast (~> 2.4.0)
-    pg (1.2.2)
+    pg (1.2.3)
     powerpack (0.1.1)
     quantile (0.2.1)
-    rack (2.0.7)
-    rack-protection (2.0.7)
+    rack (2.2.3)
+    rack-protection (2.0.8.1)
       rack
     rainbow (2.1.0)
-    redis (4.1.3)
+    redis (4.1.4)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
     rspec (3.7.0)
@@ -49,17 +50,18 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.8.1)
-    sidekiq (5.2.7)
+    ruby2_keywords (0.0.2)
+    sidekiq (5.2.9)
       connection_pool (~> 2.2, >= 2.2.2)
-      rack (>= 1.5.0)
+      rack (~> 2.0)
       rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
-    sinatra (2.0.7)
+      redis (>= 3.3.5, < 4.2)
+    sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.7)
+      rack-protection (= 2.0.8.1)
       tilt (~> 2.0)
-    tilt (2.0.9)
+    tilt (2.0.10)
     unicode-display_width (1.6.0)
 
 PLATFORMS

data/lib/gitlab_exporter/database/base.rb

@@ -16,8 +16,9 @@ module GitLab
           end
         end
 
-        def initialize(args)
+        def initialize(args, logger: nil)
           @connection_string = args[:connection_string]
+          @logger = logger
         end
 
         def run
@@ -33,10 +34,14 @@ module GitLab
             begin
               yield conn
             rescue PG::UnableToSend => e
+              @logger.warn "Error sending to the database: #{e}" if @logger
               conn.reset
               raise e
             end
           end
+        rescue PG::Error => e
+          @logger.error "Error connecting to the database: #{e}" if @logger
+          raise e
         end
       end
     end

data/lib/gitlab_exporter/database/ci_builds.rb

@@ -21,8 +21,7 @@ module GitLab
               ON namespaces.id = projects.namespace_id
             LEFT JOIN namespace_statistics
               ON namespace_statistics.namespace_id = namespaces.id
-            JOIN application_settings
-              ON application_settings.id = 1
+            JOIN application_settings ON (TRUE)
             WHERE ci_builds.type = 'Ci::Build'
               AND ci_builds.status = '%s'
               -- The created_at filter has been introduced for performance reasons only
@@ -95,8 +94,7 @@ module GitLab
               ON namespaces.id = projects.namespace_id
             LEFT JOIN namespace_statistics
               ON namespace_statistics.namespace_id = namespaces.id
-            JOIN application_settings
-              ON application_settings.id = 1
+            JOIN application_settings ON (TRUE)
             WHERE ci_builds.type = 'Ci::Build'
               AND ci_builds.status = 'running'
               -- The created_at filter has been introduced for performance reasons only
@@ -144,9 +142,9 @@ module GitLab
               ci_builds.trigger_request_id
           SQL
 
-        MIRROR_COLUMN_QUERY =
+        EE_CHECK_QUERY =
           <<~SQL.freeze
-            SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='projects' AND column_name='mirror')
+            SELECT COUNT(*) FROM licenses
           SQL
 
         REPEATED_COMMANDS_QUERY_EE =
@@ -175,8 +173,7 @@ module GitLab
                 ON namespaces.id = projects.namespace_id
               LEFT JOIN namespace_statistics
                 ON namespace_statistics.namespace_id = namespaces.id
-              JOIN application_settings
-                ON application_settings.id = 1
+              JOIN application_settings ON (TRUE)
               WHERE ci_builds.type = 'Ci::Build'
                 AND ci_builds.status IN ('running', 'pending')
                 -- The created_at filter has been introduced for performance reasons only
@@ -265,8 +262,8 @@ module GitLab
 
         DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440
 
-        def initialize(opts)
-          super(opts)
+        def initialize(opts, logger: nil)
+          super(opts, logger: logger)
 
           @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
           @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
@@ -289,7 +286,7 @@ module GitLab
         def builds(status)
           results = []
 
-          query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
+          query = ee? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
           query = query % [status] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_builds_row_to_values(row)
@@ -318,7 +315,7 @@ module GitLab
         def per_runner_builds
           results = []
 
-          query = mirror_column? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
+          query = ee? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_per_runners_builds_row_to_values(row)
           end
@@ -341,7 +338,7 @@ module GitLab
         def repeated_commands
           results = []
 
-          query = mirror_column? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
+          query = ee? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
           query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_repeated_commands_row_to_values(row)
@@ -406,11 +403,11 @@ module GitLab
           end
         end
 
-        def mirror_column?
-          @mirror_column ||=
+        def ee?
+          @ee ||=
             begin
               with_connection_pool do |conn|
-                conn.exec(MIRROR_COLUMN_QUERY)[0]["exists"] == "t"
+                conn.exec(EE_CHECK_QUERY)[0]["count"].to_i > 0 # rubocop:disable Style/NumericPredicate
               end
             rescue PG::UndefinedColumn
               false
@@ -420,14 +417,14 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class CiBuildsProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
           @metrics = metrics
 
           collector_opts = { connection_string: opts[:connection_string],
                              allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
                              created_builds_counting_disabled: opts[:created_builds_counting_disabled],
                              unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
-          @collector = CiBuildsCollector.new(collector_opts)
+          @collector = CiBuildsCollector.new(collector_opts, logger: logger)
         end
 
         def probe_db

data/lib/gitlab_exporter/database/remote_mirrors.rb

@@ -35,7 +35,7 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class RemoteMirrorsProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
           @metrics = metrics
           @collector = RemoteMirrorsCollector.new(
             connection_string: opts[:connection_string],

data/lib/gitlab_exporter/database/row_count.rb

@@ -110,7 +110,7 @@ module GitLab
                 GROUP BY members.user_id
               ) AS u
               ON users.id = u.user_id",
-            where: "ghost IS NULL AND bot_type IS NULL",
+            where: "user_type IS NULL",
             fields: {
               admin:        {},
               external:     {},
@@ -212,7 +212,7 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class RowCountProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
           @metrics = metrics
           @collector = RowCountCollector.new(
             connection_string: opts[:connection_string],

data/lib/gitlab_exporter/database/tuple_stats.rb

@@ -25,9 +25,9 @@ module GitLab
 
       # Probes the DB specified by opts[:connection_string] for tuple stats, then converts them to metrics
       class TuplesProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
           @metrics   = metrics
-          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string])
+          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string], logger: logger)
         end
 
         def probe_db

data/lib/gitlab_exporter/git.rb

@@ -62,7 +62,7 @@ module GitLab
     # Optionally takes a metrics object which by default is a PrometheusMetrics, useful to change the
     # metrics writer to something else.
     class GitProber
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @metrics = metrics
         @labels = opts[:labels] || {}
         @git = Git.new(opts[:source])
@@ -85,7 +85,7 @@ module GitLab
 
     # A special prober for git processes
     class GitProcessProber
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @opts    = opts
         @metrics = metrics
       end

data/lib/gitlab_exporter/prober.rb

@@ -2,9 +2,10 @@ module GitLab
   module Exporter
     # A class to combine multiple probers into one
     class Prober
-      def initialize(prober_opts, metrics: PrometheusMetrics.new)
+      def initialize(prober_opts, metrics: PrometheusMetrics.new, logger: nil)
         @prober_opts = prober_opts
         @metrics     = metrics
+        @logger      = logger
 
         resolve_prober_classes
       end
@@ -12,7 +13,7 @@ module GitLab
       def probe_all
         @prober_opts.each do |_probe_name, params|
           Utils.wrap_in_array(params[:opts]).each do |opts|
-            prober = params[:class].new(opts, metrics: @metrics)
+            prober = params[:class].new(opts, metrics: @metrics, logger: @logger)
             params[:methods].each do |meth|
               prober.send(meth)
             end

data/lib/gitlab_exporter/process.rb

@@ -59,7 +59,7 @@ module GitLab
 
     # Probes a process for info then writes metrics to a target
     class ProcessProber
-      def initialize(options, metrics: PrometheusMetrics.new)
+      def initialize(options, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @metrics = metrics
         @name    = options[:name]
         @pids    = if options[:pid_or_pattern] =~ /^\d+$/

data/lib/gitlab_exporter/sidekiq.rb

@@ -10,9 +10,10 @@ module GitLab
       QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
       QUEUE_JOB_STATS_SHA    = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
 
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
         @opts    = opts
         @metrics = metrics
+        @logger  = logger
 
         Sidekiq.configure_client do |config|
           config.redis = redis_options

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "7.0.2".freeze
+    VERSION = "7.1.0".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -42,6 +42,11 @@ module GitLab
 
           memory_threshold = (config[:server] && config[:server][:memory_threshold]) || 1024
           use MemoryKillerMiddleware, memory_threshold
+          use Rack::Logger
+        end
+
+        def logger
+          request.logger
         end
 
         def setup_server(config)
@@ -62,7 +67,7 @@ module GitLab
 
             get "/#{probe_name}" do
               content_type "text/plain; version=0.0.4"
-              prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false))
+              prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false), logger: logger)
 
               prober.probe_all
               prober.write_to(response)

data/spec/database/ci_builds_spec.rb

@@ -9,7 +9,7 @@ describe GitLab::Exporter::Database do
   let(:stale_builds_query) { "SELECT NOT UPDATED RUNNING" }
   let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
   let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
-  let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
+  let(:ee_check_query) { "SELECT COUNT(*) FROM licenses" }
   let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
   let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
   let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
@@ -22,11 +22,11 @@ describe GitLab::Exporter::Database do
   let(:unarchived_traces_offset_minutes) { 60 }
 
   def stub_ee
-    allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
+    allow(connection).to receive(:exec).with(ee_check_query).and_return([{ "count" => 1 }])
   end
 
   def stub_ce
-    allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "f" }])
+    allow(connection).to receive(:exec).with(ee_check_query).and_return([{ "count" => 0 }])
   end
 
   def builds_query_row_ee(shared_runners_enabled, status, namespace_id, has_minutes, count)
@@ -86,7 +86,7 @@ describe GitLab::Exporter::Database do
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::STALE_BUILDS_QUERY", stale_builds_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
-    stub_const("GitLab::Exporter::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
+    stub_const("GitLab::Exporter::Database::CiBuildsCollector::EE_CHECK_QUERY", ee_check_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 7.0.2
+  version: 7.1.0
 platform: ruby
 authors:
 - Pablo Carranza
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
@@ -191,7 +191,7 @@ homepage: http://gitlab.com
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -207,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: GitLab metrics exporter
 test_files: