gitlab-exporter 7.0.3 → 7.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e33f039a7668f804326d49f446f2557e30ef69c3c83e4016c62efa8bbc2ea991
-  data.tar.gz: 405cdd0ef307fc0deee5d695be1fd925d5c1700b6ff3067964b8d09c7da585e4
+  metadata.gz: 9ef9540e75d17cc9904aa53962399870425e6ca52d57ed994dff10d53fad201f
+  data.tar.gz: 7ab18ebbb09faaac5c4924cab93af2059dbe28e4177365771c897a153878daee
 SHA512:
-  metadata.gz: e4f6c648f48262736fe8e7f29ead0f97f4a4af9a27d8daa7a63ee5c27e35cd7cdb62f2b10e25e6aaba511de7eb5a6c593dc037ae3bbb6a04aad829ee78c269cc
-  data.tar.gz: d12c69bfb2f0754c62bb7d0bb3c436188f8322aefd5575c1eadfa9fbbf5921716207305eba9266e9880b7e4dcfca3f3ba66f937925636058643490b358f15cfc
+  metadata.gz: 05a779426c541c1371d880b9b2d4a321b78243b4bb89ce8c8a4d41e977569f88386505e205719079b0ea85a1bbc3f9d5558bdbb6e1de94de4eb283c05513ad16
+  data.tar.gz: 39fa8909ce99c35bc0e929760cf820593b3ee30526fdb4d10f3230646f63f271b4b3b8e29cc62fbde59106d5f4563335c184f18de21fe8fe5246f7b722ed0b0e

data/.gitlab-ci.yml

@@ -1,17 +1,26 @@
+include:
+  - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+  - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+  - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+  - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+
 default:
   image: "ruby:2.3"
-  before_script:
-    - git config --global user.email "bot@gitlab.com"
-    - git config --global user.name "Bot User"
-    - bundle install -j $(nproc) --path vendor
   cache:
     paths:
       - vendor
   tags:
     - gitlab-org
 
+.before_scripts: &before_scripts
+  - git config --global user.email "bot@gitlab.com"
+  - git config --global user.name "Bot User"
+  - bundle install -j $(nproc) --path vendor
+
 workflow:
-  rules:
+  rules: &workflow_rules
     # For merge requests, create a pipeline.
     - if: '$CI_MERGE_REQUEST_IID'
     # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
@@ -22,7 +31,18 @@ workflow:
 rspec:
   script:
     - bundle exec rspec spec -f d -c
+  before_script: *before_scripts
 
 rubocop:
   script:
     - bundle exec rubocop
+  before_script: *before_scripts
+
+license_scanning:
+  rules: *workflow_rules
+
+gemnasium-dependency_scanning:
+  rules: *workflow_rules
+
+secret_detection:
+  rules: *workflow_rules

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (7.0.3)
+    gitlab-exporter (7.1.1)
       connection_pool (~> 2.2.1)
       pg (~> 1.1)
       quantile (~> 0.2.0)
@@ -14,19 +14,20 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
-    mustermann (1.0.3)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     parser (2.5.1.0)
       ast (~> 2.4.0)
-    pg (1.2.2)
+    pg (1.2.3)
     powerpack (0.1.1)
     quantile (0.2.1)
-    rack (2.0.7)
-    rack-protection (2.0.7)
+    rack (2.2.3)
+    rack-protection (2.0.8.1)
       rack
     rainbow (2.1.0)
-    redis (4.1.3)
+    redis (4.1.4)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
     rspec (3.7.0)
@@ -49,17 +50,18 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.8.1)
-    sidekiq (5.2.7)
+    ruby2_keywords (0.0.2)
+    sidekiq (5.2.9)
       connection_pool (~> 2.2, >= 2.2.2)
-      rack (>= 1.5.0)
+      rack (~> 2.0)
       rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
-    sinatra (2.0.7)
+      redis (>= 3.3.5, < 4.2)
+    sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.7)
+      rack-protection (= 2.0.8.1)
       tilt (~> 2.0)
-    tilt (2.0.9)
+    tilt (2.0.10)
     unicode-display_width (1.6.0)
 
 PLATFORMS

data/lib/gitlab_exporter/database/base.rb

@@ -16,8 +16,9 @@ module GitLab
           end
         end
 
-        def initialize(args)
+        def initialize(args, logger: nil)
           @connection_string = args[:connection_string]
+          @logger = logger
         end
 
         def run
@@ -33,10 +34,14 @@ module GitLab
             begin
               yield conn
             rescue PG::UnableToSend => e
+              @logger.warn "Error sending to the database: #{e}" if @logger
               conn.reset
               raise e
             end
           end
+        rescue PG::Error => e
+          @logger.error "Error connecting to the database: #{e}" if @logger
+          raise e
         end
       end
     end

data/lib/gitlab_exporter/database/ci_builds.rb

@@ -21,8 +21,7 @@ module GitLab
               ON namespaces.id = projects.namespace_id
             LEFT JOIN namespace_statistics
               ON namespace_statistics.namespace_id = namespaces.id
-            JOIN application_settings
-              ON application_settings.id = 1
+            JOIN application_settings ON (TRUE)
             WHERE ci_builds.type = 'Ci::Build'
               AND ci_builds.status = '%s'
               -- The created_at filter has been introduced for performance reasons only
@@ -95,8 +94,7 @@ module GitLab
               ON namespaces.id = projects.namespace_id
             LEFT JOIN namespace_statistics
               ON namespace_statistics.namespace_id = namespaces.id
-            JOIN application_settings
-              ON application_settings.id = 1
+            JOIN application_settings ON (TRUE)
             WHERE ci_builds.type = 'Ci::Build'
               AND ci_builds.status = 'running'
               -- The created_at filter has been introduced for performance reasons only
@@ -144,9 +142,9 @@ module GitLab
               ci_builds.trigger_request_id
           SQL
 
-        MIRROR_COLUMN_QUERY =
+        EE_CHECK_QUERY =
           <<~SQL.freeze
-            SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='projects' AND column_name='mirror')
+            SELECT COUNT(*) FROM licenses
           SQL
 
         REPEATED_COMMANDS_QUERY_EE =
@@ -175,8 +173,7 @@ module GitLab
                 ON namespaces.id = projects.namespace_id
               LEFT JOIN namespace_statistics
                 ON namespace_statistics.namespace_id = namespaces.id
-              JOIN application_settings
-                ON application_settings.id = 1
+              JOIN application_settings ON (TRUE)
               WHERE ci_builds.type = 'Ci::Build'
                 AND ci_builds.status IN ('running', 'pending')
                 -- The created_at filter has been introduced for performance reasons only
@@ -265,8 +262,8 @@ module GitLab
 
         DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440
 
-        def initialize(opts)
-          super(opts)
+        def initialize(opts, logger: nil)
+          super(opts, logger: logger)
 
           @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
           @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
@@ -289,7 +286,7 @@ module GitLab
         def builds(status)
           results = []
 
-          query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
+          query = ee? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
           query = query % [status] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_builds_row_to_values(row)
@@ -318,7 +315,7 @@ module GitLab
         def per_runner_builds
           results = []
 
-          query = mirror_column? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
+          query = ee? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_per_runners_builds_row_to_values(row)
           end
@@ -341,7 +338,7 @@ module GitLab
         def repeated_commands
           results = []
 
-          query = mirror_column? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
+          query = ee? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
           query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_repeated_commands_row_to_values(row)
@@ -406,11 +403,11 @@ module GitLab
           end
         end
 
-        def mirror_column?
-          @mirror_column ||=
+        def ee?
+          @ee ||=
             begin
               with_connection_pool do |conn|
-                conn.exec(MIRROR_COLUMN_QUERY)[0]["exists"] == "t"
+                conn.exec(EE_CHECK_QUERY)[0]["count"].to_i > 0 # rubocop:disable Style/NumericPredicate
               end
             rescue PG::UndefinedColumn
               false
@@ -420,14 +417,14 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class CiBuildsProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
           @metrics = metrics
 
           collector_opts = { connection_string: opts[:connection_string],
                              allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
                              created_builds_counting_disabled: opts[:created_builds_counting_disabled],
                              unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
-          @collector = CiBuildsCollector.new(collector_opts)
+          @collector = CiBuildsCollector.new(collector_opts, logger: logger)
         end
 
         def probe_db

data/lib/gitlab_exporter/database/remote_mirrors.rb

@@ -35,7 +35,7 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class RemoteMirrorsProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
           @metrics = metrics
           @collector = RemoteMirrorsCollector.new(
             connection_string: opts[:connection_string],

data/lib/gitlab_exporter/database/row_count.rb

@@ -110,7 +110,7 @@ module GitLab
                 GROUP BY members.user_id
               ) AS u
               ON users.id = u.user_id",
-            where: "ghost IS NULL AND user_type IS NULL",
+            where: "user_type IS NULL",
             fields: {
               admin:        {},
               external:     {},
@@ -175,6 +175,14 @@ module GitLab
 
         def execute(query)
           with_connection_pool do |conn|
+            conn.exec(query).map_types!(type_map_for_results(conn))
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          nil
+        end
+
+        def type_map_for_results(conn)
+          @type_map_for_results ||= begin
             tm = PG::BasicTypeMapForResults.new(conn)
 
             # Remove warning message:
@@ -187,10 +195,8 @@ module GitLab
               tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
             end
 
-            conn.exec(query).map_types!(tm)
+            tm
           end
-        rescue PG::UndefinedTable, PG::UndefinedColumn
-          nil
         end
 
         # Not private so I can test it without meta programming tricks
@@ -212,7 +218,7 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class RowCountProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
           @metrics = metrics
           @collector = RowCountCollector.new(
             connection_string: opts[:connection_string],

data/lib/gitlab_exporter/database/tuple_stats.rb

@@ -25,9 +25,9 @@ module GitLab
 
       # Probes the DB specified by opts[:connection_string] for tuple stats, then converts them to metrics
       class TuplesProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
           @metrics   = metrics
-          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string])
+          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string], logger: logger)
         end
 
         def probe_db

data/lib/gitlab_exporter/git.rb

@@ -62,7 +62,7 @@ module GitLab
     # Optionally takes a metrics object which by default is a PrometheusMetrics, useful to change the
     # metrics writer to something else.
     class GitProber
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @metrics = metrics
         @labels = opts[:labels] || {}
         @git = Git.new(opts[:source])
@@ -85,7 +85,7 @@ module GitLab
 
     # A special prober for git processes
     class GitProcessProber
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @opts    = opts
         @metrics = metrics
       end

data/lib/gitlab_exporter/prober.rb

@@ -2,9 +2,10 @@ module GitLab
   module Exporter
     # A class to combine multiple probers into one
     class Prober
-      def initialize(prober_opts, metrics: PrometheusMetrics.new)
+      def initialize(prober_opts, metrics: PrometheusMetrics.new, logger: nil)
         @prober_opts = prober_opts
         @metrics     = metrics
+        @logger      = logger
 
         resolve_prober_classes
       end
@@ -12,7 +13,7 @@ module GitLab
       def probe_all
         @prober_opts.each do |_probe_name, params|
           Utils.wrap_in_array(params[:opts]).each do |opts|
-            prober = params[:class].new(opts, metrics: @metrics)
+            prober = params[:class].new(opts, metrics: @metrics, logger: @logger)
             params[:methods].each do |meth|
               prober.send(meth)
             end

data/lib/gitlab_exporter/process.rb

@@ -59,7 +59,7 @@ module GitLab
 
     # Probes a process for info then writes metrics to a target
     class ProcessProber
-      def initialize(options, metrics: PrometheusMetrics.new)
+      def initialize(options, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @metrics = metrics
         @name    = options[:name]
         @pids    = if options[:pid_or_pattern] =~ /^\d+$/

data/lib/gitlab_exporter/sidekiq.rb

@@ -10,9 +10,10 @@ module GitLab
       QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
       QUEUE_JOB_STATS_SHA    = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
 
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
         @opts    = opts
         @metrics = metrics
+        @logger  = logger
 
         Sidekiq.configure_client do |config|
           config.redis = redis_options

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "7.0.3".freeze
+    VERSION = "7.1.1".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -42,6 +42,11 @@ module GitLab
 
           memory_threshold = (config[:server] && config[:server][:memory_threshold]) || 1024
           use MemoryKillerMiddleware, memory_threshold
+          use Rack::Logger
+        end
+
+        def logger
+          request.logger
         end
 
         def setup_server(config)
@@ -62,7 +67,7 @@ module GitLab
 
             get "/#{probe_name}" do
               content_type "text/plain; version=0.0.4"
-              prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false))
+              prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false), logger: logger)
 
               prober.probe_all
               prober.write_to(response)

data/spec/database/ci_builds_spec.rb

@@ -9,7 +9,7 @@ describe GitLab::Exporter::Database do
   let(:stale_builds_query) { "SELECT NOT UPDATED RUNNING" }
   let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
   let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
-  let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
+  let(:ee_check_query) { "SELECT COUNT(*) FROM licenses" }
   let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
   let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
   let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
@@ -22,11 +22,11 @@ describe GitLab::Exporter::Database do
   let(:unarchived_traces_offset_minutes) { 60 }
 
   def stub_ee
-    allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
+    allow(connection).to receive(:exec).with(ee_check_query).and_return([{ "count" => 1 }])
   end
 
   def stub_ce
-    allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "f" }])
+    allow(connection).to receive(:exec).with(ee_check_query).and_return([{ "count" => 0 }])
   end
 
   def builds_query_row_ee(shared_runners_enabled, status, namespace_id, has_minutes, count)
@@ -86,7 +86,7 @@ describe GitLab::Exporter::Database do
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::STALE_BUILDS_QUERY", stale_builds_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
-    stub_const("GitLab::Exporter::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
+    stub_const("GitLab::Exporter::Database::CiBuildsCollector::EE_CHECK_QUERY", ee_check_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 7.0.3
+  version: 7.1.1
 platform: ruby
 authors:
 - Pablo Carranza
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
@@ -191,7 +191,7 @@ homepage: http://gitlab.com
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -207,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: GitLab metrics exporter
 test_files: