gitlab-exporter 7.0.4 → 7.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 871a04f90ba88d0584d1378778d84fa495e3d258824ca5af41c89eddcc7f6d68
-  data.tar.gz: d394f2f50050049ac4d1156ee6bfb79f5fcf422980989599afabf3eec4f2b210
+  metadata.gz: 350884c8d1f03a58cad2630e0076aa641d22d60aed65a5d2342cb7f4508ec655
+  data.tar.gz: d1857b8e6a8f842b438d12810c110ff4b8bbdf5226b5a6c5859dd949c12d1dd6
 SHA512:
-  metadata.gz: 8f2fa49fd5423c9fcf0d47512949b8ec93d95b2c1b48e48693fe1ad273af8053cdae42392b48740bc925e4d8e701254f815089a25f1311c9ecdba7e139e5d3f8
-  data.tar.gz: f9e8f5e8d42ba861bd297d0b0b9caae6b0729007ac3674fd637be7d091c8ca300cb0ff8cf6b084a07a7903e479381e2d58dceb770dfb829df585fd0d6395b6d8
+  metadata.gz: 9b62aaf4f95b4da82b1e5bc86058b97c363a1b43ae7026589e30da6af47ccf69261ec279094574927180b270be358afbba8304d10bff3fe8fb047fb54b6be538
+  data.tar.gz: 34d27b77c5b04c4a079d184ff30db9b75f37471f9ac0e5199790ce550ee360e52d6f81d2b22b6f159a780ff7525b866b79f8ab0c06265d6251f697177b86fc83

data/.gitlab-ci.yml

@@ -1,17 +1,26 @@
+include:
+  - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+  - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+  - template: Security/License-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+  - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+  - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+
 default:
   image: "ruby:2.3"
-  before_script:
-    - git config --global user.email "bot@gitlab.com"
-    - git config --global user.name "Bot User"
-    - bundle install -j $(nproc) --path vendor
   cache:
     paths:
       - vendor
   tags:
     - gitlab-org
 
+.before_scripts: &before_scripts
+  - git config --global user.email "bot@gitlab.com"
+  - git config --global user.name "Bot User"
+  - bundle install -j $(nproc) --path vendor
+
 workflow:
-  rules:
+  rules: &workflow_rules
     # For merge requests, create a pipeline.
     - if: '$CI_MERGE_REQUEST_IID'
     # For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
@@ -22,7 +31,18 @@ workflow:
 rspec:
   script:
     - bundle exec rspec spec -f d -c
+  before_script: *before_scripts
 
 rubocop:
   script:
     - bundle exec rubocop
+  before_script: *before_scripts
+
+license_scanning:
+  rules: *workflow_rules
+
+gemnasium-dependency_scanning:
+  rules: *workflow_rules
+
+secret_detection:
+  rules: *workflow_rules

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (7.0.4)
+    gitlab-exporter (7.1.2)
       connection_pool (~> 2.2.1)
       pg (~> 1.1)
       quantile (~> 0.2.0)
@@ -14,7 +14,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.0)
-    connection_pool (2.2.2)
+    connection_pool (2.2.3)
     diff-lcs (1.3)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
@@ -23,7 +23,7 @@ GEM
     pg (1.2.3)
     powerpack (0.1.1)
     quantile (0.2.1)
-    rack (2.0.9)
+    rack (2.2.3)
     rack-protection (2.0.8.1)
       rack
     rainbow (2.1.0)
@@ -51,11 +51,11 @@ GEM
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.8.1)
     ruby2_keywords (0.0.2)
-    sidekiq (5.2.8)
+    sidekiq (5.2.9)
       connection_pool (~> 2.2, >= 2.2.2)
-      rack (< 2.1.0)
+      rack (~> 2.0)
       rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 5)
+      redis (>= 3.3.5, < 4.2)
     sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)

data/lib/gitlab_exporter/database/base.rb

@@ -16,8 +16,9 @@ module GitLab
           end
         end
 
-        def initialize(args)
+        def initialize(args, logger: nil)
           @connection_string = args[:connection_string]
+          @logger = logger
         end
 
         def run
@@ -33,10 +34,14 @@ module GitLab
             begin
               yield conn
             rescue PG::UnableToSend => e
+              @logger.warn "Error sending to the database: #{e}" if @logger
               conn.reset
               raise e
             end
           end
+        rescue PG::Error => e
+          @logger.error "Error connecting to the database: #{e}" if @logger
+          raise e
         end
       end
     end

data/lib/gitlab_exporter/database/bloat.rb

@@ -3,6 +3,8 @@ module GitLab
     module Database
       # Helper to collect bloat metrics.
       class BloatCollector < Base
+        attr_writer :logger
+
         def run(type = :btree)
           execute(self.class.query_for(type)).each_with_object({}) do |row, h|
             h[row["object_name"]] = row
@@ -39,9 +41,11 @@ module GitLab
 
         def initialize(opts,
                        metrics: PrometheusMetrics.new,
-                       collector: BloatCollector.new(connection_string: opts[:connection_string]))
+                       collector: BloatCollector.new(connection_string: opts[:connection_string]),
+                       logger: nil)
           @metrics = metrics
           @collector = collector
+          @collector.logger = logger
           @bloat_types = opts[:bloat_types] || %i(btree table)
         end
 

data/lib/gitlab_exporter/database/ci_builds.rb

@@ -21,8 +21,7 @@ module GitLab
               ON namespaces.id = projects.namespace_id
             LEFT JOIN namespace_statistics
               ON namespace_statistics.namespace_id = namespaces.id
-            JOIN application_settings
-              ON application_settings.id = 1
+            JOIN application_settings ON (TRUE)
             WHERE ci_builds.type = 'Ci::Build'
               AND ci_builds.status = '%s'
               -- The created_at filter has been introduced for performance reasons only
@@ -95,8 +94,7 @@ module GitLab
               ON namespaces.id = projects.namespace_id
             LEFT JOIN namespace_statistics
               ON namespace_statistics.namespace_id = namespaces.id
-            JOIN application_settings
-              ON application_settings.id = 1
+            JOIN application_settings ON (TRUE)
             WHERE ci_builds.type = 'Ci::Build'
               AND ci_builds.status = 'running'
               -- The created_at filter has been introduced for performance reasons only
@@ -144,9 +142,9 @@ module GitLab
               ci_builds.trigger_request_id
           SQL
 
-        MIRROR_COLUMN_QUERY =
+        EE_CHECK_QUERY =
           <<~SQL.freeze
-            SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='projects' AND column_name='mirror')
+            SELECT COUNT(*) FROM licenses
           SQL
 
         REPEATED_COMMANDS_QUERY_EE =
@@ -175,8 +173,7 @@ module GitLab
                 ON namespaces.id = projects.namespace_id
               LEFT JOIN namespace_statistics
                 ON namespace_statistics.namespace_id = namespaces.id
-              JOIN application_settings
-                ON application_settings.id = 1
+              JOIN application_settings ON (TRUE)
               WHERE ci_builds.type = 'Ci::Build'
                 AND ci_builds.status IN ('running', 'pending')
                 -- The created_at filter has been introduced for performance reasons only
@@ -265,8 +262,8 @@ module GitLab
 
         DEFAULT_UNARCHIVED_TRACES_OFFSET_MINUTES = 1440
 
-        def initialize(opts)
-          super(opts)
+        def initialize(opts, logger: nil)
+          super(opts, logger: logger)
 
           @allowed_repeated_commands_count = opts[:allowed_repeated_commands_count]
           @created_builds_counting_disabled = opts[:created_builds_counting_disabled]
@@ -289,7 +286,7 @@ module GitLab
         def builds(status)
           results = []
 
-          query = mirror_column? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
+          query = ee? ? BUILDS_QUERY_EE : BUILDS_QUERY_CE
           query = query % [status] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_builds_row_to_values(row)
@@ -318,7 +315,7 @@ module GitLab
         def per_runner_builds
           results = []
 
-          query = mirror_column? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
+          query = ee? ? PER_RUNNER_QUERY_EE : PER_RUNNER_QUERY_CE
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_per_runners_builds_row_to_values(row)
           end
@@ -341,7 +338,7 @@ module GitLab
         def repeated_commands
           results = []
 
-          query = mirror_column? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
+          query = ee? ? REPEATED_COMMANDS_QUERY_EE : REPEATED_COMMANDS_QUERY_CE
           query = query % [allowed_repeated_commands_count] # rubocop:disable Style/FormatString
           exec_query_with_custom_random_page_cost(query).each do |row|
             results << transform_repeated_commands_row_to_values(row)
@@ -406,11 +403,11 @@ module GitLab
           end
         end
 
-        def mirror_column?
-          @mirror_column ||=
+        def ee?
+          @ee ||=
             begin
               with_connection_pool do |conn|
-                conn.exec(MIRROR_COLUMN_QUERY)[0]["exists"] == "t"
+                conn.exec(EE_CHECK_QUERY)[0]["count"].to_i > 0 # rubocop:disable Style/NumericPredicate
               end
             rescue PG::UndefinedColumn
               false
@@ -420,14 +417,14 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class CiBuildsProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
           @metrics = metrics
 
           collector_opts = { connection_string: opts[:connection_string],
                              allowed_repeated_commands_count: opts[:allowed_repeated_commands_count],
                              created_builds_counting_disabled: opts[:created_builds_counting_disabled],
                              unarchived_traces_offset_minutes: opts[:unarchived_traces_offset_minutes] }
-          @collector = CiBuildsCollector.new(collector_opts)
+          @collector = CiBuildsCollector.new(collector_opts, logger: logger)
         end
 
         def probe_db

data/lib/gitlab_exporter/database/remote_mirrors.rb

@@ -35,7 +35,7 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class RemoteMirrorsProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
           @metrics = metrics
           @collector = RemoteMirrorsCollector.new(
             connection_string: opts[:connection_string],

data/lib/gitlab_exporter/database/row_count.rb

@@ -175,6 +175,14 @@ module GitLab
 
         def execute(query)
           with_connection_pool do |conn|
+            conn.exec(query).map_types!(type_map_for_results(conn))
+          end
+        rescue PG::UndefinedTable, PG::UndefinedColumn
+          nil
+        end
+
+        def type_map_for_results(conn)
+          @type_map_for_results ||= begin
             tm = PG::BasicTypeMapForResults.new(conn)
 
             # Remove warning message:
@@ -187,10 +195,8 @@ module GitLab
               tm.add_coder(old_coder.dup.tap { |c| c.oid = value[:oid] })
             end
 
-            conn.exec(query).map_types!(tm)
+            tm
           end
-        rescue PG::UndefinedTable, PG::UndefinedColumn
-          nil
         end
 
         # Not private so I can test it without meta programming tricks
@@ -212,7 +218,7 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class RowCountProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
           @metrics = metrics
           @collector = RowCountCollector.new(
             connection_string: opts[:connection_string],

data/lib/gitlab_exporter/database/tuple_stats.rb

@@ -25,9 +25,9 @@ module GitLab
 
       # Probes the DB specified by opts[:connection_string] for tuple stats, then converts them to metrics
       class TuplesProber
-        def initialize(opts, metrics: PrometheusMetrics.new)
+        def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
           @metrics   = metrics
-          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string])
+          @collector = TupleStatsCollector.new(connection_string: opts[:connection_string], logger: logger)
         end
 
         def probe_db

data/lib/gitlab_exporter/git.rb

@@ -62,7 +62,7 @@ module GitLab
     # Optionally takes a metrics object which by default is a PrometheusMetrics, useful to change the
     # metrics writer to something else.
     class GitProber
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @metrics = metrics
         @labels = opts[:labels] || {}
         @git = Git.new(opts[:source])
@@ -85,7 +85,7 @@ module GitLab
 
     # A special prober for git processes
     class GitProcessProber
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @opts    = opts
         @metrics = metrics
       end

data/lib/gitlab_exporter/prober.rb

@@ -2,9 +2,10 @@ module GitLab
   module Exporter
     # A class to combine multiple probers into one
     class Prober
-      def initialize(prober_opts, metrics: PrometheusMetrics.new)
+      def initialize(prober_opts, metrics: PrometheusMetrics.new, logger: nil)
         @prober_opts = prober_opts
         @metrics     = metrics
+        @logger      = logger
 
         resolve_prober_classes
       end
@@ -12,7 +13,7 @@ module GitLab
       def probe_all
         @prober_opts.each do |_probe_name, params|
           Utils.wrap_in_array(params[:opts]).each do |opts|
-            prober = params[:class].new(opts, metrics: @metrics)
+            prober = params[:class].new(opts, metrics: @metrics, logger: @logger)
             params[:methods].each do |meth|
               prober.send(meth)
             end

data/lib/gitlab_exporter/process.rb

@@ -59,7 +59,7 @@ module GitLab
 
     # Probes a process for info then writes metrics to a target
     class ProcessProber
-      def initialize(options, metrics: PrometheusMetrics.new)
+      def initialize(options, metrics: PrometheusMetrics.new, logger: nil) # rubocop:disable Lint/UnusedMethodArgument
         @metrics = metrics
         @name    = options[:name]
         @pids    = if options[:pid_or_pattern] =~ /^\d+$/

data/lib/gitlab_exporter/sidekiq.rb

@@ -10,9 +10,10 @@ module GitLab
       QUEUE_JOB_STATS_SCRIPT = File.read(File.expand_path("#{__FILE__}/../sidekiq_queue_job_stats.lua")).freeze
       QUEUE_JOB_STATS_SHA    = Digest::SHA1.hexdigest(QUEUE_JOB_STATS_SCRIPT).freeze
 
-      def initialize(opts, metrics: PrometheusMetrics.new)
+      def initialize(opts, metrics: PrometheusMetrics.new, logger: nil)
         @opts    = opts
         @metrics = metrics
+        @logger  = logger
 
         Sidekiq.configure_client do |config|
           config.redis = redis_options

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "7.0.4".freeze
+    VERSION = "7.1.2".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -42,6 +42,11 @@ module GitLab
 
           memory_threshold = (config[:server] && config[:server][:memory_threshold]) || 1024
           use MemoryKillerMiddleware, memory_threshold
+          use Rack::Logger
+        end
+
+        def logger
+          request.logger
         end
 
         def setup_server(config)
@@ -62,7 +67,7 @@ module GitLab
 
             get "/#{probe_name}" do
               content_type "text/plain; version=0.0.4"
-              prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false))
+              prober = Prober.new(opts, metrics: PrometheusMetrics.new(include_timestamp: false), logger: logger)
 
               prober.probe_all
               prober.write_to(response)

data/spec/database/bloat_spec.rb

@@ -43,7 +43,11 @@ describe GitLab::Exporter::Database::BloatProber do
   end
 
   describe "#probe_db" do
-    subject { described_class.new(opts, metrics: metrics, collector: collector).probe_db }
+    subject { described_class.new(opts, metrics: metrics, collector: collector, logger: STDOUT).probe_db }
+
+    before do
+      expect(collector).to receive(:logger=).with(STDOUT)
+    end
 
     it "invokes the collector for each bloat type" do
       expect(collector).to receive(:run).with(:btree)

data/spec/database/ci_builds_spec.rb

@@ -9,7 +9,7 @@ describe GitLab::Exporter::Database do
   let(:stale_builds_query) { "SELECT NOT UPDATED RUNNING" }
   let(:per_runner_query_ee) { "SELECT ALL RUNNING PER RUNNER EE" }
   let(:per_runner_query_ce) { "SELECT ALL RUNNING PER RUNNER CE" }
-  let(:mirror_column_query) { "SELECT DOES MIRROR COLUMN EXISTS" }
+  let(:ee_check_query) { "SELECT COUNT(*) FROM licenses" }
   let(:repeated_commands_query_ee) { "SELECT EE REPEATED COMNANDS %d" }
   let(:repeated_commands_query_ce) { "SELECT CE REPEATED COMNANDS %d" }
   let(:unarchived_traces_query) { "SELECT UNARCHIVED TRACES %s LIST" }
@@ -22,11 +22,11 @@ describe GitLab::Exporter::Database do
   let(:unarchived_traces_offset_minutes) { 60 }
 
   def stub_ee
-    allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "t" }])
+    allow(connection).to receive(:exec).with(ee_check_query).and_return([{ "count" => 1 }])
   end
 
   def stub_ce
-    allow(connection).to receive(:exec).with(mirror_column_query).and_return([{ "exists" => "f" }])
+    allow(connection).to receive(:exec).with(ee_check_query).and_return([{ "count" => 0 }])
   end
 
   def builds_query_row_ee(shared_runners_enabled, status, namespace_id, has_minutes, count)
@@ -86,7 +86,7 @@ describe GitLab::Exporter::Database do
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::STALE_BUILDS_QUERY", stale_builds_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_EE", per_runner_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::PER_RUNNER_QUERY_CE", per_runner_query_ce)
-    stub_const("GitLab::Exporter::Database::CiBuildsCollector::MIRROR_COLUMN_QUERY", mirror_column_query)
+    stub_const("GitLab::Exporter::Database::CiBuildsCollector::EE_CHECK_QUERY", ee_check_query)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_EE", repeated_commands_query_ee)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::REPEATED_COMMANDS_QUERY_CE", repeated_commands_query_ce)
     stub_const("GitLab::Exporter::Database::CiBuildsCollector::UNARCHIVED_TRACES_QUERY", unarchived_traces_query)

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 7.0.4
+  version: 7.1.2
 platform: ruby
 authors:
 - Pablo Carranza
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
@@ -191,7 +191,7 @@ homepage: http://gitlab.com
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -207,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: GitLab metrics exporter
 test_files: