gitlab-exporter 13.5.0 → 14.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71244a8992a3116e4513746514842e6d6f0c53abbef7e0a7435a3889acce66c1
-  data.tar.gz: fc013ec18e7f7a11e3d5da3a1375fe80add92f96da1f572eb8b577ee98bedebb
+  metadata.gz: dd25ebf8533fde9c59b0db768540fcbdf338c961715617459c2a4aa2eb2cc912
+  data.tar.gz: 12e190d1ea1f1cf0a16aa68ab54995c4953555807c7e3c80ecf8a40072d811dd
 SHA512:
-  metadata.gz: b4d3c84092ca842c13123bb6573118adc44b60f090594db51b559bc20d2718c1e8a341494fd6ba376e6a30ce07985b2ae88923e7a68760a631936f3e962ed62f
-  data.tar.gz: eb579c03705bdbd4d64b5c0a5b6dc40cbfa0125939b03d250d6efe44f395fe55effabbf3733ab57a0297494530ce8efc5b3521eda12c930b7ec8ffeadc8fb7b0
+  metadata.gz: f482931b5c30361f7d9e2899c47de30e4fe296b02028818c83bf189defcd8e0dd6fa30f411c1a96e2abab809c1d45ea4c679479b61ac093c09d678154a8b6ce5
+  data.tar.gz: 5942a5b4b0ea8ecdccdab0de8a5bfdad2f7b1f125e200bfc94e64cf4328ca750cf866ccd394dbe14dfc91fe3dcc5915332e5519c9f9d37682f49251ac2a7b581

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (13.5.0)
+    gitlab-exporter (14.1.0)
       connection_pool (= 2.2.5)
+      deep_merge (~> 1.2.2)
       faraday (~> 1.8.0)
       pg (= 1.5.3)
       puma (= 5.6.5)
@@ -18,6 +19,7 @@ GEM
   specs:
     ast (2.4.1)
     connection_pool (2.2.5)
+    deep_merge (1.2.2)
     diff-lcs (1.5.0)
     faraday (1.8.0)
       faraday-em_http (~> 1.0)
@@ -107,4 +109,4 @@ DEPENDENCIES
   rubocop (~> 0.42)
 
 BUNDLED WITH
-   2.4.5
+   2.4.20

data/README.md

@@ -95,6 +95,25 @@ Once running, you can point your browser or curl to the following URLs:
 * http://localhost:9168/sidekiq
 * http://localhost:9168/metrics (to get all of the above combined)
 
+If it is undesirable to have secrets (e.g. Redis password or PostgreSQL credentials) in the config file,
+then you can specify an external command (with the `--extra-config-command <command>` flag) that outputs
+such credentials in a YAML form (same structure as the config file) and it will be merged with the file configuration.
+
+For example:
+
+```
+$ vault kv get -format=yaml secrets/gitlab/gitlab-exporter
+probes:
+  sidekiq:
+    opts:
+      redis_url: redis://hunter1@localhost:9765
+  database:
+    opts:
+      connection_string: postgres://db-admin:hunter1@localhost:6543/main-db
+
+$ bin/gitlab-exporter web -c config/gitlab-exporter.yml --extra-config-command "vault kv get -format=yaml secrets/gitlab/gitlab-exporter"
+```
+
 ### Database Bloat Metrics
 
 Database bloat is measured for indexes (`btree`) and/or tables

data/config/gitlab-exporter.yml.example

@@ -95,8 +95,6 @@ probes:
     opts:
       redis_url: "redis://localhost:6379"
       redis_enable_client: true
-      probe_namespaced: true
-      probe_non_namespaced: true
 
   ruby: &ruby
     class_name: RubyProber

data/gitlab-exporter.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.license       = "MIT"
 
   s.add_runtime_dependency "connection_pool", "2.2.5"
+  s.add_runtime_dependency "deep_merge", "~> 1.2.2"
   s.add_runtime_dependency "faraday", "~> 1.8.0"
   s.add_runtime_dependency "pg", "1.5.3"
   s.add_runtime_dependency "puma", "5.6.5"

data/lib/gitlab_exporter/cli.rb

@@ -1,4 +1,6 @@
 require "yaml"
+require "deep_merge"
+require "English"
 
 # TODO: Remove this once we're on Ruby 3
 # https://gitlab.com/gitlab-org/gitlab/-/issues/393651
@@ -188,6 +190,10 @@ module GitLab
             opts.on("-c config.yml", "Monitoring config") do |val|
               @config_file = val
             end
+            opts.on("--extra-config-command command", "Shell command that returns YAML config to be merged
+                    with the config file") do |val|
+              @extra_config_command = val
+            end
           end
         end
 
@@ -195,12 +201,17 @@ module GitLab
           @options.help
         end
 
+        def merged_config
+          config_from_file = Utils.deep_symbolize_hash_keys(YAML.safe_load_file(@config_file, aliases: true))
+          config_from_command = extra_config_from_command
+
+          config_from_file.deep_merge!(config_from_command)
+        end
+
         def run
           validate!
 
-          config = Utils.deep_symbolize_hash_keys(YAML.safe_load_file(@config_file, aliases: true))
-
-          WebExporter.setup(config)
+          WebExporter.setup(merged_config)
           WebExporter.run!
         end
 
@@ -209,6 +220,22 @@ module GitLab
         def validate!
           fail InvalidCLICommand.new(help) unless @config_file
         end
+
+        def run_extra_config_command
+          puts "Using extra config by running command `#{@extra_config_command}`"
+
+          output = `#{@extra_config_command}`
+          return output if $CHILD_STATUS == 0
+
+          puts "ERROR: command `#{@extra_config_command}` returned a non-zero code."
+          exit(2)
+        end
+
+        def extra_config_from_command
+          return {} unless @extra_config_command
+
+          Utils.deep_symbolize_hash_keys(YAML.safe_load(run_extra_config_command))
+        end
       end
 
       # Process runner

data/lib/gitlab_exporter/sidekiq.rb

@@ -7,7 +7,6 @@ module GitLab
     # A prober for Sidekiq queues
     #
     # It takes the Redis URL Sidekiq is connected to
-    # rubocop:disable Metrics/ClassLength
     class SidekiqProber
       # The maximum depth (from the head) of each queue to probe. Probing the
       # entirety of a very large queue will take longer and run the risk of
@@ -41,24 +40,35 @@ module GitLab
         @opts    = opts
         @metrics = metrics
         @logger  = logger
-
-        @probe_namespaced = !!opts[:probe_namespaced] # rubocop:disable Style/DoubleNegation
-        @probe_non_namespaced = !!opts[:probe_non_namespaced] # rubocop:disable Style/DoubleNegation
-
-        # to maintain backward compatibility if the config is missing values
-        @probe_namespaced = true if opts[:probe_namespaced].nil? && opts[:probe_non_namespaced].nil?
       end
 
       def probe_stats
-        with_sidekiq { _probe_stats } if @probe_namespaced
-        with_sidekiq(false) { _probe_stats(false) } if @probe_non_namespaced
+        with_sidekiq do
+          stats = Sidekiq::Stats.new
+
+          @metrics.add("sidekiq_jobs_processed_total", stats.processed)
+          @metrics.add("sidekiq_jobs_failed_total", stats.failed)
+          @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued)
+          @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size)
+          @metrics.add("sidekiq_jobs_retry_size", stats.retry_size)
+          @metrics.add("sidekiq_jobs_dead_size", stats.dead_size)
+
+          @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency)
+          @metrics.add("sidekiq_processes_size", stats.processes_size)
+          @metrics.add("sidekiq_workers_size", stats.workers_size)
+        end
 
         self
       end
 
       def probe_queues
-        with_sidekiq { _probe_queues } if @probe_namespaced
-        with_sidekiq(false) { _probe_queues(false) } if @probe_non_namespaced
+        with_sidekiq do
+          Sidekiq::Queue.all.each do |queue|
+            @metrics.add("sidekiq_queue_size", queue.size, name: queue.name)
+            @metrics.add("sidekiq_queue_latency_seconds", queue.latency, name: queue.name)
+            @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, name: queue.name)
+          end
+        end
 
         self
       end
@@ -71,8 +81,24 @@ module GitLab
       end
 
       def probe_future_sets
-        with_sidekiq { _probe_future_sets } if @probe_namespaced
-        with_sidekiq(false) { _probe_future_sets(false) } if @probe_non_namespaced
+        now = Time.now.to_f
+        with_sidekiq do
+          Sidekiq.redis do |conn|
+            Sidekiq::Scheduled::SETS.each do |set|
+              # Default to 0; if all jobs are due in the future, there is no "negative" delay.
+              delay = 0
+
+              _job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
+              delay = now - timestamp if timestamp
+
+              @metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay)
+
+              # zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
+              @metrics.add("sidekiq_#{set}_set_backlog_count",
+                           conn.zcount(set, "-inf", now.to_s))
+            end
+          end
+        end
       end
 
       # Count worker classes present in Sidekiq queues. This only looks at the
@@ -82,22 +108,57 @@ module GitLab
       # will not have completely accurate statistics, but the probe performance
       # will also not degrade as the queue gets larger.
       def probe_jobs_limit
-        with_sidekiq { _probe_jobs_limit } if @probe_namespaced
-        with_sidekiq(false) { _probe_jobs_limit(false) } if @probe_non_namespaced
+        with_sidekiq do
+          job_stats = Hash.new(0)
+
+          Sidekiq::Queue.all.each do |queue|
+            Sidekiq.redis do |conn|
+              conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
+                job_class = Sidekiq.load_json(job)["class"]
+
+                job_stats[job_class] += 1
+              end
+            end
+          end
+
+          job_stats.each do |class_name, count|
+            @metrics.add("sidekiq_enqueued_jobs", count, name: class_name)
+          end
+        end
 
         self
       end
 
       def probe_workers
-        with_sidekiq { _probe_workers } if @probe_namespaced
-        with_sidekiq(false) { _probe_workers(false) } if @probe_non_namespaced
+        with_sidekiq do
+          worker_stats = Hash.new(0)
+
+          Sidekiq::Workers.new.map do |_pid, _tid, work|
+            job_klass = work["payload"]["class"]
+
+            worker_stats[job_klass] += 1
+          end
+
+          worker_stats.each do |class_name, count|
+            @metrics.add("sidekiq_running_jobs", count, name: class_name)
+          end
+        end
 
         self
       end
 
       def probe_retries
-        with_sidekiq { _probe_retries } if @probe_namespaced
-        with_sidekiq(false) { _probe_retries(false) } if @probe_non_namespaced
+        with_sidekiq do
+          retry_stats = Hash.new(0)
+
+          Sidekiq::RetrySet.new.map do |job|
+            retry_stats[job.klass] += 1
+          end
+
+          retry_stats.each do |class_name, count|
+            @metrics.add("sidekiq_to_be_retried_jobs", count, name: class_name)
+          end
+        end
 
         self
       end
@@ -119,104 +180,10 @@ module GitLab
 
       private
 
-      def _probe_workers(namespace = true)
-        labels = add_namespace_labels? ? { namespaced: namespace } : {}
-        worker_stats = Hash.new(0)
-
-        Sidekiq::Workers.new.map do |_pid, _tid, work|
-          job_klass = work["payload"]["class"]
-
-          worker_stats[job_klass] += 1
-        end
-
-        worker_stats.each do |class_name, count|
-          @metrics.add("sidekiq_running_jobs", count, **labels.merge({ name: class_name }))
-        end
-      end
-
-      def _probe_future_sets(namespace = true)
-        labels = add_namespace_labels? ? { namespaced: namespace } : {}
-        now = Time.now.to_f
-        Sidekiq.redis do |conn|
-          Sidekiq::Scheduled::SETS.each do |set|
-            # Default to 0; if all jobs are due in the future, there is no "negative" delay.
-            delay = 0
-
-            _job, timestamp = conn.zrangebyscore(set, "-inf", now.to_s, limit: [0, 1], withscores: true).first
-            delay = now - timestamp if timestamp
-
-            @metrics.add("sidekiq_#{set}_set_processing_delay_seconds", delay, **labels)
-
-            # zcount is O(log(N)) (prob. binary search), so is still quick even with large sets
-            @metrics.add("sidekiq_#{set}_set_backlog_count",
-                         conn.zcount(set, "-inf", now.to_s), **labels)
-          end
-        end
-      end
-
-      def _probe_stats(namespace = true)
-        stats = Sidekiq::Stats.new
-        labels = add_namespace_labels? ? { namespaced: namespace } : {}
-
-        @metrics.add("sidekiq_jobs_processed_total", stats.processed, **labels)
-        @metrics.add("sidekiq_jobs_failed_total", stats.failed, **labels)
-        @metrics.add("sidekiq_jobs_enqueued_size", stats.enqueued, **labels)
-        @metrics.add("sidekiq_jobs_scheduled_size", stats.scheduled_size, **labels)
-        @metrics.add("sidekiq_jobs_retry_size", stats.retry_size, **labels)
-        @metrics.add("sidekiq_jobs_dead_size", stats.dead_size, **labels)
-
-        @metrics.add("sidekiq_default_queue_latency_seconds", stats.default_queue_latency, **labels)
-        @metrics.add("sidekiq_processes_size", stats.processes_size, **labels)
-        @metrics.add("sidekiq_workers_size", stats.workers_size, **labels)
-      end
-
-      def _probe_queues(namespace = true)
-        Sidekiq::Queue.all.each do |queue|
-          labels = { name: queue.name }
-          labels[:namespaced] = namespace if add_namespace_labels?
-
-          @metrics.add("sidekiq_queue_size", queue.size, **labels)
-          @metrics.add("sidekiq_queue_latency_seconds", queue.latency, **labels)
-          @metrics.add("sidekiq_queue_paused", queue.paused? ? 1 : 0, **labels)
-        end
-      end
-
-      def _probe_jobs_limit(namespace = true)
-        labels = add_namespace_labels? ? { namespaced: namespace } : {}
-        job_stats = Hash.new(0)
-
-        Sidekiq::Queue.all.each do |queue|
-          Sidekiq.redis do |conn|
-            conn.lrange("queue:#{queue.name}", 0, PROBE_JOBS_LIMIT).each do |job|
-              job_class = Sidekiq.load_json(job)["class"]
-
-              job_stats[job_class] += 1
-            end
-          end
-        end
-
-        job_stats.each do |class_name, count|
-          @metrics.add("sidekiq_enqueued_jobs", count, **labels.merge({ name: class_name }))
-        end
-      end
-
-      def _probe_retries(namespace = true)
-        labels = add_namespace_labels? ? { namespaced: namespace } : {}
-        retry_stats = Hash.new(0)
-
-        Sidekiq::RetrySet.new.map do |job|
-          retry_stats[job.klass] += 1
-        end
-
-        retry_stats.each do |class_name, count|
-          @metrics.add("sidekiq_to_be_retried_jobs", count, **labels.merge({ name: class_name }))
-        end
-      end
-
-      def with_sidekiq(namespaced = true)
+      def with_sidekiq
         SIDEKIQ_REDIS_LOCK.synchronize {
           Sidekiq.configure_client do |config|
-            config.redis = self.class.connection_pool[redis_options(namespaced)]
+            config.redis = self.class.connection_pool[redis_options]
           end
 
           return unless connected?
@@ -225,14 +192,13 @@ module GitLab
         }
       end
 
-      def redis_options(namespaced = true)
+      def redis_options
         options = {
           url: @opts[:redis_url],
           connect_timeout: 1,
           reconnect_attempts: 0
         }
 
-        options[:namespace] = "resque:gitlab" if namespaced
         options[:id] = nil unless redis_enable_client?
         options
       end
@@ -253,11 +219,6 @@ module GitLab
         @logger&.error "Error connecting to the Redis: #{e}"
         @connected = false
       end
-
-      def add_namespace_labels?
-        @probe_namespaced && @probe_non_namespaced
-      end
     end
-    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "13.5.0".freeze
+    VERSION = "14.1.0".freeze
   end
 end

data/spec/cli_spec.rb

@@ -1,5 +1,6 @@
 require "spec_helper"
 require "gitlab_exporter/cli"
+require "optparse"
 
 context "With valid pair of repositories" do
   let(:repos) { GitRepoBuilder.new }
@@ -29,3 +30,49 @@ context "With valid pair of repositories" do
     end
   end
 end
+
+describe GitLab::Exporter::CLI::Server do
+  context "extra-config-command is passed" do
+    let(:argv) do
+      ["-c", "spec/fixtures/config.yml", "--extra-config-command", extra_config_command]
+        .extend(OptionParser::Arguable)
+    end
+    let(:extra_config_command) { "vault kv get top-secrets" }
+    let(:server_cmd) { described_class.new(argv) }
+
+    before do
+      allow(server_cmd).to receive(:run_extra_config_command).and_return(<<~YAML
+        probes:
+          sidekiq:
+            methods:
+              - probe_workers
+            opts:
+              redis_url: redis://hunter1@localhost:9765
+          database:
+            opts:
+              connection_string: postgres://db-admin:hunter1@localhost:6543/main-db
+      YAML
+                                                                        )
+    end
+
+    it "merges the returned YAML from the command with the passed config" do
+      expect(server_cmd.merged_config).to eq({
+                                               probes: {
+                                                 sidekiq: {
+                                                   methods: %w[probe_stats probe_workers],
+                                                   opts: {
+                                                     redis_url: "redis://hunter1@localhost:9765"
+                                                   }
+                                                 },
+                                                 database: {
+                                                   methods: ["probe_db"],
+                                                   opts: {
+                                                     connection_string:
+                                                      "postgres://db-admin:hunter1@localhost:6543/main-db"
+                                                   }
+                                                 }
+                                               }
+                                             })
+    end
+  end
+end

data/spec/fixtures/config.yml

@@ -0,0 +1,10 @@
+probes:
+  sidekiq:
+    methods:
+      - probe_stats
+
+  database:
+    methods:
+      - probe_db
+    opts:
+      connection_string: to-be-overridden

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 13.5.0
+  version: 14.1.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -24,6 +24,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.2.5
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -229,6 +243,7 @@ files:
 - spec/database/ci_builds_spec.rb
 - spec/database/row_count_spec.rb
 - spec/elasticsearch_spec.rb
+- spec/fixtures/config.yml
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
@@ -236,7 +251,6 @@ files:
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb
-- spec/sidekiq_spec.rb
 - spec/spec_helper.rb
 - spec/util_spec.rb
 homepage: https://gitlab.com/gitlab-org/ruby/gems/gitlab-exporter
@@ -268,6 +282,7 @@ test_files:
 - spec/database/ci_builds_spec.rb
 - spec/database/row_count_spec.rb
 - spec/elasticsearch_spec.rb
+- spec/fixtures/config.yml
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
@@ -275,6 +290,5 @@ test_files:
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb
-- spec/sidekiq_spec.rb
 - spec/spec_helper.rb
 - spec/util_spec.rb

data/spec/sidekiq_spec.rb

@@ -1,64 +0,0 @@
-require "spec_helper"
-require "gitlab_exporter/sidekiq"
-
-describe GitLab::Exporter::SidekiqProber do
-  let(:probe_list) { %w[probe_stats probe_queues probe_future_sets probe_jobs_limit probe_workers probe_retries] }
-  let(:opt) { {} }
-  let(:probe) { described_class.new(**opt) }
-
-  before do
-    # stub connection as we want check if the probe methods are called
-    # with the right namespace argument
-    allow(probe).to receive(:connected?).and_return(true)
-  end
-
-  context "when no namespace options are defined" do
-    it "defaults to probing non-namespaced keys only" do
-      probe_list.each do |probe_type|
-        expect(probe).not_to receive("_#{probe_type}".to_sym).with(false)
-        expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
-
-        probe.send(probe_type)
-      end
-    end
-  end
-
-  context "when probing both namespaces" do
-    let(:opt) { { probe_non_namespaced: true, probe_namespaced: true } }
-
-    it "probes both namespaces" do
-      probe_list.each do |probe_type|
-        expect(probe).to receive("_#{probe_type}".to_sym).with(false)
-        expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
-
-        probe.send(probe_type)
-      end
-    end
-  end
-
-  context "when probing non-namespaced only" do
-    let(:opt) { { probe_non_namespaced: true } }
-
-    it "probes non-namespaced only" do
-      probe_list.each do |probe_type|
-        expect(probe).to receive("_#{probe_type}".to_sym).with(false)
-        expect(probe).not_to receive("_#{probe_type}".to_sym).with(no_args)
-
-        probe.send(probe_type)
-      end
-    end
-  end
-
-  context "when probing namespace only" do
-    let(:opt) { { probe_namespaced: true } }
-
-    it "probes namespaced only" do
-      probe_list.each do |probe_type|
-        expect(probe).not_to receive("_#{probe_type}".to_sym).with(false)
-        expect(probe).to receive("_#{probe_type}".to_sym).with(no_args)
-
-        probe.send(probe_type)
-      end
-    end
-  end
-end