gitlab-exporter 14.0.0 → 14.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6420230ed0ec27d530fa71e46c0776ea2cb21b036a198335229fc304dcdfe3e0
-  data.tar.gz: 91205ee62a32b2e00f7978f9fc7ec063be203d593cade97ea481f85ee1388bbd
+  metadata.gz: b7f9898831930837f37459941f701115d678149f1e0ccdc4dfa896ed729066db
+  data.tar.gz: b7d4a3e5310b838cbefc49475eb7893749a4fc6b6e8574b6f7d03f080790ab8b
 SHA512:
-  metadata.gz: 3be9966a1abe20b42169a4a85e20815649c87e46a31f853002dd5fb0eaff6437a692f5904334d42172fb1950627b0d3bba31a67c6992e5533051f367ed74144f
-  data.tar.gz: 20aefc5c04ee63033feeb80e986e6f239a711d9d88197859c154fe4e429573344d52d5c58ca9d9e611dace90804da5fba8341ed45ed5f592e33a5df8c3c6cc7e
+  metadata.gz: 87701dd407f197bfe6e4927f66e494bfd63aafea8c49eef7d5e86996256853b671f2d8a891b709543f89aead3f3a0609b453c104b826cf0ea8aaf7b586d8d6b7
+  data.tar.gz: e6bcf77ffc7c6b14d6f472fcb0d7c386c676a217021200b75898dbac15e8586f852782ee3e71b3fa010811263b3b01e65417a3d24a01f089db3f931f167c76ec

data/.gitlab-ci.yml

@@ -1,13 +1,10 @@
 include:
+  - component: gitlab.com/components/container-scanning/container-scanning@~latest
+  - component: gitlab.com/components/sast/sast@~latest
+  - component: gitlab.com/components/secret-detection/secret-detection@~latest
+  - component: gitlab.com/gitlab-org/components/gem-release/gem-release@~latest
   - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
-  - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
   - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
-  - template: Security/License-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
-  - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
-  - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
-  - project: 'gitlab-org/quality/pipeline-common'
-    file:
-      - '/ci/gem-release.yml'
 
 variables:
   RUBY_VERSION: "2.7"
@@ -65,9 +62,6 @@ rubocop:
     - bundle exec rubocop
   before_script: *before_scripts
 
-license_scanning:
-  rules: *workflow_rules
-
 gemnasium-dependency_scanning:
   rules: *workflow_rules
 

data/.rubocop.yml

@@ -33,4 +33,4 @@ Metrics/AbcSize:
   Enabled: false
 
 Metrics/ClassLength:
-  Max: 150
+  Max: 155

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (14.0.0)
+    gitlab-exporter (14.2.0)
       connection_pool (= 2.2.5)
+      deep_merge (~> 1.2.2)
       faraday (~> 1.8.0)
       pg (= 1.5.3)
       puma (= 5.6.5)
@@ -18,6 +19,7 @@ GEM
   specs:
     ast (2.4.1)
     connection_pool (2.2.5)
+    deep_merge (1.2.2)
     diff-lcs (1.5.0)
     faraday (1.8.0)
       faraday-em_http (~> 1.0)

data/README.md

@@ -95,6 +95,25 @@ Once running, you can point your browser or curl to the following URLs:
 * http://localhost:9168/sidekiq
 * http://localhost:9168/metrics (to get all of the above combined)
 
+If it is undesirable to have secrets (e.g. Redis password or PostgreSQL credentials) in the config file,
+then you can specify an external command (with the `--extra-config-command <command>` flag) that outputs
+such credentials in a YAML form (same structure as the config file) and it will be merged with the file configuration.
+
+For example:
+
+```
+$ vault kv get -format=yaml secrets/gitlab/gitlab-exporter
+probes:
+  sidekiq:
+    opts:
+      redis_url: redis://hunter1@localhost:9765
+  database:
+    opts:
+      connection_string: postgres://db-admin:hunter1@localhost:6543/main-db
+
+$ bin/gitlab-exporter web -c config/gitlab-exporter.yml --extra-config-command "vault kv get -format=yaml secrets/gitlab/gitlab-exporter"
+```
+
 ### Database Bloat Metrics
 
 Database bloat is measured for indexes (`btree`) and/or tables

data/config/gitlab-exporter.yml.example

@@ -94,6 +94,9 @@ probes:
       - probe_retries
     opts:
       redis_url: "redis://localhost:6379"
+      # Uncomment if username or password is not part of the redis_url specified above
+      # redis_username: 'redis-username'
+      # redis_password: 'redis-password'
       redis_enable_client: true
 
   ruby: &ruby

data/gitlab-exporter.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.license       = "MIT"
 
   s.add_runtime_dependency "connection_pool", "2.2.5"
+  s.add_runtime_dependency "deep_merge", "~> 1.2.2"
   s.add_runtime_dependency "faraday", "~> 1.8.0"
   s.add_runtime_dependency "pg", "1.5.3"
   s.add_runtime_dependency "puma", "5.6.5"

data/lib/gitlab_exporter/cli.rb

@@ -1,4 +1,6 @@
 require "yaml"
+require "deep_merge"
+require "English"
 
 # TODO: Remove this once we're on Ruby 3
 # https://gitlab.com/gitlab-org/gitlab/-/issues/393651
@@ -188,6 +190,10 @@ module GitLab
             opts.on("-c config.yml", "Monitoring config") do |val|
               @config_file = val
             end
+            opts.on("--extra-config-command command", "Shell command that returns YAML config to be merged
+                    with the config file") do |val|
+              @extra_config_command = val
+            end
           end
         end
 
@@ -195,12 +201,17 @@ module GitLab
           @options.help
         end
 
+        def merged_config
+          config_from_file = Utils.deep_symbolize_hash_keys(YAML.safe_load_file(@config_file, aliases: true))
+          config_from_command = extra_config_from_command
+
+          config_from_file.deep_merge!(config_from_command)
+        end
+
         def run
           validate!
 
-          config = Utils.deep_symbolize_hash_keys(YAML.safe_load_file(@config_file, aliases: true))
-
-          WebExporter.setup(config)
+          WebExporter.setup(merged_config)
           WebExporter.run!
         end
 
@@ -209,6 +220,22 @@ module GitLab
         def validate!
           fail InvalidCLICommand.new(help) unless @config_file
         end
+
+        def run_extra_config_command
+          puts "Using extra config by running command `#{@extra_config_command}`"
+
+          output = `#{@extra_config_command}`
+          return output if $CHILD_STATUS == 0
+
+          puts "ERROR: command `#{@extra_config_command}` returned a non-zero code."
+          exit(2)
+        end
+
+        def extra_config_from_command
+          return {} unless @extra_config_command
+
+          Utils.deep_symbolize_hash_keys(YAML.safe_load(run_extra_config_command))
+        end
       end
 
       # Process runner

data/lib/gitlab_exporter/sidekiq.rb

@@ -199,6 +199,10 @@ module GitLab
           reconnect_attempts: 0
         }
 
+        %i[username password].each do |credential|
+          options[credential] = @opts[:"redis_#{credential}"] if @opts.key?(:"redis_#{credential}")
+        end
+
         options[:id] = nil unless redis_enable_client?
         options
       end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "14.0.0".freeze
+    VERSION = "14.2.0".freeze
   end
 end

data/spec/cli_spec.rb

@@ -1,5 +1,6 @@
 require "spec_helper"
 require "gitlab_exporter/cli"
+require "optparse"
 
 context "With valid pair of repositories" do
   let(:repos) { GitRepoBuilder.new }
@@ -29,3 +30,49 @@ context "With valid pair of repositories" do
     end
   end
 end
+
+describe GitLab::Exporter::CLI::Server do
+  context "extra-config-command is passed" do
+    let(:argv) do
+      ["-c", "spec/fixtures/config.yml", "--extra-config-command", extra_config_command]
+        .extend(OptionParser::Arguable)
+    end
+    let(:extra_config_command) { "vault kv get top-secrets" }
+    let(:server_cmd) { described_class.new(argv) }
+
+    before do
+      allow(server_cmd).to receive(:run_extra_config_command).and_return(<<~YAML
+        probes:
+          sidekiq:
+            methods:
+              - probe_workers
+            opts:
+              redis_url: redis://hunter1@localhost:9765
+          database:
+            opts:
+              connection_string: postgres://db-admin:hunter1@localhost:6543/main-db
+      YAML
+                                                                        )
+    end
+
+    it "merges the returned YAML from the command with the passed config" do
+      expect(server_cmd.merged_config).to eq({
+                                               probes: {
+                                                 sidekiq: {
+                                                   methods: %w[probe_stats probe_workers],
+                                                   opts: {
+                                                     redis_url: "redis://hunter1@localhost:9765"
+                                                   }
+                                                 },
+                                                 database: {
+                                                   methods: ["probe_db"],
+                                                   opts: {
+                                                     connection_string:
+                                                      "postgres://db-admin:hunter1@localhost:6543/main-db"
+                                                   }
+                                                 }
+                                               }
+                                             })
+    end
+  end
+end

data/spec/fixtures/config.yml

@@ -0,0 +1,10 @@
+probes:
+  sidekiq:
+    methods:
+      - probe_stats
+
+  database:
+    methods:
+      - probe_db
+    opts:
+      connection_string: to-be-overridden

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 14.0.0
+  version: 14.2.0
 platform: ruby
 authors:
 - Pablo Carranza
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
@@ -24,6 +24,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.2.5
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.2
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -229,6 +243,7 @@ files:
 - spec/database/ci_builds_spec.rb
 - spec/database/row_count_spec.rb
 - spec/elasticsearch_spec.rb
+- spec/fixtures/config.yml
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
@@ -242,7 +257,7 @@ homepage: https://gitlab.com/gitlab-org/ruby/gems/gitlab-exporter
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -257,8 +272,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: GitLab metrics exporter
 test_files:
@@ -267,6 +282,7 @@ test_files:
 - spec/database/ci_builds_spec.rb
 - spec/database/row_count_spec.rb
 - spec/elasticsearch_spec.rb
+- spec/fixtures/config.yml
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb