gitlab-exporter 11.9.0 → 12.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48cad756413c6f691c0acf2251277c3f26adfc0ad6669fa2b15e02302b35f4d2
-  data.tar.gz: aea36d63dce2dd0727a7d2a6689ebbd10939069c53f8a617324427c4712e8846
+  metadata.gz: 10750ec2de43b0651c18fe3babdff54837e6e69b9efb921fd48bfda67d287bb6
+  data.tar.gz: bfcf564c0092a85b12f126772d06e5c04710088c6251252d69e2caf605cd1c9d
 SHA512:
-  metadata.gz: 75b9cc9ae044929dbaac0480c462cf4ba6cc36fc4d726fbd0fa87305594c9bd6a486e1856fc49e4de3a83cb5fb8d478f52dcb8a12a198da6c64b126ffd513e56
-  data.tar.gz: 2acd3d4acf2bae7c2ce60236b1576b33cbc13ef15d72c5123931624221e87bc1434294e4d4a0952d54c2ff52fa017d825c7cc3507427031ed12424923183b4df
+  metadata.gz: ab38e15c3fd76461f508921d1c960a0029b84afa673e30f7f95d57bd20d3c110232eff726723c3e7c69e154acd0b7d844fd10d960181d8de161e571423a4c151
+  data.tar.gz: 389adc0b2611a4e235b07ef72b92c77f2669db4092b624f0d5d0486fb18d894420090c172890625558cbc8f65b2a82c1906670bdf46a98a685667b757f2690d0

data/.gitignore

@@ -1,2 +1,3 @@
 .bundle
+vendor
 config/gitlab-exporter*.yml

data/.gitlab-ci.yml

@@ -12,6 +12,7 @@ variables:
 stages:
   - test
   - dast
+  - publish
 
 default:
   image: ruby:${RUBY_VERSION}
@@ -24,7 +25,8 @@ default:
 .before_scripts: &before_scripts
   - git config --global user.email "bot@gitlab.com"
   - git config --global user.name "Bot User"
-  - bundle install -j $(nproc) --path vendor
+  - bundle config set --local deployment true
+  - bundle install -j $(nproc)
 
 workflow:
   rules: &workflow_rules
@@ -41,8 +43,19 @@ rspec:
   before_script: *before_scripts
   parallel:
     matrix:
-      - RUBY_VERSION: "2.7"
-      - RUBY_VERSION: "3.0"
+      - RUBY_VERSION: ["2.7", "3.0"]
+
+rspec_integration:
+  script:
+    - bundle exec rspec spec -t integration -f d -c
+  before_script: *before_scripts
+  services:
+    - redis:latest
+  variables:
+    REDIS_URL: "redis://redis"
+  parallel:
+    matrix:
+      - RUBY_VERSION: ["2.7", "3.0"]
 
 rubocop:
   script:
@@ -57,3 +70,16 @@ gemnasium-dependency_scanning:
 
 secret_detection:
   rules: *workflow_rules
+
+publish_to_rubygems:
+  stage: publish
+  script:
+    - mkdir -p ~/.gem
+    - 'echo ":rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials'
+    - chmod 0600 ~/.gem/credentials
+    - gem build gitlab-exporter.gemspec --output=gitlab-exporter.gem
+    - gem push gitlab-exporter.gem
+  before_script: *before_scripts
+  rules:
+    # Only push to RubyGems.org when we tag a new version
+    - if: '$CI_COMMIT_TAG'

data/.ruby-version

@@ -1 +1 @@
-2.7.4
+2.7.6

data/Gemfile.lock

@@ -1,16 +1,16 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (11.9.0)
+    gitlab-exporter (12.0.1)
       connection_pool (= 2.2.5)
       faraday (~> 1.8.0)
       pg (= 1.2.3)
-      puma (= 5.5.2)
+      puma (= 5.6.5)
       quantile (= 0.2.1)
       redis (= 4.4.0)
-      redis-namespace (= 1.6.0)
-      sidekiq (= 6.3.1)
-      sinatra (~> 2.1.0)
+      redis-namespace (= 1.9.0)
+      sidekiq (= 6.4.0)
+      sinatra (~> 2.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -37,24 +37,24 @@ GEM
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    multipart-post (2.1.1)
-    mustermann (1.1.1)
+    multipart-post (2.2.3)
+    mustermann (2.0.2)
       ruby2_keywords (~> 0.0.1)
     nio4r (2.5.8)
     parallel (1.20.1)
     parser (3.0.0.0)
       ast (~> 2.4.1)
     pg (1.2.3)
-    puma (5.5.2)
+    puma (5.6.5)
       nio4r (~> 2.0)
     quantile (0.2.1)
-    rack (2.2.3)
-    rack-protection (2.1.0)
+    rack (2.2.4)
+    rack-protection (2.2.2)
       rack
     rainbow (3.0.0)
     redis (4.4.0)
-    redis-namespace (1.6.0)
-      redis (>= 3.0.4)
+    redis-namespace (1.9.0)
+      redis (>= 4)
     regexp_parser (2.0.3)
     rexml (3.2.4)
     rspec (3.7.0)
@@ -83,16 +83,16 @@ GEM
       parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    sidekiq (6.3.1)
+    sidekiq (6.4.0)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
-    sinatra (2.1.0)
-      mustermann (~> 1.0)
+    sinatra (2.2.2)
+      mustermann (~> 2.0)
       rack (~> 2.2)
-      rack-protection (= 2.1.0)
+      rack-protection (= 2.2.2)
       tilt (~> 2.0)
-    tilt (2.0.10)
+    tilt (2.0.11)
     unicode-display_width (1.7.0)
     webrick (1.7.0)
 

data/config/gitlab-exporter.yml.example

@@ -6,11 +6,15 @@ db_common: &db_common
 
 # Web server config
 server:
-  name: puma # cf. https://github.com/sinatra/sinatra#available-settings
+  name: webrick # cf. https://github.com/sinatra/sinatra#available-settings
   listen_address: 0.0.0.0
   listen_port: 9168
   # Maximum amount of memory to use in megabytes, after which the process is killed
   memory_threshold: 1024
+  # TLS settings
+  tls_enabled: false
+  tls_cert_path: /tmp/server.crt
+  tls_key_path: /tmp/server.key
 
 # Probes config
 probes:

data/gitlab-exporter.gemspec

@@ -23,12 +23,12 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "connection_pool", "2.2.5"
   s.add_runtime_dependency "faraday", "~> 1.8.0"
   s.add_runtime_dependency "pg", "1.2.3"
-  s.add_runtime_dependency "puma", "5.5.2"
+  s.add_runtime_dependency "puma", "5.6.5"
   s.add_runtime_dependency "quantile", "0.2.1"
   s.add_runtime_dependency "redis", "4.4.0"
-  s.add_runtime_dependency "redis-namespace", "1.6.0"
-  s.add_runtime_dependency "sidekiq", "6.3.1"
-  s.add_runtime_dependency "sinatra", "~> 2.1.0"
+  s.add_runtime_dependency "redis-namespace", "1.9.0"
+  s.add_runtime_dependency "sidekiq", "6.4.0"
+  s.add_runtime_dependency "sinatra", "~> 2.2.0"
 
   s.add_development_dependency "rspec", "~> 3.7.0"
   s.add_development_dependency "rspec-expectations", "~> 3.7.0"

data/lib/gitlab_exporter/database/row_count.rb

@@ -125,12 +125,145 @@ module GitLab
               archived: {}
             }
           },
-          groups: {
+          namespaces: {
             select: :namespaces,
             fields: {
+              type: {},
               visibility_level: {},
               root: { definition: "(parent_id IS NULL)" }
             }
+          },
+          container_repositories_migration_pending: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state <> 'import_done'
+              AND created_at < '2022-08-17 00:00:00'
+            SQL
+          },
+          container_repositories_pre_importing: {
+            select: :container_repositories,
+            where: "migration_state = 'pre_importing'"
+          },
+          container_repositories_importing: {
+            select: :container_repositories,
+            where: "migration_state = 'importing'"
+          },
+          container_repositories_pre_import_done: {
+            select: :container_repositories,
+            where: "migration_state = 'pre_import_done'"
+          },
+          container_repositories_import_done: {
+            select: :container_repositories,
+            where: "migration_state = 'import_done'"
+          },
+          container_repositories_import_skipped: {
+            select: :container_repositories,
+            where: "migration_state = 'import_skipped'"
+          },
+          container_repositories_import_aborted: {
+            select: :container_repositories,
+            where: "migration_state = 'import_aborted'"
+          },
+          container_repositories_migration_all_free: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_plan IN ('free', 'early_adopter')
+              OR migration_plan IS NULL
+            SQL
+          },
+          container_repositories_migration_pending_free: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state <> 'import_done'
+              AND created_at < '2022-08-17 00:00:00'
+              AND (migration_plan IN ('free', 'early_adopter')
+                OR migration_plan IS NULL)
+            SQL
+          },
+          container_repositories_import_done_free: {
+            select: :container_repositories,
+            where: <<~SQL
+              (migration_state = 'import_done' OR created_at >= '2022-08-17 00:00:00')
+              AND (migration_plan IN ('free', 'early_adopter')
+                OR migration_plan IS NULL)
+            SQL
+          },
+          container_repositories_stalled_pre_importing: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'pre_importing'
+              AND (COALESCE(migration_pre_import_started_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '20 minutes'))
+            SQL
+          },
+          container_repositories_stalled_pre_import_done: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'pre_import_done'
+              AND (COALESCE(migration_pre_import_done_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '5 minutes'))
+            SQL
+          },
+          container_repositories_stalled_importing: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'importing'
+              AND (COALESCE(migration_import_started_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '5 minutes'))
+            SQL
+          },
+          container_repositories_skipped_not_in_plan: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 0
+            SQL
+          },
+          container_repositories_skipped_too_many_retries: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 1
+            SQL
+          },
+          container_repositories_skipped_too_many_tags: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 2
+            SQL
+          },
+          container_repositories_skipped_deny_list: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 3
+            SQL
+          },
+          container_repositories_skipped_canceled: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason IN (4, 8)
+            SQL
+          },
+          container_repositories_skipped_not_found: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 5
+            SQL
+          },
+          container_repositories_skipped_native: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 6
+            SQL
+          },
+          container_repositories_skipped_force_canceled: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 7
+            SQL
           }
         }.freeze
 

data/lib/gitlab_exporter/rack_vulndb_255039_patch.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+if Gem.loaded_specs["rack"].version >= Gem::Version.new("3.0.0")
+  fail <<~ERR
+    This patch is unnecessary in Rack versions 3.0.0 or newer.
+    Please remove this file and the associated spec.
+
+    See https://github.com/rack/rack/blob/main/CHANGELOG.md#security (issue #1733)
+  ERR
+end
+
+# Patches a cache poisoning attack vector in Rack by not allowing semicolons
+# to delimit query parameters.
+# See https://github.com/rack/rack/issues/1732.
+#
+# Solution is taken from the same issue.
+#
+# The actual patch is due for release in Rack 3.0.0.
+module Rack
+  class Request # rubocop:disable Style/Documentation
+    Helpers.module_eval do
+      # rubocop: disable Naming/MethodName
+      def GET
+        if get_header(RACK_REQUEST_QUERY_STRING) == query_string
+          get_header(RACK_REQUEST_QUERY_HASH)
+        else
+          query_hash = parse_query(query_string, "&") # only allow ampersand here
+          set_header(RACK_REQUEST_QUERY_STRING, query_string)
+          set_header(RACK_REQUEST_QUERY_HASH, query_hash)
+        end
+      end
+      # rubocop: enable Naming/MethodName
+    end
+  end
+end

data/lib/gitlab_exporter/tls_helper.rb

@@ -0,0 +1,39 @@
+# Contains helper methods to generate TLS related configuration for web servers
+module TLSHelper
+  CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze
+
+  def validate_tls_config(config)
+    %i[tls_cert_path tls_key_path].each do |key|
+      fail "TLS enabled, but #{key} not specified in config" unless config.key?(key)
+
+      fail "File specified via #{key} not found: #{config[key]}" unless File.exist?(config[key])
+    end
+  end
+
+  def webrick_tls_config(config)
+    # This monkey-patches WEBrick::GenericServer, so never require this unless TLS is enabled.
+    require "webrick/ssl"
+
+    certs = load_ca_certs_bundle(File.binread(config[:tls_cert_path]))
+
+    {
+      SSLEnable: true,
+      SSLCertificate: certs.shift,
+      SSLPrivateKey: OpenSSL::PKey.read(File.binread(config[:tls_key_path])),
+      # SSLStartImmediately is true by default according to the docs, but when WEBrick creates the
+      # SSLServer internally, the switch was always nil for some reason. Setting this explicitly fixes this.
+      SSLStartImmediately: true,
+      SSLExtraChainCert: certs
+    }
+  end
+
+  # In Ruby OpenSSL v3.0.0, this can be replaced by OpenSSL::X509::Certificate.load
+  # https://github.com/ruby/openssl/issues/254
+  def load_ca_certs_bundle(ca_certs_string)
+    return [] unless ca_certs_string
+
+    ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string|
+      OpenSSL::X509::Certificate.new(ca_cert_string)
+    end
+  end
+end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "11.9.0".freeze
+    VERSION = "12.0.1".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -1,5 +1,9 @@
 require "sinatra/base"
 require "English"
+require "cgi"
+
+require_relative "rack_vulndb_255039_patch"
+require_relative "tls_helper"
 
 module GitLab
   module Exporter
@@ -51,6 +55,8 @@ module GitLab
       end
 
       class << self
+        include TLSHelper
+
         DEFAULT_WEB_SERVER = "webrick".freeze
 
         def setup(config)
@@ -74,8 +80,47 @@ module GitLab
           config ||= {}
 
           set(:server, config.fetch(:name, DEFAULT_WEB_SERVER))
-          set(:bind, config.fetch(:listen_address, "0.0.0.0"))
           set(:port, config.fetch(:listen_port, 9168))
+
+          # Depending on whether TLS is enabled or not, bind string
+          # will be different.
+          if config.fetch(:tls_enabled, "false").to_s == "true"
+            set_tls_config(config)
+          else
+            set(:bind, config.fetch(:listen_address, "0.0.0.0"))
+          end
+        end
+
+        def set_tls_config(config) # rubocop:disable Naming/AccessorMethodName
+          validate_tls_config(config)
+
+          web_server = config.fetch(:name, DEFAULT_WEB_SERVER)
+          if web_server == "webrick"
+            set_webrick_tls(config)
+          elsif web_server == "puma"
+            set_puma_tls(config)
+          else
+            fail "TLS not supported for web server `#{web_server}`."
+          end
+        end
+
+        def set_webrick_tls(config) # rubocop:disable Naming/AccessorMethodName
+          server_settings = {}
+          server_settings.merge!(webrick_tls_config(config))
+
+          set(:bind, config.fetch(:listen_address, "0.0.0.0"))
+          set(:server_settings, server_settings)
+        end
+
+        def set_puma_tls(config) # rubocop:disable Naming/AccessorMethodName
+          listen_address = config.fetch(:listen_address, "0.0.0.0")
+          listen_port = config.fetch(:listen_port, 8443)
+          tls_cert_path = CGI.escape(config.fetch(:tls_cert_path))
+          tls_key_path = CGI.escape(config.fetch(:tls_key_path))
+
+          bind_string = "ssl://#{listen_address}:#{listen_port}?cert=#{tls_cert_path}&key=#{tls_key_path}"
+
+          set(:bind, bind_string)
         end
 
         def setup_probes(config)

data/spec/integration/cli_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+require "gitlab_exporter/cli"
+
+module GitLab
+  module Exporter
+    module CLI
+      describe SidekiqRunner, :integration do
+        let(:redis_url) { ENV.fetch("REDIS_URL", "redis://localhost:6379") }
+        let(:io) { StringIO.new }
+
+        it "can properly reach out to redis" do
+          args = CLIArgs.new([io], options: { /^--redis-url/ => redis_url })
+          runner = SidekiqRunner.new(args)
+
+          expect { runner.run }.not_to raise_error
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -7,6 +7,10 @@ require "tmpdir"
 $LOAD_PATH.unshift File.expand_path(".")
 Dir["spec/support/**/*.rb"].each do |f| require f end
 
+RSpec.configure do |config|
+  config.filter_run_excluding integration: true
+end
+
 class GitRepoBuilder
   def origin
     @origin ||= create_origin
@@ -41,19 +45,30 @@ class GitRepoBuilder
 end
 
 class CLIArgs
-  def initialize(args)
+  def initialize(args, options: {})
     @arguments = args
+    @options = options
   end
 
   def options
     yield self
+
+    self
   end
 
-  def on(*args); end
+  def on(flag, *_)
+    match = @options.find { |regex, _| regex.match?(flag) }
+
+    yield match[1] if match
+  end
 
   def banner=(banner); end
 
   def parse!
     @arguments
   end
+
+  def shift
+    @arguments.shift
+  end
 end

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 11.9.0
+  version: 12.0.1
 platform: ruby
 authors:
 - Pablo Carranza
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.5.2
+        version: 5.6.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.5.2
+        version: 5.6.5
 - !ruby/object:Gem::Dependency
   name: quantile
   requirement: !ruby/object:Gem::Requirement
@@ -100,42 +100,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.3.1
+        version: 6.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.3.1
+        version: 6.4.0
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -202,8 +202,10 @@ files:
 - lib/gitlab_exporter/prober.rb
 - lib/gitlab_exporter/process.rb
 - lib/gitlab_exporter/prometheus.rb
+- lib/gitlab_exporter/rack_vulndb_255039_patch.rb
 - lib/gitlab_exporter/ruby.rb
 - lib/gitlab_exporter/sidekiq.rb
+- lib/gitlab_exporter/tls_helper.rb
 - lib/gitlab_exporter/util.rb
 - lib/gitlab_exporter/version.rb
 - lib/gitlab_exporter/web_exporter.rb
@@ -215,6 +217,7 @@ files:
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
+- spec/integration/cli_spec.rb
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb
@@ -224,7 +227,7 @@ homepage: http://gitlab.com
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -239,8 +242,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: GitLab metrics exporter
 test_files:
@@ -252,6 +255,7 @@ test_files:
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
+- spec/integration/cli_spec.rb
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb