gitlab-exporter 11.15.0 → 12.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 50218ad0f2683c122db031d1dcadc7f53d17b9336999f1a3469b397282204b59
-  data.tar.gz: cf4dde17a48c734c00233fd74438058d7e94a9312c5619eb6eabdb9c3235d53d
+  metadata.gz: 7c67811051d44f3c3e2fe6213eeb60f7a1c7997ddd2409008d52949cff5560fa
+  data.tar.gz: 0d750aef0b50d237783a09c22c35fdbcef0651a4998f6363cef2447786b9f8e1
 SHA512:
-  metadata.gz: c3e8259f3bf5660aa80b08ab44cf77eff1304ae494508d8faa6a0e45184ca96fa69aae03fb6ddbfcd4bcb1d12c1746342c87333268c65101fd0c2b121243eda7
-  data.tar.gz: 14f46e6a42d6cd0776f0687cee9f0091ceff4bcd4b3683692ca275b14e20c1f5e4be4b8be26dcc896ac37cb1ff235b7be7989979949acf675c476452f012de27
+  metadata.gz: 5e60b685e6005754b165963ad218462875a80eb26aef8f5a01753fb90c138f6e98e2ceb13a43aea058487579c2a9068328611af78b01ba6b58a391cc0d0e84a0
+  data.tar.gz: 611b3b9f1a66a50e7754a27f16f8fbd12f143559607fa1a4097c252823ace586043d334df8ae2a6a7060a52adb01b24ff6978141267bdddb2dc9b595d8f004e1

data/.gitignore

@@ -1,2 +1,3 @@
 .bundle
+vendor
 config/gitlab-exporter*.yml

data/.gitlab-ci.yml

@@ -12,6 +12,7 @@ variables:
 stages:
   - test
   - dast
+  - publish
 
 default:
   image: ruby:${RUBY_VERSION}
@@ -24,7 +25,8 @@ default:
 .before_scripts: &before_scripts
   - git config --global user.email "bot@gitlab.com"
   - git config --global user.name "Bot User"
-  - bundle install -j $(nproc) --path vendor
+  - bundle config set --local deployment true
+  - bundle install -j $(nproc)
 
 workflow:
   rules: &workflow_rules
@@ -41,8 +43,19 @@ rspec:
   before_script: *before_scripts
   parallel:
     matrix:
-      - RUBY_VERSION: "2.7"
-      - RUBY_VERSION: "3.0"
+      - RUBY_VERSION: ["2.7", "3.0"]
+
+rspec_integration:
+  script:
+    - bundle exec rspec spec -t integration -f d -c
+  before_script: *before_scripts
+  services:
+    - redis:latest
+  variables:
+    REDIS_URL: "redis://redis"
+  parallel:
+    matrix:
+      - RUBY_VERSION: ["2.7", "3.0"]
 
 rubocop:
   script:
@@ -57,3 +70,16 @@ gemnasium-dependency_scanning:
 
 secret_detection:
   rules: *workflow_rules
+
+publish_to_rubygems:
+  stage: publish
+  script:
+    - mkdir -p ~/.gem
+    - 'echo ":rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials'
+    - chmod 0600 ~/.gem/credentials
+    - gem build gitlab-exporter.gemspec --output=gitlab-exporter.gem
+    - gem push gitlab-exporter.gem
+  before_script: *before_scripts
+  rules:
+    # Only push to RubyGems.org when we tag a new version
+    - if: '$CI_COMMIT_TAG'

data/.ruby-version

@@ -1 +1 @@
-2.7.4
+2.7.7

data/Gemfile.lock

@@ -1,16 +1,16 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (11.15.0)
+    gitlab-exporter (12.1.0)
       connection_pool (= 2.2.5)
       faraday (~> 1.8.0)
       pg (= 1.2.3)
-      puma (= 5.6.2)
+      puma (= 5.6.5)
       quantile (= 0.2.1)
       redis (= 4.4.0)
-      redis-namespace (= 1.6.0)
+      redis-namespace (= 1.9.0)
       sidekiq (= 6.4.0)
-      sinatra (~> 2.1.0)
+      sinatra (~> 2.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -37,24 +37,24 @@ GEM
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    multipart-post (2.1.1)
-    mustermann (1.1.1)
+    multipart-post (2.2.3)
+    mustermann (2.0.2)
       ruby2_keywords (~> 0.0.1)
     nio4r (2.5.8)
     parallel (1.20.1)
     parser (3.0.0.0)
       ast (~> 2.4.1)
     pg (1.2.3)
-    puma (5.6.2)
+    puma (5.6.5)
       nio4r (~> 2.0)
     quantile (0.2.1)
-    rack (2.2.3)
-    rack-protection (2.1.0)
+    rack (2.2.5)
+    rack-protection (2.2.4)
       rack
     rainbow (3.0.0)
     redis (4.4.0)
-    redis-namespace (1.6.0)
-      redis (>= 3.0.4)
+    redis-namespace (1.9.0)
+      redis (>= 4)
     regexp_parser (2.0.3)
     rexml (3.2.4)
     rspec (3.7.0)
@@ -87,12 +87,12 @@ GEM
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
-    sinatra (2.1.0)
-      mustermann (~> 1.0)
+    sinatra (2.2.4)
+      mustermann (~> 2.0)
       rack (~> 2.2)
-      rack-protection (= 2.1.0)
+      rack-protection (= 2.2.4)
       tilt (~> 2.0)
-    tilt (2.0.10)
+    tilt (2.0.11)
     unicode-display_width (1.7.0)
     webrick (1.7.0)
 
@@ -107,4 +107,4 @@ DEPENDENCIES
   webrick (~> 1.7)
 
 BUNDLED WITH
-   2.2.29
+   2.2.22

data/config/gitlab-exporter.yml.example

@@ -6,11 +6,15 @@ db_common: &db_common
 
 # Web server config
 server:
-  name: puma # cf. https://github.com/sinatra/sinatra#available-settings
+  name: webrick # cf. https://github.com/sinatra/sinatra#available-settings
   listen_address: 0.0.0.0
   listen_port: 9168
   # Maximum amount of memory to use in megabytes, after which the process is killed
   memory_threshold: 1024
+  # TLS settings
+  tls_enabled: false
+  tls_cert_path: /tmp/server.crt
+  tls_key_path: /tmp/server.key
 
 # Probes config
 probes:

data/gitlab-exporter.gemspec

@@ -23,12 +23,12 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "connection_pool", "2.2.5"
   s.add_runtime_dependency "faraday", "~> 1.8.0"
   s.add_runtime_dependency "pg", "1.2.3"
-  s.add_runtime_dependency "puma", "5.6.2"
+  s.add_runtime_dependency "puma", "5.6.5"
   s.add_runtime_dependency "quantile", "0.2.1"
   s.add_runtime_dependency "redis", "4.4.0"
-  s.add_runtime_dependency "redis-namespace", "1.6.0"
+  s.add_runtime_dependency "redis-namespace", "1.9.0"
   s.add_runtime_dependency "sidekiq", "6.4.0"
-  s.add_runtime_dependency "sinatra", "~> 2.1.0"
+  s.add_runtime_dependency "sinatra", "~> 2.2.0"
 
   s.add_development_dependency "rspec", "~> 3.7.0"
   s.add_development_dependency "rspec-expectations", "~> 3.7.0"

data/lib/gitlab_exporter/database/row_count.rb

@@ -19,11 +19,6 @@ module GitLab
           projects.mirror = true
           AND projects.archived = false
           AND project_mirror_data.retry_count <= 14
-          AND (
-            (projects.visibility_level = 20 AND root_namespaces.visibility_level = 20)
-            OR
-            plans.name IN ('early_adopter', 'bronze', 'silver', 'gold')
-          )
           AND project_mirror_data.next_execution_timestamp > '2020-03-28'
         SQL
 
@@ -31,15 +26,6 @@ module GitLab
           select: :projects,
           joins: <<~SQL,
             INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id
-            INNER JOIN namespaces AS root_namespaces ON root_namespaces.id = (
-              WITH RECURSIVE "base_and_ancestors" AS (
-                (SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" = projects.namespace_id)
-                UNION
-                (SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."id" = "base_and_ancestors"."parent_id")
-              ) SELECT "namespaces".id FROM "base_and_ancestors" AS "namespaces" WHERE "namespaces"."parent_id" IS NULL
-            )
-            LEFT JOIN gitlab_subscriptions ON gitlab_subscriptions.namespace_id = root_namespaces.id
-            LEFT JOIN plans ON plans.id = gitlab_subscriptions.hosted_plan_id
           SQL
           check: "SELECT 1 FROM information_schema.tables WHERE table_name='plans'"
         }.freeze
@@ -125,9 +111,10 @@ module GitLab
               archived: {}
             }
           },
-          groups: {
+          namespaces: {
             select: :namespaces,
             fields: {
+              type: {},
               visibility_level: {},
               root: { definition: "(parent_id IS NULL)" }
             }
@@ -136,7 +123,7 @@ module GitLab
             select: :container_repositories,
             where: <<~SQL
               migration_state <> 'import_done'
-              AND created_at < '2022-01-23 00:00:00'
+              AND created_at < '2022-08-17 00:00:00'
             SQL
           },
           container_repositories_pre_importing: {
@@ -174,7 +161,7 @@ module GitLab
             select: :container_repositories,
             where: <<~SQL
               migration_state <> 'import_done'
-              AND created_at < '2022-01-23 00:00:00'
+              AND created_at < '2022-08-17 00:00:00'
               AND (migration_plan IN ('free', 'early_adopter')
                 OR migration_plan IS NULL)
             SQL
@@ -182,7 +169,7 @@ module GitLab
           container_repositories_import_done_free: {
             select: :container_repositories,
             where: <<~SQL
-              migration_state = 'import_done'
+              (migration_state = 'import_done' OR created_at >= '2022-08-17 00:00:00')
               AND (migration_plan IN ('free', 'early_adopter')
                 OR migration_plan IS NULL)
             SQL
@@ -191,21 +178,21 @@ module GitLab
             select: :container_repositories,
             where: <<~SQL
               migration_state = 'pre_importing'
-              AND (COALESCE(migration_pre_import_started_at, '01-01-1970') < (now() - INTERVAL '10 minutes'))
+              AND (COALESCE(migration_pre_import_started_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '20 minutes'))
             SQL
           },
           container_repositories_stalled_pre_import_done: {
             select: :container_repositories,
             where: <<~SQL
               migration_state = 'pre_import_done'
-              AND (COALESCE(migration_pre_import_done_at, '01-01-1970') < (now() - INTERVAL '5 minutes'))
+              AND (COALESCE(migration_pre_import_done_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '5 minutes'))
             SQL
           },
           container_repositories_stalled_importing: {
             select: :container_repositories,
             where: <<~SQL
               migration_state = 'importing'
-              AND (COALESCE(migration_import_started_at, '01-01-1970') < (now() - INTERVAL '5 minutes'))
+              AND (COALESCE(migration_import_started_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '5 minutes'))
             SQL
           },
           container_repositories_skipped_not_in_plan: {

data/lib/gitlab_exporter/rack_vulndb_255039_patch.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+if Gem.loaded_specs["rack"].version >= Gem::Version.new("3.0.0")
+  fail <<~ERR
+    This patch is unnecessary in Rack versions 3.0.0 or newer.
+    Please remove this file and the associated spec.
+
+    See https://github.com/rack/rack/blob/main/CHANGELOG.md#security (issue #1733)
+  ERR
+end
+
+# Patches a cache poisoning attack vector in Rack by not allowing semicolons
+# to delimit query parameters.
+# See https://github.com/rack/rack/issues/1732.
+#
+# Solution is taken from the same issue.
+#
+# The actual patch is due for release in Rack 3.0.0.
+module Rack
+  class Request # rubocop:disable Style/Documentation
+    Helpers.module_eval do
+      # rubocop: disable Naming/MethodName
+      def GET
+        if get_header(RACK_REQUEST_QUERY_STRING) == query_string
+          get_header(RACK_REQUEST_QUERY_HASH)
+        else
+          query_hash = parse_query(query_string, "&") # only allow ampersand here
+          set_header(RACK_REQUEST_QUERY_STRING, query_string)
+          set_header(RACK_REQUEST_QUERY_HASH, query_hash)
+        end
+      end
+      # rubocop: enable Naming/MethodName
+    end
+  end
+end

data/lib/gitlab_exporter/tls_helper.rb

@@ -0,0 +1,39 @@
+# Contains helper methods to generate TLS related configuration for web servers
+module TLSHelper
+  CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze
+
+  def validate_tls_config(config)
+    %i[tls_cert_path tls_key_path].each do |key|
+      fail "TLS enabled, but #{key} not specified in config" unless config.key?(key)
+
+      fail "File specified via #{key} not found: #{config[key]}" unless File.exist?(config[key])
+    end
+  end
+
+  def webrick_tls_config(config)
+    # This monkey-patches WEBrick::GenericServer, so never require this unless TLS is enabled.
+    require "webrick/ssl"
+
+    certs = load_ca_certs_bundle(File.binread(config[:tls_cert_path]))
+
+    {
+      SSLEnable: true,
+      SSLCertificate: certs.shift,
+      SSLPrivateKey: OpenSSL::PKey.read(File.binread(config[:tls_key_path])),
+      # SSLStartImmediately is true by default according to the docs, but when WEBrick creates the
+      # SSLServer internally, the switch was always nil for some reason. Setting this explicitly fixes this.
+      SSLStartImmediately: true,
+      SSLExtraChainCert: certs
+    }
+  end
+
+  # In Ruby OpenSSL v3.0.0, this can be replaced by OpenSSL::X509::Certificate.load
+  # https://github.com/ruby/openssl/issues/254
+  def load_ca_certs_bundle(ca_certs_string)
+    return [] unless ca_certs_string
+
+    ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string|
+      OpenSSL::X509::Certificate.new(ca_cert_string)
+    end
+  end
+end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "11.15.0".freeze
+    VERSION = "12.1.0".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -1,5 +1,9 @@
 require "sinatra/base"
 require "English"
+require "cgi"
+
+require_relative "rack_vulndb_255039_patch"
+require_relative "tls_helper"
 
 module GitLab
   module Exporter
@@ -51,6 +55,8 @@ module GitLab
       end
 
       class << self
+        include TLSHelper
+
         DEFAULT_WEB_SERVER = "webrick".freeze
 
         def setup(config)
@@ -74,8 +80,47 @@ module GitLab
           config ||= {}
 
           set(:server, config.fetch(:name, DEFAULT_WEB_SERVER))
-          set(:bind, config.fetch(:listen_address, "0.0.0.0"))
           set(:port, config.fetch(:listen_port, 9168))
+
+          # Depending on whether TLS is enabled or not, bind string
+          # will be different.
+          if config.fetch(:tls_enabled, "false").to_s == "true"
+            set_tls_config(config)
+          else
+            set(:bind, config.fetch(:listen_address, "0.0.0.0"))
+          end
+        end
+
+        def set_tls_config(config) # rubocop:disable Naming/AccessorMethodName
+          validate_tls_config(config)
+
+          web_server = config.fetch(:name, DEFAULT_WEB_SERVER)
+          if web_server == "webrick"
+            set_webrick_tls(config)
+          elsif web_server == "puma"
+            set_puma_tls(config)
+          else
+            fail "TLS not supported for web server `#{web_server}`."
+          end
+        end
+
+        def set_webrick_tls(config) # rubocop:disable Naming/AccessorMethodName
+          server_settings = {}
+          server_settings.merge!(webrick_tls_config(config))
+
+          set(:bind, config.fetch(:listen_address, "0.0.0.0"))
+          set(:server_settings, server_settings)
+        end
+
+        def set_puma_tls(config) # rubocop:disable Naming/AccessorMethodName
+          listen_address = config.fetch(:listen_address, "0.0.0.0")
+          listen_port = config.fetch(:listen_port, 8443)
+          tls_cert_path = CGI.escape(config.fetch(:tls_cert_path))
+          tls_key_path = CGI.escape(config.fetch(:tls_key_path))
+
+          bind_string = "ssl://#{listen_address}:#{listen_port}?cert=#{tls_cert_path}&key=#{tls_key_path}"
+
+          set(:bind, bind_string)
         end
 
         def setup_probes(config)

data/spec/integration/cli_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+require "gitlab_exporter/cli"
+
+module GitLab
+  module Exporter
+    module CLI
+      describe SidekiqRunner, :integration do
+        let(:redis_url) { ENV.fetch("REDIS_URL", "redis://localhost:6379") }
+        let(:io) { StringIO.new }
+
+        it "can properly reach out to redis" do
+          args = CLIArgs.new([io], options: { /^--redis-url/ => redis_url })
+          runner = SidekiqRunner.new(args)
+
+          expect { runner.run }.not_to raise_error
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -7,6 +7,10 @@ require "tmpdir"
 $LOAD_PATH.unshift File.expand_path(".")
 Dir["spec/support/**/*.rb"].each do |f| require f end
 
+RSpec.configure do |config|
+  config.filter_run_excluding integration: true
+end
+
 class GitRepoBuilder
   def origin
     @origin ||= create_origin
@@ -41,19 +45,30 @@ class GitRepoBuilder
 end
 
 class CLIArgs
-  def initialize(args)
+  def initialize(args, options: {})
     @arguments = args
+    @options = options
   end
 
   def options
     yield self
+
+    self
   end
 
-  def on(*args); end
+  def on(flag, *_)
+    match = @options.find { |regex, _| regex.match?(flag) }
+
+    yield match[1] if match
+  end
 
   def banner=(banner); end
 
   def parse!
     @arguments
   end
+
+  def shift
+    @arguments.shift
+  end
 end

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 11.15.0
+  version: 12.1.0
 platform: ruby
 authors:
 - Pablo Carranza
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.6.2
+        version: 5.6.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.6.2
+        version: 5.6.5
 - !ruby/object:Gem::Dependency
   name: quantile
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -202,8 +202,10 @@ files:
 - lib/gitlab_exporter/prober.rb
 - lib/gitlab_exporter/process.rb
 - lib/gitlab_exporter/prometheus.rb
+- lib/gitlab_exporter/rack_vulndb_255039_patch.rb
 - lib/gitlab_exporter/ruby.rb
 - lib/gitlab_exporter/sidekiq.rb
+- lib/gitlab_exporter/tls_helper.rb
 - lib/gitlab_exporter/util.rb
 - lib/gitlab_exporter/version.rb
 - lib/gitlab_exporter/web_exporter.rb
@@ -215,6 +217,7 @@ files:
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
+- spec/integration/cli_spec.rb
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb
@@ -224,7 +227,7 @@ homepage: http://gitlab.com
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -239,8 +242,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: GitLab metrics exporter
 test_files:
@@ -252,6 +255,7 @@ test_files:
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
+- spec/integration/cli_spec.rb
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb