gitlab-exporter 11.4.0 → 12.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1629fb16b7451c28e085202cc001961dc3f1ac66308146ca992a7eeaf179c6e2
-  data.tar.gz: a6158d5da5323e132a4a07e9b33238bb6b224658bc60a6885225c6d6a3729698
+  metadata.gz: 7c67811051d44f3c3e2fe6213eeb60f7a1c7997ddd2409008d52949cff5560fa
+  data.tar.gz: 0d750aef0b50d237783a09c22c35fdbcef0651a4998f6363cef2447786b9f8e1
 SHA512:
-  metadata.gz: c1f03be5bb43004977d466b5dd4985a1626cc1cf430ee4d2137ad1c5983da527521ca3b3785e28e7daf20f18405b1bf43e9964106d60900b5c26e102e035d7c6
-  data.tar.gz: 5433100b784c35f2c66e676e2c361ed6945fbcd4cc29d77be831f4f0c7d2f2d4a9b6271aeb8492c7af13f266975e053322a84e288c12502ba65ff03996f13a25
+  metadata.gz: 5e60b685e6005754b165963ad218462875a80eb26aef8f5a01753fb90c138f6e98e2ceb13a43aea058487579c2a9068328611af78b01ba6b58a391cc0d0e84a0
+  data.tar.gz: 611b3b9f1a66a50e7754a27f16f8fbd12f143559607fa1a4097c252823ace586043d334df8ae2a6a7060a52adb01b24ff6978141267bdddb2dc9b595d8f004e1

data/.gitignore

@@ -1,2 +1,3 @@
 .bundle
+vendor
 config/gitlab-exporter*.yml

data/.gitlab-ci.yml

@@ -12,6 +12,7 @@ variables:
 stages:
   - test
   - dast
+  - publish
 
 default:
   image: ruby:${RUBY_VERSION}
@@ -24,7 +25,8 @@ default:
 .before_scripts: &before_scripts
   - git config --global user.email "bot@gitlab.com"
   - git config --global user.name "Bot User"
-  - bundle install -j $(nproc) --path vendor
+  - bundle config set --local deployment true
+  - bundle install -j $(nproc)
 
 workflow:
   rules: &workflow_rules
@@ -41,8 +43,19 @@ rspec:
   before_script: *before_scripts
   parallel:
     matrix:
-      - RUBY_VERSION: "2.7"
-      - RUBY_VERSION: "3.0"
+      - RUBY_VERSION: ["2.7", "3.0"]
+
+rspec_integration:
+  script:
+    - bundle exec rspec spec -t integration -f d -c
+  before_script: *before_scripts
+  services:
+    - redis:latest
+  variables:
+    REDIS_URL: "redis://redis"
+  parallel:
+    matrix:
+      - RUBY_VERSION: ["2.7", "3.0"]
 
 rubocop:
   script:
@@ -57,3 +70,16 @@ gemnasium-dependency_scanning:
 
 secret_detection:
   rules: *workflow_rules
+
+publish_to_rubygems:
+  stage: publish
+  script:
+    - mkdir -p ~/.gem
+    - 'echo ":rubygems_api_key: ${RUBYGEMS_API_KEY}" > ~/.gem/credentials'
+    - chmod 0600 ~/.gem/credentials
+    - gem build gitlab-exporter.gemspec --output=gitlab-exporter.gem
+    - gem push gitlab-exporter.gem
+  before_script: *before_scripts
+  rules:
+    # Only push to RubyGems.org when we tag a new version
+    - if: '$CI_COMMIT_TAG'

data/.ruby-version

@@ -1 +1 @@
-2.7.4
+2.7.7

data/Gemfile.lock

@@ -1,15 +1,16 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (11.4.0)
+    gitlab-exporter (12.1.0)
       connection_pool (= 2.2.5)
+      faraday (~> 1.8.0)
       pg (= 1.2.3)
-      puma (= 5.3.2)
+      puma (= 5.6.5)
       quantile (= 0.2.1)
-      redis (= 4.1.4)
-      redis-namespace (= 1.6.0)
-      sidekiq (= 5.2.9)
-      sinatra (~> 2.1.0)
+      redis (= 4.4.0)
+      redis-namespace (= 1.9.0)
+      sidekiq (= 6.4.0)
+      sinatra (~> 2.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -17,23 +18,43 @@ GEM
     ast (2.4.1)
     connection_pool (2.2.5)
     diff-lcs (1.3)
-    mustermann (1.1.1)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    multipart-post (2.2.3)
+    mustermann (2.0.2)
       ruby2_keywords (~> 0.0.1)
     nio4r (2.5.8)
     parallel (1.20.1)
     parser (3.0.0.0)
       ast (~> 2.4.1)
     pg (1.2.3)
-    puma (5.3.2)
+    puma (5.6.5)
       nio4r (~> 2.0)
     quantile (0.2.1)
-    rack (2.2.3)
-    rack-protection (2.1.0)
+    rack (2.2.5)
+    rack-protection (2.2.4)
       rack
     rainbow (3.0.0)
-    redis (4.1.4)
-    redis-namespace (1.6.0)
-      redis (>= 3.0.4)
+    redis (4.4.0)
+    redis-namespace (1.9.0)
+      redis (>= 4)
     regexp_parser (2.0.3)
     rexml (3.2.4)
     rspec (3.7.0)
@@ -62,17 +83,16 @@ GEM
       parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    sidekiq (5.2.9)
-      connection_pool (~> 2.2, >= 2.2.2)
+    sidekiq (6.4.0)
+      connection_pool (>= 2.2.2)
       rack (~> 2.0)
-      rack-protection (>= 1.5.0)
-      redis (>= 3.3.5, < 4.2)
-    sinatra (2.1.0)
-      mustermann (~> 1.0)
+      redis (>= 4.2.0)
+    sinatra (2.2.4)
+      mustermann (~> 2.0)
       rack (~> 2.2)
-      rack-protection (= 2.1.0)
+      rack-protection (= 2.2.4)
       tilt (~> 2.0)
-    tilt (2.0.10)
+    tilt (2.0.11)
     unicode-display_width (1.7.0)
     webrick (1.7.0)
 

data/README.md

@@ -59,6 +59,8 @@ metrics.
     * `sidekiq_schedule_set_backlog_count`
     * `sidekiq_retry_set_processing_delay_seconds`
     * `sidekiq_retry_set_backlog_count`
+1. [Elasticsearch](lib/gitlab_exporter/elasticsearch.rb)
+  * [Migrations](https://docs.gitlab.com/ee/integration/elasticsearch.html#advanced-search-migrations) -- `elasticsearch_migrations_state`
 
 ### Setup with GitLab Development Kit
 

data/config/gitlab-exporter.yml.example

@@ -6,11 +6,15 @@ db_common: &db_common
 
 # Web server config
 server:
-  name: puma # cf. https://github.com/sinatra/sinatra#available-settings
+  name: webrick # cf. https://github.com/sinatra/sinatra#available-settings
   listen_address: 0.0.0.0
   listen_port: 9168
   # Maximum amount of memory to use in megabytes, after which the process is killed
   memory_threshold: 1024
+  # TLS settings
+  tls_enabled: false
+  tls_cert_path: /tmp/server.crt
+  tls_key_path: /tmp/server.key
 
 # Probes config
 probes:
@@ -99,6 +103,22 @@ probes:
     opts:
       quantiles: false
 
+
+  # Uncomment below to add GitLab specific Elasticsearch metrics.
+  #
+  # To support multiple authorization types, opts are passed to a Faraday connection object.
+  # See: https://www.rubydoc.info/github/lostisland/faraday/Faraday%2FConnection:initialize
+  #
+  # elasticsearch: &elasticsearch
+  #   methods:
+  #     - probe_migrations
+  #
+  #   opts:
+  #     - url: http://localhost:9200
+  #       options:
+  #         headers:
+  #           Authorization: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="
+
   metrics:
     multiple: true
     git_process:

data/gitlab-exporter.gemspec

@@ -21,13 +21,14 @@ Gem::Specification.new do |s|
   s.license       = "MIT"
 
   s.add_runtime_dependency "connection_pool", "2.2.5"
+  s.add_runtime_dependency "faraday", "~> 1.8.0"
   s.add_runtime_dependency "pg", "1.2.3"
-  s.add_runtime_dependency "puma", "5.3.2"
+  s.add_runtime_dependency "puma", "5.6.5"
   s.add_runtime_dependency "quantile", "0.2.1"
-  s.add_runtime_dependency "redis", "4.1.4"
-  s.add_runtime_dependency "redis-namespace", "1.6.0"
-  s.add_runtime_dependency "sidekiq", "5.2.9"
-  s.add_runtime_dependency "sinatra", "~> 2.1.0"
+  s.add_runtime_dependency "redis", "4.4.0"
+  s.add_runtime_dependency "redis-namespace", "1.9.0"
+  s.add_runtime_dependency "sidekiq", "6.4.0"
+  s.add_runtime_dependency "sinatra", "~> 2.2.0"
 
   s.add_development_dependency "rspec", "~> 3.7.0"
   s.add_development_dependency "rspec-expectations", "~> 3.7.0"

data/lib/gitlab_exporter/database/row_count.rb

@@ -19,11 +19,6 @@ module GitLab
           projects.mirror = true
           AND projects.archived = false
           AND project_mirror_data.retry_count <= 14
-          AND (
-            (projects.visibility_level = 20 AND root_namespaces.visibility_level = 20)
-            OR
-            plans.name IN ('early_adopter', 'bronze', 'silver', 'gold')
-          )
           AND project_mirror_data.next_execution_timestamp > '2020-03-28'
         SQL
 
@@ -31,15 +26,6 @@ module GitLab
           select: :projects,
           joins: <<~SQL,
             INNER JOIN project_mirror_data ON project_mirror_data.project_id = projects.id
-            INNER JOIN namespaces AS root_namespaces ON root_namespaces.id = (
-              WITH RECURSIVE "base_and_ancestors" AS (
-                (SELECT "namespaces".* FROM "namespaces" WHERE "namespaces"."id" = projects.namespace_id)
-                UNION
-                (SELECT "namespaces".* FROM "namespaces", "base_and_ancestors" WHERE "namespaces"."id" = "base_and_ancestors"."parent_id")
-              ) SELECT "namespaces".id FROM "base_and_ancestors" AS "namespaces" WHERE "namespaces"."parent_id" IS NULL
-            )
-            LEFT JOIN gitlab_subscriptions ON gitlab_subscriptions.namespace_id = root_namespaces.id
-            LEFT JOIN plans ON plans.id = gitlab_subscriptions.hosted_plan_id
           SQL
           check: "SELECT 1 FROM information_schema.tables WHERE table_name='plans'"
         }.freeze
@@ -125,29 +111,146 @@ module GitLab
               archived: {}
             }
           },
-          groups: {
+          namespaces: {
             select: :namespaces,
             fields: {
+              type: {},
               visibility_level: {},
               root: { definition: "(parent_id IS NULL)" }
             }
           },
-          registry_gc_manifest_review_queue: { select: :gc_manifest_review_queue },
-          registry_gc_manifest_review_queue_overdue: {
-            select: :gc_manifest_review_queue,
-            where: "review_after < NOW()"
-          },
-          registry_gc_blob_review_queue: { select: :gc_blob_review_queue },
-          registry_gc_blob_review_queue_overdue: {
-            select: :gc_blob_review_queue,
-            where: "review_after < NOW()"
-          },
-          registry_top_level_namespaces: { select: :top_level_namespaces },
-          # Please note that the tables below are partitioned, so a SELECT COUNT(*)
-          # will scale poorly. Avoid using these in production:
-          registry_repositories: { select: :repositories },
-          registry_manifests: { select: :manifests },
-          registry_blobs: { select: :blobs }
+          container_repositories_migration_pending: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state <> 'import_done'
+              AND created_at < '2022-08-17 00:00:00'
+            SQL
+          },
+          container_repositories_pre_importing: {
+            select: :container_repositories,
+            where: "migration_state = 'pre_importing'"
+          },
+          container_repositories_importing: {
+            select: :container_repositories,
+            where: "migration_state = 'importing'"
+          },
+          container_repositories_pre_import_done: {
+            select: :container_repositories,
+            where: "migration_state = 'pre_import_done'"
+          },
+          container_repositories_import_done: {
+            select: :container_repositories,
+            where: "migration_state = 'import_done'"
+          },
+          container_repositories_import_skipped: {
+            select: :container_repositories,
+            where: "migration_state = 'import_skipped'"
+          },
+          container_repositories_import_aborted: {
+            select: :container_repositories,
+            where: "migration_state = 'import_aborted'"
+          },
+          container_repositories_migration_all_free: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_plan IN ('free', 'early_adopter')
+              OR migration_plan IS NULL
+            SQL
+          },
+          container_repositories_migration_pending_free: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state <> 'import_done'
+              AND created_at < '2022-08-17 00:00:00'
+              AND (migration_plan IN ('free', 'early_adopter')
+                OR migration_plan IS NULL)
+            SQL
+          },
+          container_repositories_import_done_free: {
+            select: :container_repositories,
+            where: <<~SQL
+              (migration_state = 'import_done' OR created_at >= '2022-08-17 00:00:00')
+              AND (migration_plan IN ('free', 'early_adopter')
+                OR migration_plan IS NULL)
+            SQL
+          },
+          container_repositories_stalled_pre_importing: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'pre_importing'
+              AND (COALESCE(migration_pre_import_started_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '20 minutes'))
+            SQL
+          },
+          container_repositories_stalled_pre_import_done: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'pre_import_done'
+              AND (COALESCE(migration_pre_import_done_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '5 minutes'))
+            SQL
+          },
+          container_repositories_stalled_importing: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'importing'
+              AND (COALESCE(migration_import_started_at, TO_TIMESTAMP(0)) < (now() - INTERVAL '5 minutes'))
+            SQL
+          },
+          container_repositories_skipped_not_in_plan: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 0
+            SQL
+          },
+          container_repositories_skipped_too_many_retries: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 1
+            SQL
+          },
+          container_repositories_skipped_too_many_tags: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 2
+            SQL
+          },
+          container_repositories_skipped_deny_list: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 3
+            SQL
+          },
+          container_repositories_skipped_canceled: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason IN (4, 8)
+            SQL
+          },
+          container_repositories_skipped_not_found: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 5
+            SQL
+          },
+          container_repositories_skipped_native: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 6
+            SQL
+          },
+          container_repositories_skipped_force_canceled: {
+            select: :container_repositories,
+            where: <<~SQL
+              migration_state = 'import_skipped'
+              AND migration_skipped_reason = 7
+            SQL
+          }
         }.freeze
 
         def initialize(selected_queries: nil, **args)

data/lib/gitlab_exporter/elasticsearch.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+
+module GitLab
+  module Exporter
+    # Exports GitLab specific Elasticsearch metrics.
+    #
+    # For generic operational metrics, see elasticsearch_exporter.
+    # https://github.com/prometheus-community/elasticsearch_exporter
+    class ElasticsearchProber
+      MIGRATION_STATE_MAP = {
+        unknown: -9,
+        # TODO: failed: -1
+        pending: 0,
+        running: 1,
+        halted: 2,
+        completed: 3
+      }.freeze
+
+      def initialize(metrics: PrometheusMetrics.new, logger: nil, **opts)
+        @metrics = metrics
+        @logger  = logger
+        @opts    = opts
+      end
+
+      # Probes the state of Advanced Search Migrations
+      # https://docs.gitlab.com/ee/integration/elasticsearch.html#advanced-search-migrations
+      def probe_migrations
+        elastic_probe do |conn|
+          resp = conn.get "/gitlab-*-migrations/_search"
+          return unless resp.status == 200
+
+          JSON.parse(resp.body).dig("hits", "hits").each do |hit|
+            @metrics.add(
+              "elasticsearch_migrations_info", 1, # 1 is a noop.
+              state: inferred_migration_state(hit.fetch("_source")),
+              name: hit.fetch("_id")
+            )
+          end
+        end
+      rescue StandardError => e
+        @logger&.error "ElasticsearchProper encountered an error: #{e}"
+      end
+
+      private
+
+      def elastic_probe
+        yield Faraday.new(@opts.fetch(:url), @opts.fetch(:options, {}))
+      end
+
+      def inferred_migration_state(migration)
+        return :pending if migration["started_at"] == ""
+
+        if migration["started_at"] != "" && migration["completed_at"] == "" && !migration.dig("state", "halted")
+          return :running
+        end
+
+        return :completed if migration["completed"]
+        return :halted if migration.dig("state", "halted")
+
+        @logger&.error("Elasticsearch probe doesn't know the state of a migration")
+        :unknown
+      end
+    end
+  end
+end

data/lib/gitlab_exporter/rack_vulndb_255039_patch.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+if Gem.loaded_specs["rack"].version >= Gem::Version.new("3.0.0")
+  fail <<~ERR
+    This patch is unnecessary in Rack versions 3.0.0 or newer.
+    Please remove this file and the associated spec.
+
+    See https://github.com/rack/rack/blob/main/CHANGELOG.md#security (issue #1733)
+  ERR
+end
+
+# Patches a cache poisoning attack vector in Rack by not allowing semicolons
+# to delimit query parameters.
+# See https://github.com/rack/rack/issues/1732.
+#
+# Solution is taken from the same issue.
+#
+# The actual patch is due for release in Rack 3.0.0.
+module Rack
+  class Request # rubocop:disable Style/Documentation
+    Helpers.module_eval do
+      # rubocop: disable Naming/MethodName
+      def GET
+        if get_header(RACK_REQUEST_QUERY_STRING) == query_string
+          get_header(RACK_REQUEST_QUERY_HASH)
+        else
+          query_hash = parse_query(query_string, "&") # only allow ampersand here
+          set_header(RACK_REQUEST_QUERY_STRING, query_string)
+          set_header(RACK_REQUEST_QUERY_HASH, query_hash)
+        end
+      end
+      # rubocop: enable Naming/MethodName
+    end
+  end
+end

data/lib/gitlab_exporter/tls_helper.rb

@@ -0,0 +1,39 @@
+# Contains helper methods to generate TLS related configuration for web servers
+module TLSHelper
+  CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze
+
+  def validate_tls_config(config)
+    %i[tls_cert_path tls_key_path].each do |key|
+      fail "TLS enabled, but #{key} not specified in config" unless config.key?(key)
+
+      fail "File specified via #{key} not found: #{config[key]}" unless File.exist?(config[key])
+    end
+  end
+
+  def webrick_tls_config(config)
+    # This monkey-patches WEBrick::GenericServer, so never require this unless TLS is enabled.
+    require "webrick/ssl"
+
+    certs = load_ca_certs_bundle(File.binread(config[:tls_cert_path]))
+
+    {
+      SSLEnable: true,
+      SSLCertificate: certs.shift,
+      SSLPrivateKey: OpenSSL::PKey.read(File.binread(config[:tls_key_path])),
+      # SSLStartImmediately is true by default according to the docs, but when WEBrick creates the
+      # SSLServer internally, the switch was always nil for some reason. Setting this explicitly fixes this.
+      SSLStartImmediately: true,
+      SSLExtraChainCert: certs
+    }
+  end
+
+  # In Ruby OpenSSL v3.0.0, this can be replaced by OpenSSL::X509::Certificate.load
+  # https://github.com/ruby/openssl/issues/254
+  def load_ca_certs_bundle(ca_certs_string)
+    return [] unless ca_certs_string
+
+    ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string|
+      OpenSSL::X509::Certificate.new(ca_cert_string)
+    end
+  end
+end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "11.4.0".freeze
+    VERSION = "12.1.0".freeze
   end
 end

data/lib/gitlab_exporter/web_exporter.rb

@@ -1,5 +1,9 @@
 require "sinatra/base"
 require "English"
+require "cgi"
+
+require_relative "rack_vulndb_255039_patch"
+require_relative "tls_helper"
 
 module GitLab
   module Exporter
@@ -51,6 +55,8 @@ module GitLab
       end
 
       class << self
+        include TLSHelper
+
         DEFAULT_WEB_SERVER = "webrick".freeze
 
         def setup(config)
@@ -74,8 +80,47 @@ module GitLab
           config ||= {}
 
           set(:server, config.fetch(:name, DEFAULT_WEB_SERVER))
-          set(:bind, config.fetch(:listen_address, "0.0.0.0"))
           set(:port, config.fetch(:listen_port, 9168))
+
+          # Depending on whether TLS is enabled or not, bind string
+          # will be different.
+          if config.fetch(:tls_enabled, "false").to_s == "true"
+            set_tls_config(config)
+          else
+            set(:bind, config.fetch(:listen_address, "0.0.0.0"))
+          end
+        end
+
+        def set_tls_config(config) # rubocop:disable Naming/AccessorMethodName
+          validate_tls_config(config)
+
+          web_server = config.fetch(:name, DEFAULT_WEB_SERVER)
+          if web_server == "webrick"
+            set_webrick_tls(config)
+          elsif web_server == "puma"
+            set_puma_tls(config)
+          else
+            fail "TLS not supported for web server `#{web_server}`."
+          end
+        end
+
+        def set_webrick_tls(config) # rubocop:disable Naming/AccessorMethodName
+          server_settings = {}
+          server_settings.merge!(webrick_tls_config(config))
+
+          set(:bind, config.fetch(:listen_address, "0.0.0.0"))
+          set(:server_settings, server_settings)
+        end
+
+        def set_puma_tls(config) # rubocop:disable Naming/AccessorMethodName
+          listen_address = config.fetch(:listen_address, "0.0.0.0")
+          listen_port = config.fetch(:listen_port, 8443)
+          tls_cert_path = CGI.escape(config.fetch(:tls_cert_path))
+          tls_key_path = CGI.escape(config.fetch(:tls_key_path))
+
+          bind_string = "ssl://#{listen_address}:#{listen_port}?cert=#{tls_cert_path}&key=#{tls_key_path}"
+
+          set(:bind, bind_string)
         end
 
         def setup_probes(config)

data/lib/gitlab_exporter.rb

@@ -1,19 +1,20 @@
 module GitLab
   # GitLab Monitoring
   module Exporter
-    autoload :CLI,               "gitlab_exporter/cli"
-    autoload :TimeTracker,       "gitlab_exporter/util"
-    autoload :Utils,             "gitlab_exporter/util"
-    autoload :PrometheusMetrics, "gitlab_exporter/prometheus"
-    autoload :Utils,             "gitlab_exporter/util"
-    autoload :Git,               "gitlab_exporter/git"
-    autoload :GitProber,         "gitlab_exporter/git"
-    autoload :GitProcessProber,  "gitlab_exporter/git"
-    autoload :Database,          "gitlab_exporter/database"
-    autoload :ProcessProber,     "gitlab_exporter/process"
-    autoload :WebExporter,       "gitlab_exporter/web_exporter"
-    autoload :Prober,            "gitlab_exporter/prober"
-    autoload :SidekiqProber,     "gitlab_exporter/sidekiq"
-    autoload :RubyProber,        "gitlab_exporter/ruby"
+    autoload :CLI,                 "gitlab_exporter/cli"
+    autoload :TimeTracker,         "gitlab_exporter/util"
+    autoload :Utils,               "gitlab_exporter/util"
+    autoload :PrometheusMetrics,   "gitlab_exporter/prometheus"
+    autoload :Utils,               "gitlab_exporter/util"
+    autoload :Git,                 "gitlab_exporter/git"
+    autoload :GitProber,           "gitlab_exporter/git"
+    autoload :GitProcessProber,    "gitlab_exporter/git"
+    autoload :Database,            "gitlab_exporter/database"
+    autoload :ProcessProber,       "gitlab_exporter/process"
+    autoload :WebExporter,         "gitlab_exporter/web_exporter"
+    autoload :Prober,              "gitlab_exporter/prober"
+    autoload :SidekiqProber,       "gitlab_exporter/sidekiq"
+    autoload :RubyProber,          "gitlab_exporter/ruby"
+    autoload :ElasticsearchProber, "gitlab_exporter/elasticsearch"
   end
 end

data/spec/elasticsearch_spec.rb

@@ -0,0 +1,132 @@
+require "spec_helper"
+require "gitlab_exporter/elasticsearch"
+
+describe GitLab::Exporter::ElasticsearchProber do
+  subject(:prober) { GitLab::Exporter::Prober.new(**options) }
+  let(:options) do
+    {
+      elasticsearch: {
+        class_name: described_class.to_s,
+        methods: %i[probe_migrations],
+        opts: {
+          url: "http://elasticsearch"
+        }
+      }
+    }
+  end
+
+  let(:connection) { instance_double(Faraday::Connection) }
+
+  describe "probe_migrations" do
+    let(:response) { double(:response, body: results, status: 200) }
+    let(:metric_name) { "elasticsearch_migrations_info" }
+
+    let(:results) do
+      <<-RESULTS
+      {
+        "took": 5,
+        "timed_out": false,
+        "_shards": {
+          "total": 1,
+          "successful": 1,
+          "skipped": 0,
+          "failed": 0
+        },
+        "hits": {
+          "total": {
+            "value": 1,
+            "relation": "eq"
+          },
+          "max_score": 1,
+          "hits": [#{document.to_json}]
+        }
+      }
+      RESULTS
+    end
+
+    let(:document) do
+      { "_id" => "123", "_source" => migration }
+    end
+
+    let(:migration) do
+      {
+        "started_at" => "",
+        "completed_at" => "",
+        "completed" => false,
+        "state" => {}
+      }.tap do |m|
+        case migration_state
+        when "running"
+          m["started_at"] = "2021-09-23T11:47:48,890Z"
+        when "completed"
+          m["started_at"] = "2021-09-23T11:47:48,890Z"
+          m["completed_at"] = "2021-09-24T11:47:48,890Z"
+          m["completed"] = true
+        when "halted"
+          m["started_at"] = "2021-09-23T11:47:48,890Z"
+          m["state"]["halted"] = true
+        end
+      end
+    end
+
+    let(:output) do
+      prober.probe_all
+      StringIO.new.tap do |s|
+        prober.write_to(s)
+      end
+    end
+
+    before do
+      allow_any_instance_of(described_class).to receive(:elastic_probe).and_yield(connection)
+    end
+
+    describe "when an Exception is raised" do
+      before do
+        allow(connection).to receive(:get).with("/gitlab-*-migrations/_search").and_raise("boom")
+      end
+
+      it "does NOT raise an error" do
+        expect { output }.to_not raise_error
+      end
+
+      it "does NOT export any metrics" do
+        expect(output.string).not_to match(/#{metric_name}/)
+      end
+    end
+
+    describe "when elasticsearch is reachable" do
+      let(:expected_output) { /#{metric_name}{state="#{migration_state}",name="123"} 1 \d+/ }
+      before do
+        allow(connection).to receive(:get).with("/gitlab-*-migrations/_search").and_return response
+      end
+
+      context "when there is a pending migration" do
+        let(:migration_state) { "pending" }
+        it "exports state correctly" do
+          expect(output.string).to match(expected_output)
+        end
+      end
+
+      context "when there is a running migration" do
+        let(:migration_state) { "running" }
+        it "exports state correctly" do
+          expect(output.string).to match(expected_output)
+        end
+      end
+
+      context "when there is a halted migration" do
+        let(:migration_state) { "halted" }
+        it "exports state correctly" do
+          expect(output.string).to match(expected_output)
+        end
+      end
+
+      context "when there is a completed migration" do
+        let(:migration_state) { "completed" }
+        it "exports state correctly" do
+          expect(output.string).to match(expected_output)
+        end
+      end
+    end
+  end
+end

data/spec/integration/cli_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+require "gitlab_exporter/cli"
+
+module GitLab
+  module Exporter
+    module CLI
+      describe SidekiqRunner, :integration do
+        let(:redis_url) { ENV.fetch("REDIS_URL", "redis://localhost:6379") }
+        let(:io) { StringIO.new }
+
+        it "can properly reach out to redis" do
+          args = CLIArgs.new([io], options: { /^--redis-url/ => redis_url })
+          runner = SidekiqRunner.new(args)
+
+          expect { runner.run }.not_to raise_error
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -7,6 +7,10 @@ require "tmpdir"
 $LOAD_PATH.unshift File.expand_path(".")
 Dir["spec/support/**/*.rb"].each do |f| require f end
 
+RSpec.configure do |config|
+  config.filter_run_excluding integration: true
+end
+
 class GitRepoBuilder
   def origin
     @origin ||= create_origin
@@ -41,19 +45,30 @@ class GitRepoBuilder
 end
 
 class CLIArgs
-  def initialize(args)
+  def initialize(args, options: {})
     @arguments = args
+    @options = options
   end
 
   def options
     yield self
+
+    self
   end
 
-  def on(*args); end
+  def on(flag, *_)
+    match = @options.find { |regex, _| regex.match?(flag) }
+
+    yield match[1] if match
+  end
 
   def banner=(banner); end
 
   def parse!
     @arguments
   end
+
+  def shift
+    @arguments.shift
+  end
 end

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 11.4.0
+  version: 12.1.0
 platform: ruby
 authors:
 - Pablo Carranza
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
 date: 2016-07-27 00:00:00.000000000 Z
@@ -24,6 +24,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.2.5
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +58,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.3.2
+        version: 5.6.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.3.2
+        version: 5.6.5
 - !ruby/object:Gem::Dependency
   name: quantile
   requirement: !ruby/object:Gem::Requirement
@@ -72,56 +86,56 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.1.4
+        version: 4.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.1.4
+        version: 4.4.0
 - !ruby/object:Gem::Dependency
   name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.2.9
+        version: 6.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.2.9
+        version: 6.4.0
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -181,14 +195,17 @@ files:
 - lib/gitlab_exporter/database/remote_mirrors.rb
 - lib/gitlab_exporter/database/row_count.rb
 - lib/gitlab_exporter/database/tuple_stats.rb
+- lib/gitlab_exporter/elasticsearch.rb
 - lib/gitlab_exporter/git.rb
 - lib/gitlab_exporter/memstats.rb
 - lib/gitlab_exporter/memstats/mapping.rb
 - lib/gitlab_exporter/prober.rb
 - lib/gitlab_exporter/process.rb
 - lib/gitlab_exporter/prometheus.rb
+- lib/gitlab_exporter/rack_vulndb_255039_patch.rb
 - lib/gitlab_exporter/ruby.rb
 - lib/gitlab_exporter/sidekiq.rb
+- lib/gitlab_exporter/tls_helper.rb
 - lib/gitlab_exporter/util.rb
 - lib/gitlab_exporter/version.rb
 - lib/gitlab_exporter/web_exporter.rb
@@ -196,9 +213,11 @@ files:
 - spec/database/bloat_spec.rb
 - spec/database/ci_builds_spec.rb
 - spec/database/row_count_spec.rb
+- spec/elasticsearch_spec.rb
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
+- spec/integration/cli_spec.rb
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb
@@ -208,7 +227,7 @@ homepage: http://gitlab.com
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -223,8 +242,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: GitLab metrics exporter
 test_files:
@@ -232,9 +251,11 @@ test_files:
 - spec/database/bloat_spec.rb
 - spec/database/ci_builds_spec.rb
 - spec/database/row_count_spec.rb
+- spec/elasticsearch_spec.rb
 - spec/fixtures/smaps/sample.txt
 - spec/git_process_proper_spec.rb
 - spec/git_spec.rb
+- spec/integration/cli_spec.rb
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb