gitlab-exporter 14.5.0 → 15.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08d5fac10fb60048757b6f0ad48ac3672d34d64d01e096eac72b1e1d2e859ed2'
-  data.tar.gz: f33f247f22c6d9f1022d33e8d22b01ced782eea825cd22529cabcbe54e9f08f0
+  metadata.gz: 73d7d8a42035a6cb634f0c234e6bcd3e723492f2903663d7ecd0ccacde53279c
+  data.tar.gz: 012ce7e6313ec24c72c605bf016477c75d6de08ec2c37339e0a12c09c64aaa23
 SHA512:
-  metadata.gz: 155b43cb6cb074afd58462dc0c2f2e6bdef31310d3e8e8a9781340b1e07aab321ef2d5f5d5f9a95cd69e710aadb21d32cc7d6a5901d171c92f68259d95124f54
-  data.tar.gz: 237c25cfe1df48c137ba7f6a513e566f0f4bb0305f43c99e4a94014dc961a2f6deebd923b4adbfcedd8d0cd9ae893841013c290bbb7d127bd61765ecf3452d93
+  metadata.gz: dc31a10b7051bb5c76667d817cdd2761cd9ce9b755949567770b390f4362b37722d5658300b63607ff8f2566e577786ddb6e5b8edf3263bbbba3c478d28e08e7
+  data.tar.gz: 96b24d081ca6172197f6bdf44b739b3d053f32f2944378bb09609a02e485a053251c752ad49c921202eeee165437284aea73f875cce06a85edc455fb2d4e573d

data/.gitlab-ci.yml

@@ -50,9 +50,32 @@ rspec_integration:
     - bundle exec rspec spec -t integration -f d -c
   before_script: *before_scripts
   services:
-    - redis:latest
+    - name: redis:${REDIS_VERSION}
+      alias: redis-master
+    - name: bitnami/redis-sentinel:${REDIS_SENTINEL_VERSION}
+      alias: redis-sentinel
+      command:
+        - /bin/sh
+        - -c
+        - |
+          echo "Starting Redis Sentinel..."
+          cat <<EOF > /opt/bitnami/redis-sentinel/etc/sentinel.conf
+          user default off
+          user ${REDIS_SENTINEL_USERNAME} on >${REDIS_SENTINEL_PASSWORD} +@all
+          sentinel monitor mymaster redis-master 6379 2
+          sentinel resolve-hostnames yes
+          sentinel down-after-milliseconds mymaster 5000
+          EOF
+          redis-sentinel /opt/bitnami/redis-sentinel/etc/sentinel.conf
+
   variables:
     REDIS_URL: "redis://redis"
+    REDIS_SENTINEL_URL: "redis://mymaster"
+    REDIS_SENTINELS: "redis-sentinel:26379"
+    REDIS_VERSION: latest
+    REDIS_SENTINEL_VERSION: latest
+    REDIS_SENTINEL_USERNAME: "my-sentinel-username"
+    REDIS_SENTINEL_PASSWORD: "my-sentinel-password"
   parallel:
     matrix:
       - RUBY_VERSION: ["2.7", "3.0", "3.1", "3.2"]

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (14.5.0)
+    gitlab-exporter (15.1.0)
       connection_pool (= 2.2.5)
       deep_merge (~> 1.2.2)
       faraday (>= 1.8.0, <= 2.8.1)

data/README.md

@@ -19,6 +19,9 @@ metrics.
 1. Database
     * [Per-table tuple stats](lib/gitlab_exporter/database/tuple_stats.rb) --
       `gitlab_database_stat_table_*`
+   * [Per-sequence stats](lib/gitlab_exporter/database/pg_sequences.rb) --
+     `gitlab_pg_sequences_min_value`, `gitlab_pg_sequences_max_value`, `gitlab_pg_sequences_current_value`,
+     `gitlab_pg_sequences_saturation_ratio`
     * [Row count queries](lib/gitlab_exporter/database/row_count.rb) --
       `gitlab_database_rows`
     * [CI builds](lib/gitlab_exporter/database/ci_builds.rb) --

data/config/gitlab-exporter.yml.example

@@ -41,6 +41,10 @@ probes:
     class_name: Database::BloatProber
     <<: *db_common
 
+  pg_sequences:
+    class_name: Database::PgSequencesProber
+    <<: *db_common
+
   # We can group multiple probes under a single endpoint by setting the `multiple` key to `true`, followed
   # by probe definitions as usual.
   database:
@@ -98,6 +102,14 @@ probes:
       # redis_username: 'redis-username'
       # redis_password: 'redis-password'
       redis_enable_client: true
+      # Uncomment if Redis Sentinels are needed
+      # redis_sentinels:
+      #   - host: 'localhost'
+      #     port: 26380
+      #   - host: 'localhost'
+      #     port: 26381
+      # redis_sentinel_username: 'redis-sentinel-username'
+      # redis_sentinel_password: 'redis-sentinel-password'
 
   ruby: &ruby
     class_name: RubyProber

data/lib/gitlab_exporter/cli.rb

@@ -297,12 +297,24 @@ module GitLab
           @target = File.open(@target, "a") if @target.is_a?(String)
         end
 
-        def options(args)
+        def options(args) # rubocop:disable Metrics/MethodLength
           args.options do |opts|
             opts.banner = "Usage: #{EXECUTABLE_NAME} #{COMMAND_NAME} [options]"
             opts.on("--redis-url=\"redis://localhost:6379\"", "Redis URL") do |val|
               @redis_url = val
             end
+            opts.on("--redis-sentinel-username=\"my-sentinel-username\"", "Redis Sentinel username") do |val|
+              @redis_sentinel_username = val
+            end
+            opts.on("--redis-sentinel-password=\"my-sentinel-password\"", "Redis Sentinel password") do |val|
+              @redis_sentinel_password = val
+            end
+            opts.on("--redis-sentinels=HOST1:PORT1,HOST2:PORT2", Array, "List of Redis Sentinels") do |val|
+              @redis_sentinels = val.map { |item|
+                host, port = item.split(":")
+                { host: host, port: port.to_i }
+              }
+            end
           end
         end
 
@@ -313,13 +325,19 @@ module GitLab
         def run
           validate!
 
-          ::GitLab::Exporter::SidekiqProber.new(redis_url: @redis_url)
-                                           .probe_stats
-                                           .probe_queues
-                                           .probe_jobs_limit
-                                           .probe_workers
-                                           .probe_retries
-                                           .write_to(@target)
+          prober = ::GitLab::Exporter::SidekiqProber.new(redis_url: @redis_url,
+                                                         redis_sentinels: @redis_sentinels,
+                                                         redis_sentinel_username: @redis_sentinel_username,
+                                                         redis_sentinel_password: @redis_sentinel_password,
+                                                         logger: Logger.new(STDERR))
+
+          prober
+            .probe_stats
+            .probe_queues
+            .probe_jobs_limit
+            .probe_workers
+            .probe_retries
+            .write_to(@target)
         end
 
         private

data/lib/gitlab_exporter/database/base.rb

@@ -62,6 +62,10 @@ module GitLab
             conn.reset
             raise e
           end
+        rescue PG::ConnectionBad => e
+          @logger.error "Bad connection to the database, resetting pool: #{e}" if @logger
+          connection_pool.reload(&:close)
+          raise e
         rescue PG::Error => e
           @logger.error "Error connecting to the database: #{e}" if @logger
           raise e

data/lib/gitlab_exporter/database/pg_sequences.rb

@@ -0,0 +1,60 @@
+module GitLab
+  module Exporter
+    module Database
+      # A helper class to collect sequences metrics. Mainly used to monitor Cells sequences.
+      class PgSequencesCollector < Base
+        QUERY = <<~SQL.freeze
+          SELECT
+            schemaname,
+            sequencename,
+            CONCAT(schemaname, '.', sequencename) AS fully_qualified_sequencename,
+            start_value,
+            min_value,
+            max_value,
+            last_value AS current_value
+          FROM pg_catalog.pg_sequences;
+        SQL
+
+        def run
+          with_connection_pool do |conn|
+            conn.exec(QUERY).each.with_object({}) do |row, stats|
+              stats[row.delete("fully_qualified_sequencename")] = row
+            end
+          end
+        end
+      end
+
+      # The prober which is called when gathering metrics
+      class PgSequencesProber
+        METRIC_NAME = "gitlab_pg_sequences".freeze
+        METRIC_KEYS = %w[min_value max_value current_value].freeze
+
+        def initialize(metrics: PrometheusMetrics.new, **opts)
+          @metrics = metrics
+          @collector = PgSequencesCollector.new(**opts)
+        end
+
+        def probe_db
+          result = @collector.run
+
+          result.each do |fully_qualified_sequencename, sequence_info|
+            METRIC_KEYS.each do |key|
+              value = sequence_info.fetch(key).to_f
+              tags = {
+                schemaname: sequence_info.fetch("schemaname"),
+                sequencename: sequence_info.fetch("sequencename"),
+                fully_qualified_sequencename: fully_qualified_sequencename
+              }
+
+              @metrics.add("#{METRIC_NAME}_#{key}", value, **tags)
+            end
+          end
+
+          self
+        rescue PG::ConnectionBad
+          self
+        end
+      end
+    end
+  end
+end

data/lib/gitlab_exporter/database.rb

@@ -8,6 +8,7 @@ module GitLab
       autoload :RowCountProber,       "gitlab_exporter/database/row_count"
       autoload :BloatProber,          "gitlab_exporter/database/bloat"
       autoload :RemoteMirrorsProber,  "gitlab_exporter/database/remote_mirrors"
+      autoload :PgSequencesProber,    "gitlab_exporter/database/pg_sequences"
     end
   end
 end

data/lib/gitlab_exporter/sidekiq.rb

@@ -7,7 +7,7 @@ module GitLab
     # A prober for Sidekiq queues
     #
     # It takes the Redis URL Sidekiq is connected to
-    class SidekiqProber
+    class SidekiqProber # rubocop:disable Metrics/ClassLength
       # The maximum depth (from the head) of each queue to probe. Probing the
       # entirety of a very large queue will take longer and run the risk of
       # timing out. But when we have a very large queue, we are most in need of
@@ -195,6 +195,7 @@ module GitLab
       def redis_options
         options = {
           url: @opts[:redis_url],
+          sentinels: redis_sentinel_options,
           connect_timeout: 1,
           reconnect_attempts: 0
         }
@@ -207,6 +208,19 @@ module GitLab
         options
       end
 
+      def redis_sentinel_options
+        sentinels = @opts[:redis_sentinels]
+
+        return sentinels unless sentinels.is_a?(Array)
+
+        sentinels.each do |sentinel_config|
+          sentinel_config[:username] = @opts[:redis_sentinel_username] if @opts.key?(:redis_sentinel_username)
+          sentinel_config[:password] = @opts[:redis_sentinel_password] if @opts.key?(:redis_sentinel_password)
+        end
+
+        sentinels
+      end
+
       def redis_enable_client?
         return true if @opts[:redis_enable_client].nil?
 
@@ -221,6 +235,8 @@ module GitLab
         end
       rescue Redis::BaseConnectionError => e
         @logger&.error "Error connecting to the Redis: #{e}"
+        pool = self.class.connection_pool[redis_options]
+        pool.reload(&:close)
         @connected = false
       end
     end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "14.5.0".freeze
+    VERSION = "15.1.0".freeze
   end
 end

data/spec/database/ci_builds_spec.rb

@@ -290,6 +290,22 @@ describe GitLab::Exporter::Database do
           expect(output).to match(/^ci_stale_builds 0.0$/m)
         end
       end
+
+      context "when PG connection shuts down" do
+        before do
+          allow(connection).to receive(:exec).and_raise(PG::ConnectionBad)
+        end
+
+        it "responds with Prometheus metrics" do
+          expect(connection_pool).to receive(:reload)
+
+          prober.probe_db
+          prober.write_to(writer)
+          output = writer.string
+
+          expect(output).to be_empty
+        end
+      end
     end
 
     context "when executed on EE" do

data/spec/database/pg_sequences_spec.rb

@@ -0,0 +1,133 @@
+require "spec_helper"
+require "gitlab_exporter/database/pg_sequences"
+
+describe GitLab::Exporter::Database::PgSequencesCollector do
+  subject { described_class.new(connection_string: "") }
+
+  let(:connection_pool) { double("connection pool") }
+  let(:connection) { double("connection", exec: query_result) }
+  let(:type) { :btree }
+  let(:query_result) do
+    [
+      {
+        "schemaname" => "public",
+        "sequencename" => "seq_one",
+        "fully_qualified_sequencename" => "public.seq_one",
+        "min_value" => 1,
+        "max_value" => 9_223_372_036_854_775_807,
+        "current_value" => 6_223_372_036_854_745_121,
+        "saturation_ratio" => 0.6747e2
+      },
+      {
+        "schemaname" => "public",
+        "sequencename" => "seq_two",
+        "fully_qualified_sequencename" => "public.seq_two",
+        "min_value" => 1,
+        "max_value" => 9_223_372_036_854_775_807,
+        "current_value" => nil,
+        "saturation_ratio" => 0.0
+      }
+    ]
+  end
+
+  before do
+    allow_any_instance_of(described_class).to receive(:connection_pool).and_return(connection_pool)
+    allow(connection_pool).to receive(:with).and_yield(connection)
+  end
+
+  it "converts query results into a hash" do
+    expect(subject.run).to(
+      eq(
+        "public.seq_one" => {
+          "schemaname" => "public",
+          "sequencename" => "seq_one",
+          "min_value" => 1,
+          "max_value" => 9_223_372_036_854_775_807,
+          "current_value" => 6_223_372_036_854_745_121,
+          "saturation_ratio" => 0.6747e2
+        },
+        "public.seq_two" => {
+          "schemaname" => "public",
+          "sequencename" => "seq_two",
+          "min_value" => 1,
+          "max_value" => 9_223_372_036_854_775_807,
+          "current_value" => nil,
+          "saturation_ratio" => 0.0
+        }
+      )
+    )
+  end
+end
+
+describe GitLab::Exporter::Database::PgSequencesProber do
+  subject(:prober) { described_class.new(metrics: metrics, connection_string: "") }
+
+  let(:metrics) { double("PrometheusMetrics", add: nil) }
+  let(:collector) { double("PgSequencesCollector", run: data) }
+  let(:data) do
+    {
+      "public.seq_one" => {
+        "schemaname" => "public",
+        "sequencename" => "seq_one",
+        "min_value" => 1,
+        "max_value" => 10,
+        "current_value" => 2,
+        "saturation_ratio" => 0.2
+      }
+    }
+  end
+
+  describe "#probe_db" do
+    before do
+      allow(GitLab::Exporter::Database::PgSequencesCollector).to receive(:new).and_return(collector)
+    end
+
+    it "properly invokes the collector" do
+      expect(collector).to receive(:run)
+
+      prober.probe_db
+    end
+
+    it "adds min_value metric" do
+      expect(metrics).to(
+        receive(:add).with(
+          "gitlab_pg_sequences_min_value",
+          1.0,
+          schemaname: "public",
+          sequencename: "seq_one",
+          fully_qualified_sequencename: "public.seq_one"
+        )
+      )
+
+      prober.probe_db
+    end
+
+    it "adds max_value metric" do
+      expect(metrics).to(
+        receive(:add).with(
+          "gitlab_pg_sequences_max_value",
+          10.0,
+          schemaname: "public",
+          sequencename: "seq_one",
+          fully_qualified_sequencename: "public.seq_one"
+        )
+      )
+
+      prober.probe_db
+    end
+
+    it "adds current_value metric" do
+      expect(metrics).to(
+        receive(:add).with(
+          "gitlab_pg_sequences_current_value",
+          2.0,
+          schemaname: "public",
+          sequencename: "seq_one",
+          fully_qualified_sequencename: "public.seq_one"
+        )
+      )
+
+      prober.probe_db
+    end
+  end
+end

data/spec/integration/cli_spec.rb

@@ -6,6 +6,13 @@ module GitLab
     module CLI
       describe SidekiqRunner, :integration do
         let(:redis_url) { ENV.fetch("REDIS_URL", "redis://localhost:6379") }
+        let(:redis_sentinel_url) { ENV.fetch("REDIS_SENTINEL_URL", "redis://mymaster:6379") }
+        let(:redis_sentinel_username) { ENV.fetch("REDIS_SENTINEL_USERNAME", "my-sentinel-user") }
+        let(:redis_sentinel_password) { ENV.fetch("REDIS_SENTINEL_PASSWORD", "my-sentinel-password") }
+        let(:redis_sentinels) do
+          sentinels = ENV.fetch("REDIS_SENTINELS", "localhost:26379")
+          sentinels.split(" ")
+        end
         let(:io) { StringIO.new }
 
         it "can properly reach out to redis" do
@@ -14,6 +21,16 @@ module GitLab
 
           expect { runner.run }.not_to raise_error
         end
+
+        it "can properly reach out to redis-sentinel" do
+          args = CLIArgs.new([io], options: { /^--redis-url/ => redis_sentinel_url,
+                                              /^--redis-sentinel-username/ => redis_sentinel_username,
+                                              /^--redis-sentinel-password/ => redis_sentinel_password,
+                                              /^--redis-sentinels/ => redis_sentinels })
+          runner = SidekiqRunner.new(args)
+
+          expect { runner.run }.not_to raise_error
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 14.5.0
+  version: 15.1.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -227,6 +227,7 @@ files:
 - lib/gitlab_exporter/database/bloat_btree.sql
 - lib/gitlab_exporter/database/bloat_table.sql
 - lib/gitlab_exporter/database/ci_builds.rb
+- lib/gitlab_exporter/database/pg_sequences.rb
 - lib/gitlab_exporter/database/remote_mirrors.rb
 - lib/gitlab_exporter/database/row_count.rb
 - lib/gitlab_exporter/database/tuple_stats.rb
@@ -247,6 +248,7 @@ files:
 - spec/cli_spec.rb
 - spec/database/bloat_spec.rb
 - spec/database/ci_builds_spec.rb
+- spec/database/pg_sequences_spec.rb
 - spec/database/row_count_spec.rb
 - spec/elasticsearch_spec.rb
 - spec/fixtures/config.yml
@@ -286,6 +288,7 @@ test_files:
 - spec/cli_spec.rb
 - spec/database/bloat_spec.rb
 - spec/database/ci_builds_spec.rb
+- spec/database/pg_sequences_spec.rb
 - spec/database/row_count_spec.rb
 - spec/elasticsearch_spec.rb
 - spec/fixtures/config.yml