gitlab-exporter 16.3.1 → 16.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfb5ffd66eceaaada65415073b18a42568888669d775689fbf2aaf3e58b02d7f
-  data.tar.gz: 2cda7d12814c7164425331e2be436000dc91ea8551387ce207bfafe49bcc69aa
+  metadata.gz: ddf051049df2fc97f6075da3453f40c99dbd5924613e52f7613f68f4061369d1
+  data.tar.gz: 2e5d19430d6de12d9ef679a74719e0047671955edf2a7a5cfbfa16128920493e
 SHA512:
-  metadata.gz: 0f91104bdeb03655fff74687478fcd81d1ffc03ac6f4be391c4bb9ff93e6b0171a57471aee54947b2536fe6cbdfd561ef0150fd7e306e3533ba3a2ea830b5d42
-  data.tar.gz: 195a6184776a297eadf4fa74883ff3780cd23f33a8d93fb67f19fcf79a202e818f88ac830656fb5d8fc9f64fd7378d6d4db6ae565b15866ad274d9fe7ccc2443
+  metadata.gz: 62fe67971ee82117f04fa6189867450469f67ba2818719ec4f0bec39a601cce5a47bfcecbaab02ce22c4f07fd16094a8e12d6fee258611555fd6f2605ae3f209
+  data.tar.gz: 706d36baae273e77e2211ade5dec5848316f1d5a473c33aa89e7e11ce20814bc5ffbd83735f3f20c1090c072b4fed5f4f056ab2472c2bd1ef2096b08f018151c

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (16.3.1)
+    gitlab-exporter (16.4.1)
       connection_pool (= 2.5.5)
       deep_merge (~> 1.2.2)
       faraday (= 2.14.0)
-      pg (= 1.6.2)
-      puma (= 7.1.0)
+      pg (= 1.6.3)
+      puma (= 7.2.0)
       quantile (= 0.2.1)
       redis (= 4.8.1)
       redis-namespace (= 1.11.0)
@@ -38,9 +38,9 @@ GEM
     parser (3.3.9.0)
       ast (~> 2.4.1)
       racc
-    pg (1.6.2)
-    prism (1.4.0)
-    puma (7.1.0)
+    pg (1.6.3)
+    prism (1.9.0)
+    puma (7.2.0)
       nio4r (~> 2.0)
     quantile (0.2.1)
     racc (1.8.1)
@@ -66,7 +66,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.81.7)
+    rubocop (1.84.1)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -74,12 +74,12 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.47.1, < 2.0)
+      rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.47.1)
+    rubocop-ast (1.49.0)
       parser (>= 3.3.7.2)
-      prism (~> 1.4)
+      prism (~> 1.7)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     sidekiq (6.5.12)

data/config/gitlab-exporter.yml.example

@@ -113,6 +113,11 @@ probes:
       #     port: 26381
       # redis_sentinel_username: 'redis-sentinel-username'
       # redis_sentinel_password: 'redis-sentinel-password'
+      # Uncomment if SSL parameters are needed
+      # redis_ssl_params:
+      #   ca_file: '/path/to/ca.crt'
+      #   cert: '/path/to/client.crt'
+      #   key: '/path/to/client.key'
 
   ruby: &ruby
     class_name: RubyProber

data/gitlab-exporter.gemspec

@@ -25,8 +25,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "connection_pool", "2.5.5"
   s.add_runtime_dependency "deep_merge", "~> 1.2.2"
   s.add_runtime_dependency "faraday", "2.14.0"
-  s.add_runtime_dependency "pg", "1.6.2"
-  s.add_runtime_dependency "puma", "7.1.0"
+  s.add_runtime_dependency "pg", "1.6.3"
+  s.add_runtime_dependency "puma", "7.2.0"
   s.add_runtime_dependency "quantile", "0.2.1"
   s.add_runtime_dependency "redis", "4.8.1"
   s.add_runtime_dependency "redis-namespace", "1.11.0"

data/lib/gitlab_exporter/database/zoekt.rb

@@ -28,12 +28,11 @@ module GitLab
         SQL
 
         ZOEKT_NODES_QUERY = <<~SQL.freeze
-          SELECT schema_version, id FROM zoekt_nodes
+          SELECT schema_version, id, metadata ->> 'name' AS node_name FROM zoekt_nodes
         SQL
 
         ZOEKT_NODES_STATUS_QUERY = <<~SQL.freeze
-          SELECT
-            id,
+          SELECT id, metadata ->> 'name' AS node_name,
             CASE
               WHEN last_seen_at < NOW() - INTERVAL '2 minute' THEN 0
               ELSE 1
@@ -88,22 +87,25 @@ module GitLab
 
           hash = {}
           zoekt_nodes.each do |row|
-            hash[row["id"]] = row["schema_version"]
+            hash[row["id"]] = [row["schema_version"], row["node_name"]]
           end
           hash
         end
 
         def zoekt_repository_schema_version_result(hash)
           result = Set.new
-          hash.each do |id, schema_version|
-            query_result = execute(ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY, [id, schema_version])
+          hash.each do |id, schema_version_name_arr|
+            query_result = execute(ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY, [id, schema_version_name_arr[0]])
             if query_result.nil?
               result = nil
               break
             end
-
-            count = query_result[0]["count"].to_i
-            result.add({ target_schema_version: schema_version, zoekt_node_id: id, unfinished_repository_count: count })
+            result.add(
+              {
+                target_schema_version: schema_version_name_arr[0], zoekt_node_id: id,
+                zoekt_node_name: schema_version_name_arr[1], unfinished_repository_count: query_result[0]["count"].to_i
+              }
+            )
           end
           result
         end
@@ -174,8 +176,10 @@ module GitLab
           @metrics.add(
             "search_zoekt_task_processing_queue_size",
             row["task_count"].to_i,
-            node_name: row["node_name"],
-            node_id: row["node_id"]
+            **{
+              node_name: row["node_name"],
+              node_id: row["node_id"]
+            }.compact
           )
         end
 
@@ -183,8 +187,11 @@ module GitLab
           @metrics.add(
             "search_zoekt_repositories_schema_version_count",
             row[:unfinished_repository_count].to_i,
-            target_schema_version: row[:target_schema_version],
-            zoekt_node_id: row[:zoekt_node_id]
+            **{
+              target_schema_version: row[:target_schema_version],
+              zoekt_node_id: row[:zoekt_node_id],
+              zoekt_node_name: row[:zoekt_node_name]
+            }.compact
           )
         end
 
@@ -192,7 +199,10 @@ module GitLab
           @metrics.add(
             "search_zoekt_nodes_status",
             row["status"].to_i,
-            zoekt_node_id: row["id"]
+            **{
+              zoekt_node_id: row["id"],
+              zoekt_node_name: row["node_name"]
+            }.compact
           )
         end
 

data/lib/gitlab_exporter/sidekiq.rb

@@ -1,3 +1,4 @@
+require "openssl"
 require "sidekiq/api"
 require "sidekiq/scheduled"
 require "digest"
@@ -204,6 +205,7 @@ module GitLab
           options[credential] = @opts[:"redis_#{credential}"] if @opts.key?(:"redis_#{credential}")
         end
 
+        options[:ssl_params] = parse_ssl_params(@opts[:redis_ssl_params]) if @opts.key?(:redis_ssl_params)
         options[:id] = nil unless redis_enable_client?
         options
       end
@@ -216,6 +218,9 @@ module GitLab
         sentinels.each do |sentinel_config|
           sentinel_config[:username] = @opts[:redis_sentinel_username] if @opts.key?(:redis_sentinel_username)
           sentinel_config[:password] = @opts[:redis_sentinel_password] if @opts.key?(:redis_sentinel_password)
+          if sentinel_config.key?(:ssl_params)
+            sentinel_config[:ssl_params] = parse_ssl_params(sentinel_config[:ssl_params])
+          end
         end
 
         sentinels
@@ -227,6 +232,27 @@ module GitLab
         @opts[:redis_enable_client]
       end
 
+      # redis v4.5 needs OpenSSL objects for the SSL parameters.
+      # redis v5 does not need to do this because is uses redis-client, which
+      # already does this: https://github.com/redis-rb/redis-client/blob/2b044dbf6266cbc0b0e52b4b9d6940dacb25ccf3/lib/redis_client/ruby_connection.rb#L14-L39
+      def parse_ssl_params(ssl_params)
+        params = ssl_params&.dup || {}
+
+        cert = params[:cert]
+        if cert.is_a?(String)
+          cert = File.read(cert) if File.exist?(cert)
+          params[:cert] = OpenSSL::X509::Certificate.new(cert)
+        end
+
+        key = params[:key]
+        if key.is_a?(String)
+          key = File.read(key) if File.exist?(key)
+          params[:key] = OpenSSL::PKey.read(key)
+        end
+
+        params
+      end
+
       def connected?
         return @connected unless @connected.nil?
 

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "16.3.1".freeze
+    VERSION = "16.4.1".freeze
   end
 end

data/spec/database/zoekt_spec.rb

@@ -6,11 +6,11 @@ describe GitLab::Exporter::Database::ZoektCollector do
   let(:connection) { double("connection") }
   let(:zoekt_tasks_processing_query) { described_class::ZOEKT_TASKS_PROCESSING_QUERY }
   let(:zoekt_nodes_query) { described_class::ZOEKT_NODES_QUERY }
-  let(:zoekt_nodes_query_results) { [{ "id" => "1", "schema_version" => "2302" }] }
+  let(:zoekt_nodes_query_results) { [{ "id" => "1", "schema_version" => "2302", "node_name" => "foo" }] }
   let(:zoekt_enabled_query) { described_class::ZOEKT_ENABLED_QUERY }
   let(:zoekt_repository_schema_version_query) { described_class::ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY }
   let(:zoekt_nodes_status_query) { described_class::ZOEKT_NODES_STATUS_QUERY }
-  let(:zoekt_nodes_status_query_results) { [{ "id" => "1", "status" => "1" }] }
+  let(:zoekt_nodes_status_query_results) { [{ "id" => "1", "name" => "foo", "status" => "1" }] }
 
   let(:zoekt_repository_schema_version_query_results) { [{ "count" => "1" }] }
 
@@ -41,7 +41,9 @@ describe GitLab::Exporter::Database::ZoektCollector do
     end
 
     let(:zoekt_repositories_schema_version_query_results) do
-      Set.new([{ target_schema_version: "2302", zoekt_node_id: "1", unfinished_repository_count: 1 }])
+      Set.new(
+        [{ target_schema_version: "2302", zoekt_node_id: "1", zoekt_node_name: "foo", unfinished_repository_count: 1 }]
+      )
     end
 
     let(:result) do
@@ -136,7 +138,7 @@ describe GitLab::Exporter::Database::ZoektProber do
   let(:data) do
     {
       repositories_schema_version_query_result: Set.new(
-        [{ target_schema_version: "2302", zoekt_node_id: "1", unfinished_repository_count: 1 }]
+        [{ target_schema_version: "2302", zoekt_node_id: "1", unfinished_repository_count: 1, zoekt_node_name: "foo" }]
       ),
       task_processing_query_result: [
         { "node_id" => "1", "node_name" => "zoekt-1", "task_count" => "5" },
@@ -165,10 +167,12 @@ describe GitLab::Exporter::Database::ZoektProber do
 
       data[:task_processing_query_result].each do |node_data|
         expect(metrics).to receive(:add)
-          .with("search_zoekt_task_processing_queue_size",
-                node_data["task_count"].to_i,
-                node_name: node_data["node_name"],
-                node_id: node_data["node_id"])
+          .with(
+            "search_zoekt_task_processing_queue_size",
+            node_data["task_count"].to_i,
+            node_name: node_data["node_name"],
+            node_id: node_data["node_id"]
+          )
       end
 
       data[:repositories_schema_version_query_result].each do |node_data|
@@ -177,7 +181,8 @@ describe GitLab::Exporter::Database::ZoektProber do
             "search_zoekt_repositories_schema_version_count",
             node_data[:unfinished_repository_count].to_i,
             target_schema_version: node_data[:target_schema_version],
-            zoekt_node_id: node_data[:zoekt_node_id]
+            zoekt_node_id: node_data[:zoekt_node_id],
+            zoekt_node_name: node_data[:zoekt_node_name]
           )
       end
 

data/spec/sidekiq_spec.rb

@@ -0,0 +1,94 @@
+require "spec_helper"
+require "gitlab_exporter/sidekiq"
+require "tempfile"
+require "openssl"
+
+describe GitLab::Exporter::SidekiqProber do
+  let(:metrics) { GitLab::Exporter::PrometheusMetrics.new }
+
+  def create_test_certificate_and_key
+    key = OpenSSL::PKey::RSA.new(2048)
+    cert = OpenSSL::X509::Certificate.new
+    cert.version = 2
+    cert.serial = 1
+    cert.subject = OpenSSL::X509::Name.parse("CN=localhost")
+    cert.issuer = cert.subject
+    cert.public_key = key.public_key
+    cert.not_before = Time.now
+    cert.not_after = Time.now + 365 * 24 * 60 * 60
+    cert.sign(key, OpenSSL::Digest.new("SHA256"))
+    [cert, key]
+  end
+
+  describe "#redis_options" do
+    it "includes ssl_params from file paths" do
+      cert, key = create_test_certificate_and_key
+
+      cert_file = Tempfile.new("cert.crt")
+      key_file = Tempfile.new("key.key")
+
+      cert_file.write(cert.to_pem)
+      cert_file.rewind
+      key_file.write(key.to_pem)
+      key_file.rewind
+
+      prober = described_class.new(
+        metrics: metrics,
+        redis_url: "rediss://localhost:6380",
+        redis_ssl_params: { cert: cert_file.path, key: key_file.path }
+      )
+
+      options = prober.send(:redis_options)
+
+      expect(options[:url]).to eq("rediss://localhost:6380")
+      expect(options[:ssl_params]).to be_a(Hash)
+      expect(options[:ssl_params][:cert]).to be_a(OpenSSL::X509::Certificate)
+      expect(options[:ssl_params][:key]).to be_a(OpenSSL::PKey::RSA)
+
+      cert_file.close
+      key_file.close
+    end
+
+    it "includes ssl_params when already parsed" do
+      cert, key = create_test_certificate_and_key
+
+      prober = described_class.new(
+        metrics: metrics,
+        redis_url: "rediss://localhost:6380",
+        redis_ssl_params: { cert: cert, key: key }
+      )
+
+      options = prober.send(:redis_options)
+
+      expect(options[:url]).to eq("rediss://localhost:6380")
+      expect(options[:ssl_params][:cert]).to equal(cert)
+      expect(options[:ssl_params][:key]).to equal(key)
+    end
+
+    it "excludes ssl_params when not provided" do
+      prober = described_class.new(
+        metrics: metrics,
+        redis_url: "redis://localhost:6379"
+      )
+
+      options = prober.send(:redis_options)
+
+      expect(options[:url]).to eq("redis://localhost:6379")
+      expect(options).not_to have_key(:ssl_params)
+    end
+
+    it "includes credentials when provided" do
+      prober = described_class.new(
+        metrics: metrics,
+        redis_url: "redis://localhost:6379",
+        redis_username: "user",
+        redis_password: "pass"
+      )
+
+      options = prober.send(:redis_options)
+
+      expect(options[:username]).to eq("user")
+      expect(options[:password]).to eq("pass")
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 16.3.1
+  version: 16.4.1
 platform: ruby
 authors:
 - Pablo Carranza
@@ -58,28 +58,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.2
+        version: 1.6.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.2
+        version: 1.6.3
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.1.0
+        version: 7.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 7.1.0
+        version: 7.2.0
 - !ruby/object:Gem::Dependency
   name: quantile
   requirement: !ruby/object:Gem::Requirement
@@ -255,6 +255,7 @@ files:
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb
+- spec/sidekiq_spec.rb
 - spec/spec_helper.rb
 - spec/util_spec.rb
 homepage: https://gitlab.com/gitlab-org/ruby/gems/gitlab-exporter
@@ -296,5 +297,6 @@ test_files:
 - spec/memstats_spec.rb
 - spec/prometheus_metrics_spec.rb
 - spec/ruby_spec.rb
+- spec/sidekiq_spec.rb
 - spec/spec_helper.rb
 - spec/util_spec.rb