gitlab-exporter 16.1.0 → 16.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8f9e18ad6f1758709b718bb67c3491f654a01078aa7835462a0e346731adff9
-  data.tar.gz: affa763080262ed9621f7d51f8342aaebb2ca8cd5f6296b730a02a042d62b816
+  metadata.gz: f525375bfd6495a98af906badecaa459b07d6464327b47abd754e9a54b002522
+  data.tar.gz: 13388a0735f269ed0ad9c43a324e8e77a4c008ac2f8d9ec9a10a48a623886141
 SHA512:
-  metadata.gz: 7148d67304d3605314d65ce4c33a4450f33853a086da26150ee19532353db76d05d43326de068894f5d88ee879496ead866917ca2753e37623affab5e9b02292
-  data.tar.gz: 8d677307d6a128a01c695ce1cec716f31673032ad744bb600a89e713e9ae32ffa39e93ab22790d4e09fca6f485c1664e000ddf627d1bab31ad49e18d5e6f3891
+  metadata.gz: 0e754a5a16e88b93604313e719ae13441ffad1c4062d531c375aa11d7747a2aa4235037a42665d318b7534c3d6568ccc1f0c4fd8321cbed5fde3333e4a81bbbb
+  data.tar.gz: 230d9ac1d8925987027961bde296501e3dd665082f7f8140c29582997249f0342a721b58b5ad351e2c204c16afac46bdc1ee8ceed9095533a8614c7ad0f250b2

data/.gitlab-ci.yml

@@ -62,7 +62,7 @@ rspec_integration:
   services:
     - name: redis:${REDIS_VERSION}
       alias: redis-master
-    - name: bitnami/redis-sentinel:${REDIS_SENTINEL_VERSION}
+    - name: bitnamilegacy/redis-sentinel:${REDIS_SENTINEL_VERSION}
       alias: redis-sentinel
       command:
         - /bin/sh

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (16.1.0)
-      connection_pool (= 2.5.3)
+    gitlab-exporter (16.3.0)
+      connection_pool (= 2.5.4)
       deep_merge (~> 1.2.2)
-      faraday (= 2.13.4)
-      pg (= 1.6.1)
+      faraday (= 2.14.0)
+      pg (= 1.6.2)
       puma (= 6.6.1)
       quantile (= 0.2.1)
       redis (= 4.8.1)
@@ -19,10 +19,10 @@ GEM
   specs:
     ast (2.4.3)
     base64 (0.2.0)
-    connection_pool (2.5.3)
+    connection_pool (2.5.4)
     deep_merge (1.2.2)
     diff-lcs (1.5.0)
-    faraday (2.13.4)
+    faraday (2.14.0)
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger
@@ -38,7 +38,7 @@ GEM
     parser (3.3.9.0)
       ast (~> 2.4.1)
       racc
-    pg (1.6.1)
+    pg (1.6.2)
     prism (1.4.0)
     puma (6.6.1)
       nio4r (~> 2.0)
@@ -66,7 +66,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)
-    rubocop (1.79.1)
+    rubocop (1.81.6)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -74,10 +74,10 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.46.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.46.0)
+    rubocop-ast (1.47.1)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     ruby-progressbar (1.13.0)

data/gitlab-exporter.gemspec

@@ -22,10 +22,10 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = Gem::Requirement.new(">= 3.0")
 
-  s.add_runtime_dependency "connection_pool", "2.5.3"
+  s.add_runtime_dependency "connection_pool", "2.5.4"
   s.add_runtime_dependency "deep_merge", "~> 1.2.2"
-  s.add_runtime_dependency "faraday", "2.13.4"
-  s.add_runtime_dependency "pg", "1.6.1"
+  s.add_runtime_dependency "faraday", "2.14.0"
+  s.add_runtime_dependency "pg", "1.6.2"
   s.add_runtime_dependency "puma", "6.6.1"
   s.add_runtime_dependency "quantile", "0.2.1"
   s.add_runtime_dependency "redis", "4.8.1"

data/lib/gitlab_exporter/database/zoekt.rb

@@ -3,7 +3,8 @@ module GitLab
     module Database
       # A helper class to collect zoekt metrics.
       class ZoektCollector < Base
-        QUERY = <<~SQL.freeze
+        # Query to get processing zoekt_tasks distribution by zoekt_node_id
+        ZOEKT_TASKS_PROCESSING_QUERY = <<~SQL.freeze
           WITH task_counts AS (
             SELECT
               zoekt_node_id,
@@ -26,6 +27,34 @@ module GitLab
             task_counts tc ON n.id = tc.zoekt_node_id
         SQL
 
+        ZOEKT_NODES_QUERY = <<~SQL.freeze
+          SELECT schema_version, id FROM zoekt_nodes
+        SQL
+
+        ZOEKT_NODES_STATUS_QUERY = <<~SQL.freeze
+          SELECT
+            id,
+            CASE
+              WHEN last_seen_at < NOW() - INTERVAL '2 minute' THEN 0
+              ELSE 1
+            END AS status
+          FROM zoekt_nodes;
+        SQL
+
+        ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY = <<~SQL.freeze
+          SELECT
+            COUNT(*)
+          FROM
+            zoekt_repositories
+          INNER JOIN
+            zoekt_indices
+            ON zoekt_repositories.zoekt_index_id = zoekt_indices.id
+          WHERE
+            zoekt_indices.zoekt_node_id = $1
+            AND zoekt_repositories.schema_version < $2
+            AND zoekt_repositories.state = 10;
+        SQL
+
         ZOEKT_ENABLED_QUERY = <<~SQL.freeze
           SELECT
             zoekt_settings ->> 'zoekt_indexing_enabled' AS zoekt_indexing_enabled
@@ -35,13 +64,50 @@ module GitLab
         SQL
 
         def run
-          return unless zoekt_indexing_enabled?
+          return {} unless zoekt_indexing_enabled?
 
-          execute(QUERY, [Time.now.utc])
+          {
+            task_processing_query_result: execute(ZOEKT_TASKS_PROCESSING_QUERY, [Time.now.utc]),
+            repositories_schema_version_query_result: repositories_schema_version_query_result,
+            zoekt_nodes_status_query_result: execute(ZOEKT_NODES_STATUS_QUERY)
+          }.compact
         end
 
         private
 
+        def repositories_schema_version_query_result
+          hash = zoekt_node_schema_version_hash
+          return if hash.nil?
+
+          zoekt_repository_schema_version_result(hash)
+        end
+
+        def zoekt_node_schema_version_hash
+          zoekt_nodes = execute(ZOEKT_NODES_QUERY)
+          return if zoekt_nodes.nil?
+
+          hash = {}
+          zoekt_nodes.each do |row|
+            hash[row["id"]] = row["schema_version"]
+          end
+          hash
+        end
+
+        def zoekt_repository_schema_version_result(hash)
+          result = Set.new
+          hash.each do |id, schema_version|
+            query_result = execute(ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY, [id, schema_version])
+            if query_result.nil?
+              result = nil
+              break
+            end
+
+            count = query_result[0]["count"].to_i
+            result.add({ target_schema_version: schema_version, zoekt_node_id: id, unfinished_repository_count: count })
+          end
+          result
+        end
+
         def zoekt_indexing_enabled?
           @zoekt_indexing_enabled ||=
             begin
@@ -53,7 +119,7 @@ module GitLab
             end
         end
 
-        def execute(query, params)
+        def execute(query, params = [])
           with_connection_pool do |conn|
             conn.exec_params(query, params)
           end
@@ -64,9 +130,23 @@ module GitLab
 
       # The prober which is called when gathering metrics
       class ZoektProber
-        PrometheusMetrics.describe("search_zoekt_task_processing_queue_size",
-                                   "Number of tasks waiting to be processed by Zoekt",
-                                   "gauge")
+        PrometheusMetrics.describe(
+          "search_zoekt_task_processing_queue_size",
+          "Number of tasks waiting to be processed by Zoekt",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "search_zoekt_repositories_schema_version_count",
+          "Number of zoekt_repositories which do not have the latest schema version",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "search_zoekt_nodes_status",
+          "Status of each zoekt_node. 0 is offline (last seen more than 2 minutes ago), 1 is online",
+          "gauge"
+        )
 
         def initialize(metrics: PrometheusMetrics.new, **opts)
           @metrics = metrics
@@ -75,13 +155,14 @@ module GitLab
 
         def probe_db
           results = @collector.run
-          results.to_a.each do |row|
-            @metrics.add(
-              "search_zoekt_task_processing_queue_size",
-              row["task_count"].to_i,
-              node_name: row["node_name"],
-              node_id: row["node_id"]
-            )
+          results[:task_processing_query_result].to_a.each do |row|
+            add_processing_zoekt_tasks_to_metric(row)
+          end
+          results[:repositories_schema_version_query_result]&.each do |row|
+            add_zoekt_repositories_by_schema_version_to_metric(row)
+          end
+          results[:zoekt_nodes_status_query_result]&.each do |row|
+            add_zoekt_nodes_status_to_metric(row)
           end
 
           self
@@ -89,6 +170,32 @@ module GitLab
           self
         end
 
+        def add_processing_zoekt_tasks_to_metric(row)
+          @metrics.add(
+            "search_zoekt_task_processing_queue_size",
+            row["task_count"].to_i,
+            node_name: row["node_name"],
+            node_id: row["node_id"]
+          )
+        end
+
+        def add_zoekt_repositories_by_schema_version_to_metric(row)
+          @metrics.add(
+            "search_zoekt_repositories_schema_version_count",
+            row[:unfinished_repository_count].to_i,
+            target_schema_version: row[:target_schema_version],
+            zoekt_node_id: row[:zoekt_node_id]
+          )
+        end
+
+        def add_zoekt_nodes_status_to_metric(row)
+          @metrics.add(
+            "search_zoekt_nodes_status",
+            row["status"].to_i,
+            zoekt_node_id: row["id"]
+          )
+        end
+
         def write_to(target)
           target.write(@metrics.to_s)
         end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "16.1.0".freeze
+    VERSION = "16.3.0".freeze
   end
 end

data/spec/database/zoekt_spec.rb

@@ -4,8 +4,15 @@ require "gitlab_exporter/database/zoekt"
 describe GitLab::Exporter::Database::ZoektCollector do
   let(:connection_pool) { double("connection pool") }
   let(:connection) { double("connection") }
-  let(:query) { described_class::QUERY }
+  let(:zoekt_tasks_processing_query) { described_class::ZOEKT_TASKS_PROCESSING_QUERY }
+  let(:zoekt_nodes_query) { described_class::ZOEKT_NODES_QUERY }
+  let(:zoekt_nodes_query_results) { [{ "id" => "1", "schema_version" => "2302" }] }
   let(:zoekt_enabled_query) { described_class::ZOEKT_ENABLED_QUERY }
+  let(:zoekt_repository_schema_version_query) { described_class::ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY }
+  let(:zoekt_nodes_status_query) { described_class::ZOEKT_NODES_STATUS_QUERY }
+  let(:zoekt_nodes_status_query_results) { [{ "id" => "1", "status" => "1" }] }
+
+  let(:zoekt_repository_schema_version_query_results) { [{ "count" => "1" }] }
 
   subject(:collector) { described_class.new(connection_string: "host=localhost") }
 
@@ -26,13 +33,25 @@ describe GitLab::Exporter::Database::ZoektCollector do
   describe "#run" do
     let(:zoekt_enabled_results) {}
     let(:frozen_time) { Time.new(2023, 1, 1, 0, 0, 0, 0) }
-    let(:query_results) do
+    let(:zoekt_tasks_processing_query_results) do
       [
         { "node_id" => "1", "node_name" => "zoekt-1", "task_count" => "5" },
         { "node_id" => "2", "node_name" => "zoekt-2", "task_count" => "10" }
       ]
     end
 
+    let(:zoekt_repositories_schema_version_query_results) do
+      Set.new([{ target_schema_version: "2302", zoekt_node_id: "1", unfinished_repository_count: 1 }])
+    end
+
+    let(:result) do
+      {
+        repositories_schema_version_query_result: zoekt_repositories_schema_version_query_results,
+        task_processing_query_result: zoekt_tasks_processing_query_results,
+        zoekt_nodes_status_query_result: zoekt_nodes_status_query_results
+      }
+    end
+
     before do
       allow(Time).to receive(:now).and_return(frozen_time)
     end
@@ -42,10 +61,8 @@ describe GitLab::Exporter::Database::ZoektCollector do
         stub_zoekt_not_enabled
       end
 
-      it "returns nil" do
-        allow(connection).to receive(:exec_params).with(query, frozen_time).and_raise(PG::UndefinedTable)
-
-        expect(collector.run).to be_nil
+      it "returns {}" do
+        expect(collector.run).to eq({})
       end
     end
 
@@ -54,25 +71,43 @@ describe GitLab::Exporter::Database::ZoektCollector do
         stub_zoekt_enabled
       end
 
-      it "executes the query with the current time" do
-        expect(connection).to receive(:exec_params).with(query, [frozen_time]).and_return(query_results)
-
-        expect(collector.run).to eq(query_results)
+      it "executes the query with correct params" do
+        expect(connection).to receive(:exec_params).with(zoekt_tasks_processing_query, [frozen_time])
+                                                   .and_return(zoekt_tasks_processing_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_nodes_query, [])
+                                                   .and_return(zoekt_nodes_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_repository_schema_version_query, %w[1 2302])
+                                                   .and_return(zoekt_repository_schema_version_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_nodes_status_query, [])
+                                                   .and_return(zoekt_nodes_status_query_results)
+
+        expect(collector.run).to eq(result)
       end
 
       context "when PG::UndefinedTable is raised" do
-        it "returns nil" do
-          allow(connection).to receive(:exec_params).with(query, [frozen_time]).and_raise(PG::UndefinedTable)
-
-          expect(collector.run).to be_nil
+        it "does not includes the key which got the exception" do
+          allow(connection).to receive(:exec_params).with(zoekt_tasks_processing_query, [frozen_time])
+                                                    .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_nodes_query, [])
+                                                    .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_nodes_status_query, [])
+                                                    .and_raise(PG::UndefinedTable)
+
+          expect(collector.run).to eq({})
         end
       end
 
       context "when PG::UndefinedColumn is raised" do
-        it "returns nil" do
-          allow(connection).to receive(:exec_params).with(query, [frozen_time]).and_raise(PG::UndefinedColumn)
-
-          expect(collector.run).to be_nil
+        it "does not includes the key which got the exception" do
+          allow(connection).to receive(:exec_params).with(zoekt_tasks_processing_query, [frozen_time])
+                                                    .and_return(zoekt_tasks_processing_query_results)
+          expect(connection).to receive(:exec_params).with(zoekt_nodes_query, [])
+                                                     .and_return(zoekt_nodes_query_results)
+          expect(connection).to receive(:exec_params).with(zoekt_repository_schema_version_query, %w[1 2302])
+                                                     .and_raise(PG::UndefinedTable)
+          expect(connection).to receive(:exec_params).with(zoekt_nodes_status_query, []).and_raise(PG::UndefinedTable)
+
+          expect(collector.run).to eq({ task_processing_query_result: zoekt_tasks_processing_query_results })
         end
       end
     end
@@ -99,10 +134,15 @@ describe GitLab::Exporter::Database::ZoektProber do
   let(:metrics) { double("PrometheusMetrics", add: nil) }
   let(:collector) { double(GitLab::Exporter::Database::ZoektCollector, run: data) }
   let(:data) do
-    [
-      { "node_id" => "1", "node_name" => "zoekt-1", "task_count" => "5" },
-      { "node_id" => "2", "node_name" => "zoekt-2", "task_count" => "10" }
-    ]
+    {
+      repositories_schema_version_query_result: Set.new(
+        [{ target_schema_version: "2302", zoekt_node_id: "1", unfinished_repository_count: 1 }]
+      ),
+      task_processing_query_result: [
+        { "node_id" => "1", "node_name" => "zoekt-1", "task_count" => "5" },
+        { "node_id" => "2", "node_name" => "zoekt-2", "task_count" => "10" }
+      ]
+    }
   end
 
   describe "#probe_db" do
@@ -123,7 +163,7 @@ describe GitLab::Exporter::Database::ZoektProber do
     it "adds metrics for each node" do
       expect(collector).to receive(:run).and_return(data)
 
-      data.each do |node_data|
+      data[:task_processing_query_result].each do |node_data|
         expect(metrics).to receive(:add)
           .with("search_zoekt_task_processing_queue_size",
                 node_data["task_count"].to_i,
@@ -131,6 +171,16 @@ describe GitLab::Exporter::Database::ZoektProber do
                 node_id: node_data["node_id"])
       end
 
+      data[:repositories_schema_version_query_result].each do |node_data|
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_repositories_schema_version_count",
+            node_data[:unfinished_repository_count].to_i,
+            target_schema_version: node_data[:target_schema_version],
+            zoekt_node_id: node_data[:zoekt_node_id]
+          )
+      end
+
       probe_db
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 16.1.0
+  version: 16.3.0
 platform: ruby
 authors:
 - Pablo Carranza
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.5.3
+        version: 2.5.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.5.3
+        version: 2.5.4
 - !ruby/object:Gem::Dependency
   name: deep_merge
   requirement: !ruby/object:Gem::Requirement
@@ -44,28 +44,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.13.4
+        version: 2.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.13.4
+        version: 2.14.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 1.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.1
+        version: 1.6.2
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement