gitlab-exporter 16.4.1 → 16.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ddf051049df2fc97f6075da3453f40c99dbd5924613e52f7613f68f4061369d1
-  data.tar.gz: 2e5d19430d6de12d9ef679a74719e0047671955edf2a7a5cfbfa16128920493e
+  metadata.gz: 2ccb9b4710efb99847099a897fd1b8459c0d3e8d835b6f2ab4e8569d0a565242
+  data.tar.gz: c04beba06c2aa5866f5c09faf93f44cd14d01a0e5e174004ccaf81f8f9ba1171
 SHA512:
-  metadata.gz: 62fe67971ee82117f04fa6189867450469f67ba2818719ec4f0bec39a601cce5a47bfcecbaab02ce22c4f07fd16094a8e12d6fee258611555fd6f2605ae3f209
-  data.tar.gz: 706d36baae273e77e2211ade5dec5848316f1d5a473c33aa89e7e11ce20814bc5ffbd83735f3f20c1090c072b4fed5f4f056ab2472c2bd1ef2096b08f018151c
+  metadata.gz: ff2ca3937e3b3beebfdc5bf60cb967e16384ed50de585610aec8a86c5e2f985b35eba1f8108d99fad7fb7972b4683496e1cc70f17a791ec9847977f41613fe7e
+  data.tar.gz: 55d3cb8d8b971124f8f170a061efa23348f506477829cd59f2f7d9a7fc3902a04fc297b06c6638818b7283121106ab8e53bdfea75be00c05773f9a597ce7d1e5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-exporter (16.4.1)
+    gitlab-exporter (16.5.0)
       connection_pool (= 2.5.5)
       deep_merge (~> 1.2.2)
       faraday (= 2.14.0)

data/lib/gitlab_exporter/database/zoekt.rb

@@ -62,13 +62,23 @@ module GitLab
           LIMIT 1
         SQL
 
+        ZOEKT_NODE_STORAGE_QUERY = <<~SQL.freeze
+          SELECT
+            id,
+            metadata ->> 'name' AS node_name,
+            unclaimed_storage_bytes,
+            storage_percent_used
+          FROM zoekt_nodes
+        SQL
+
         def run
           return {} unless zoekt_indexing_enabled?
 
           {
             task_processing_query_result: execute(ZOEKT_TASKS_PROCESSING_QUERY, [Time.now.utc]),
             repositories_schema_version_query_result: repositories_schema_version_query_result,
-            zoekt_nodes_status_query_result: execute(ZOEKT_NODES_STATUS_QUERY)
+            zoekt_nodes_status_query_result: execute(ZOEKT_NODES_STATUS_QUERY),
+            node_storage_query_result: execute(ZOEKT_NODE_STORAGE_QUERY)
           }.compact
         end
 
@@ -150,6 +160,18 @@ module GitLab
           "gauge"
         )
 
+        PrometheusMetrics.describe(
+          "search_zoekt_node_unclaimed_storage_bytes",
+          "Unclaimed storage bytes for Zoekt node",
+          "gauge"
+        )
+
+        PrometheusMetrics.describe(
+          "search_zoekt_node_storage_percent_used",
+          "Fraction of storage used on Zoekt node (0..1)",
+          "gauge"
+        )
+
         def initialize(metrics: PrometheusMetrics.new, **opts)
           @metrics = metrics
           @collector = opts[:collector] || ZoektCollector.new(**opts)
@@ -157,21 +179,40 @@ module GitLab
 
         def probe_db
           results = @collector.run
-          results[:task_processing_query_result].to_a.each do |row|
-            add_processing_zoekt_tasks_to_metric(row)
-          end
-          results[:repositories_schema_version_query_result]&.each do |row|
-            add_zoekt_repositories_by_schema_version_to_metric(row)
-          end
-          results[:zoekt_nodes_status_query_result]&.each do |row|
-            add_zoekt_nodes_status_to_metric(row)
-          end
-
+          process_results(results)
           self
         rescue PG::ConnectionBad
           self
         end
 
+        private
+
+        def process_results(results)
+          process_task_results(results[:task_processing_query_result])
+          process_repository_results(results[:repositories_schema_version_query_result])
+          process_node_status_results(results[:zoekt_nodes_status_query_result])
+          process_node_storage_results(results[:node_storage_query_result])
+        end
+
+        def process_task_results(results)
+          results.to_a.each { |row| add_processing_zoekt_tasks_to_metric(row) }
+        end
+
+        def process_repository_results(results)
+          results&.each { |row| add_zoekt_repositories_by_schema_version_to_metric(row) }
+        end
+
+        def process_node_status_results(results)
+          results&.each { |row| add_zoekt_nodes_status_to_metric(row) }
+        end
+
+        def process_node_storage_results(results)
+          results&.each do |row|
+            add_zoekt_node_unclaimed_storage_to_metric(row)
+            add_zoekt_node_storage_percent_used_to_metric(row)
+          end
+        end
+
         def add_processing_zoekt_tasks_to_metric(row)
           @metrics.add(
             "search_zoekt_task_processing_queue_size",
@@ -206,6 +247,28 @@ module GitLab
           )
         end
 
+        def add_zoekt_node_unclaimed_storage_to_metric(row)
+          @metrics.add(
+            "search_zoekt_node_unclaimed_storage_bytes",
+            row["unclaimed_storage_bytes"].to_i,
+            **{
+              zoekt_node_id: row["id"],
+              zoekt_node_name: row["node_name"]
+            }.compact
+          )
+        end
+
+        def add_zoekt_node_storage_percent_used_to_metric(row)
+          @metrics.add(
+            "search_zoekt_node_storage_percent_used",
+            row["storage_percent_used"].to_f,
+            **{
+              zoekt_node_id: row["id"],
+              zoekt_node_name: row["node_name"]
+            }.compact
+          )
+        end
+
         def write_to(target)
           target.write(@metrics.to_s)
         end

data/lib/gitlab_exporter/version.rb

@@ -1,5 +1,5 @@
 module GitLab
   module Exporter
-    VERSION = "16.4.1".freeze
+    VERSION = "16.5.0".freeze
   end
 end

data/spec/database/zoekt_spec.rb

@@ -11,6 +11,10 @@ describe GitLab::Exporter::Database::ZoektCollector do
   let(:zoekt_repository_schema_version_query) { described_class::ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY }
   let(:zoekt_nodes_status_query) { described_class::ZOEKT_NODES_STATUS_QUERY }
   let(:zoekt_nodes_status_query_results) { [{ "id" => "1", "name" => "foo", "status" => "1" }] }
+  let(:zoekt_node_storage_query) { described_class::ZOEKT_NODE_STORAGE_QUERY }
+  let(:zoekt_node_storage_query_results) do
+    [{ "id" => "1", "node_name" => "foo", "unclaimed_storage_bytes" => "1000000", "storage_percent_used" => "0.75" }]
+  end
 
   let(:zoekt_repository_schema_version_query_results) { [{ "count" => "1" }] }
 
@@ -50,7 +54,8 @@ describe GitLab::Exporter::Database::ZoektCollector do
       {
         repositories_schema_version_query_result: zoekt_repositories_schema_version_query_results,
         task_processing_query_result: zoekt_tasks_processing_query_results,
-        zoekt_nodes_status_query_result: zoekt_nodes_status_query_results
+        zoekt_nodes_status_query_result: zoekt_nodes_status_query_results,
+        node_storage_query_result: zoekt_node_storage_query_results
       }
     end
 
@@ -82,6 +87,8 @@ describe GitLab::Exporter::Database::ZoektCollector do
                                                    .and_return(zoekt_repository_schema_version_query_results)
         expect(connection).to receive(:exec_params).with(zoekt_nodes_status_query, [])
                                                    .and_return(zoekt_nodes_status_query_results)
+        expect(connection).to receive(:exec_params).with(zoekt_node_storage_query, [])
+                                                   .and_return(zoekt_node_storage_query_results)
 
         expect(collector.run).to eq(result)
       end
@@ -94,6 +101,8 @@ describe GitLab::Exporter::Database::ZoektCollector do
                                                     .and_raise(PG::UndefinedTable)
           allow(connection).to receive(:exec_params).with(zoekt_nodes_status_query, [])
                                                     .and_raise(PG::UndefinedTable)
+          allow(connection).to receive(:exec_params).with(zoekt_node_storage_query, [])
+                                                    .and_raise(PG::UndefinedTable)
 
           expect(collector.run).to eq({})
         end
@@ -108,6 +117,7 @@ describe GitLab::Exporter::Database::ZoektCollector do
           expect(connection).to receive(:exec_params).with(zoekt_repository_schema_version_query, %w[1 2302])
                                                      .and_raise(PG::UndefinedTable)
           expect(connection).to receive(:exec_params).with(zoekt_nodes_status_query, []).and_raise(PG::UndefinedTable)
+          expect(connection).to receive(:exec_params).with(zoekt_node_storage_query, []).and_raise(PG::UndefinedColumn)
 
           expect(collector.run).to eq({ task_processing_query_result: zoekt_tasks_processing_query_results })
         end
@@ -143,6 +153,16 @@ describe GitLab::Exporter::Database::ZoektProber do
       task_processing_query_result: [
         { "node_id" => "1", "node_name" => "zoekt-1", "task_count" => "5" },
         { "node_id" => "2", "node_name" => "zoekt-2", "task_count" => "10" }
+      ],
+      node_storage_query_result: [
+        {
+          "id" => "1", "node_name" => "zoekt-1", "unclaimed_storage_bytes" => "1000000",
+          "storage_percent_used" => "0.75"
+        },
+        {
+          "id" => "2", "node_name" => "zoekt-2", "unclaimed_storage_bytes" => "2000000",
+          "storage_percent_used" => "0.50"
+        }
       ]
     }
   end
@@ -186,6 +206,23 @@ describe GitLab::Exporter::Database::ZoektProber do
           )
       end
 
+      data[:node_storage_query_result].each do |node_data|
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_node_unclaimed_storage_bytes",
+            node_data["unclaimed_storage_bytes"].to_i,
+            zoekt_node_id: node_data["id"],
+            zoekt_node_name: node_data["node_name"]
+          )
+        expect(metrics).to receive(:add)
+          .with(
+            "search_zoekt_node_storage_percent_used",
+            node_data["storage_percent_used"].to_f,
+            zoekt_node_id: node_data["id"],
+            zoekt_node_name: node_data["node_name"]
+          )
+      end
+
       probe_db
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-exporter
 version: !ruby/object:Gem::Version
-  version: 16.4.1
+  version: 16.5.0
 platform: ruby
 authors:
 - Pablo Carranza